feat(transport) : synchronisation du référentiel codes IDTF (ERP-149) (#101)

## ERP-149 — Récupération des codes IDTF (transport routier) > ⚠️ MR **empilée** sur `feat/erp-39-qualimat-sync` (PR #99), elle-même sur la PR #97. Ordre de merge : **#97 → #99 → celle-ci**. Les bases se recibleront automatiquement. Commande console `app:idtf:sync` : récupère l'export Excel des codes IDTF (régimes de nettoyage transport) depuis icrt-idtf.com, le parse et synchronise une table référentielle. Scope **road** ; discriminant `schema` road/water conservé pour un futur fluvial. ### Contenu - **Migration** `Version20260612160000` (namespace racine) : `idtf_product` + `idtf_sync_log`, `COMMENT ON COLUMN` sur chaque colonne, unique `(schema, idtf_number)`, `cas_numbers` JSONB, soft-delete. - **`IdtfSheetParser`** : parsing **pur** d'une matrice (sans dépendance PhpSpreadsheet) — détection **dynamique** de la ligne d'en-tête, mapping par libellé normalisé (résiste au réordonnancement), CAS split sur `;`, date `dd-mm-yyyy` → ISO + `checkdate`, skip des lignes non numériques. - **`SyncIdtfCommand`** : options `--schema` (road|water) / `--file` / `--dry-run`. POST avec les **10 `fields[]` explicites** (le piège `fields[]=all` ne sort que 6 colonnes) → export 11 colonnes ; garde-fou content-type/signature ZIP. Upsert DBAL transactionnel + soft-delete + journal. - Cible `make idtf-sync`. ### Tests - Unitaires (`IdtfSheetParser` : en-tête dynamique, mapping, CAS, date, skip, ordre de colonnes). - Fonctionnels de la commande via un `.xlsx` **généré** par PhpSpreadsheet (parsing → upsert → journal → soft-delete + schéma invalide rejeté). - Suite complète **608** verte (hors flaky JWT connu). `ColumnsHaveSqlCommentTest` ✅. - Bout-en-bout réel : sync de **687 codes IDTF** (road). ### Décisions - Migration **namespace racine** (convention réelle ; pas de FK cross-module). - **Aucun changement Composer** : `phpoffice/phpspreadsheet` était déjà une dépendance (^5.7) — le bump initial vers ^5.8 a été reverté. - Réutilise `framework.http_client` activé par la PR QUALIMAT (raison de l'empilement sur #99). --------- Co-authored-by: THOLOT DECHENE Matthieu <matthieu@yuno.malio.fr> Co-authored-by: Matthieu <contact@malio.fr> Reviewed-on: #101 Co-authored-by: tristan <tristan@yuno.malio.fr> Co-committed-by: tristan <tristan@yuno.malio.fr>
2026-06-15 15:45:23 +00:00
parent 4f8ed075b6
commit f9fec3e908
8 changed files with 946 additions and 3 deletions
@@ -0,0 +1,110 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Tests\Module\Transport\Application\Idtf;
+
+use App\Module\Transport\Application\Idtf\IdtfSheetParser;
+use PHPUnit\Framework\TestCase;
+use RuntimeException;
+
+/**
+ * @internal
+ */
+final class IdtfSheetParserTest extends TestCase
+{
+    public function testExtractsExportDate(): void
+    {
+        $parsed = IdtfSheetParser::parse($this->sampleMatrix());
+        self::assertSame('2026-06-12', $parsed['exportDate']);
+    }
+
+    public function testParsesAndNormalizesFirstRow(): void
+    {
+        $parsed = IdtfSheetParser::parse($this->sampleMatrix());
+        $row    = $parsed['rows'][0];
+
+        self::assertSame(30748, $row['idtf_number']);
+        self::assertSame('Argiles avec régime de nettoyage C', $row['name']);
+        self::assertSame('C', $row['cleaning_regime']);
+        self::assertSame('2026-04-02', $row['mandatory_date']);
+        self::assertSame('Al2O3', $row['formula']);
+        self::assertSame('01 01 01', $row['eural_code']);
+        self::assertSame(['7631-86-9', '1344-28-1'], $row['cas_numbers']);
+        self::assertSame('Note 1', $row['footnotes']);
+    }
+
+    public function testSkipsEmptyAndNonNumericRows(): void
+    {
+        $parsed = IdtfSheetParser::parse($this->sampleMatrix());
+
+        // 2 lignes exploitables (30748 et 30744) ; vide + "abc" ignorees.
+        self::assertCount(2, $parsed['rows']);
+        self::assertSame(30744, $parsed['rows'][1]['idtf_number']);
+    }
+
+    public function testEmptyOptionalCellsBecomeNullAndCasEmpty(): void
+    {
+        $parsed = IdtfSheetParser::parse($this->sampleMatrix());
+        $row    = $parsed['rows'][1]; // 30744
+
+        self::assertNull($row['mandatory_date']);
+        self::assertNull($row['formula']);
+        self::assertNull($row['product_group']);
+        self::assertSame([], $row['cas_numbers']);
+    }
+
+    public function testColumnOrderIsResolvedByLabel(): void
+    {
+        // En-tete dans un ordre different : le mapping doit suivre les libelles.
+        $matrix = [
+            ['Export date: 1-1-2026'],
+            ['Numéro CAS', 'Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'],
+            ['7440-44-0', '99', 'Carbone', 'B'],
+        ];
+
+        $parsed = IdtfSheetParser::parse($matrix);
+        $row    = $parsed['rows'][0];
+
+        self::assertSame(99, $row['idtf_number']);
+        self::assertSame('Carbone', $row['name']);
+        self::assertSame('B', $row['cleaning_regime']);
+        self::assertSame(['7440-44-0'], $row['cas_numbers']);
+    }
+
+    public function testThrowsWhenHeaderMissing(): void
+    {
+        $this->expectException(RuntimeException::class);
+        IdtfSheetParser::parse([['foo', 'bar'], ['1', '2']]);
+    }
+
+    public function testExportDateNullWhenAbsent(): void
+    {
+        $matrix = [
+            ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'],
+            ['1', 'X', 'A'],
+        ];
+
+        self::assertNull(IdtfSheetParser::parse($matrix)['exportDate']);
+    }
+
+    /**
+     * Matrice representative de l'export reel : preambule (lignes 0-1), ligne
+     * vide (2), en-tete (3) puis donnees.
+     *
+     * @return array<int, array<int, mixed>>
+     */
+    private function sampleMatrix(): array
+    {
+        return [
+            ['Export date: 12-6-2026'],
+            ['Changes in the database after this date...'],
+            [],
+            ['Numéro IDTF', 'Product Group', 'Nom de la marchandise', 'Régime de nettoyage', 'Exigences importantes', 'Date d’application obligatoire', 'Produits apparentés', 'Formule', 'Code EURAL', 'Numéro CAS', 'Annotations'],
+            ['30748', 'Substances inorganiques', 'Argiles avec régime de nettoyage C', 'C', 'Exigence X', '02-04-2026', 'Poudre argile', 'Al2O3', '01 01 01', '7631-86-9 ; 1344-28-1', 'Note 1'],
+            ['', '', '', '', '', '', '', '', '', '', ''],
+            ['abc', 'ligne non numerique a ignorer', '', '', '', '', '', '', '', '', ''],
+            ['30744', '', 'Additifs alimentaires', 'A', '', '', '', '', '', '', ''],
+        ];
+    }
+}
@@ -0,0 +1,163 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Tests\Module\Transport\Infrastructure\Console;
+
+use Doctrine\DBAL\Connection;
+use PhpOffice\PhpSpreadsheet\Spreadsheet;
+use PhpOffice\PhpSpreadsheet\Writer\Xlsx;
+use Symfony\Bundle\FrameworkBundle\Console\Application;
+use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
+use Symfony\Component\Console\Tester\CommandTester;
+
+/**
+ * Test fonctionnel de `app:idtf:sync` via --file : genere un vrai .xlsx, le
+ * passe a la commande et verifie le parsing, l'upsert, le journal et le
+ * soft-delete (chemin complet IOFactory -> parser -> DBAL).
+ *
+ * @internal
+ */
+final class SyncIdtfCommandTest extends KernelTestCase
+{
+    private Connection $connection;
+
+    protected function setUp(): void
+    {
+        self::bootKernel();
+
+        /** @var Connection $connection */
+        $connection       = self::getContainer()->get('doctrine.dbal.default_connection');
+        $this->connection = $connection;
+        $this->purge();
+    }
+
+    protected function tearDown(): void
+    {
+        $this->purge();
+        parent::tearDown();
+    }
+
+    public function testSyncParsesXlsxUpsertsAndLogs(): void
+    {
+        $path = $this->makeXlsx([
+            ['Export date: 12-6-2026'],
+            ['Avertissement preambule'],
+            [],
+            ['Numéro IDTF', 'Product Group', 'Nom de la marchandise', 'Régime de nettoyage', 'Exigences importantes', 'Date d’application obligatoire', 'Produits apparentés', 'Formule', 'Code EURAL', 'Numéro CAS', 'Annotations'],
+            ['30748', 'Inorganiques', 'Argiles régime C', 'C', 'Exig X', '02-04-2026', 'Poudre', 'Al2O3', '01 01 01', '7631-86-9 ; 1344-28-1', 'Note'],
+            ['', '', '', '', '', '', '', '', '', '', ''],
+            ['30744', '', 'Additifs', 'A', '', '', '', '', '', '', ''],
+        ]);
+
+        $tester = $this->runSync($path);
+        $tester->assertCommandIsSuccessful();
+
+        self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product'));
+
+        $row = $this->connection->fetchAssociative("SELECT * FROM idtf_product WHERE idtf_number = 30748 AND schema = 'road'");
+        self::assertNotFalse($row);
+        self::assertSame('Argiles régime C', $row['name']);
+        self::assertSame('C', $row['cleaning_regime']);
+        self::assertSame('2026-04-02', $row['mandatory_date']);
+        self::assertSame('2026-06-12', $row['source_export_date']);
+        self::assertSame(['7631-86-9', '1344-28-1'], json_decode((string) $row['cas_numbers'], true));
+
+        $log = $this->connection->fetchAssociative('SELECT * FROM idtf_sync_log ORDER BY id DESC LIMIT 1');
+        self::assertNotFalse($log);
+        self::assertSame('road', $log['schema']);
+        self::assertSame('2026-06-12', $log['export_date']);
+        self::assertSame(2, (int) $log['rows_total']);
+        self::assertSame(2, (int) $log['rows_upserted']);
+        self::assertSame(0, (int) $log['rows_deactivated']);
+    }
+
+    public function testSecondSyncSoftDeletesMissing(): void
+    {
+        $header = ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'];
+
+        $this->runSync($this->makeXlsx([
+            ['Export date: 1-6-2026'],
+            $header,
+            ['100', 'Produit 100', 'A'],
+            ['200', 'Produit 200', 'B'],
+        ]))->assertCommandIsSuccessful();
+        self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE is_active = TRUE'));
+
+        // 2e export sans 200 -> soft-delete de 200, mise a jour de 100.
+        $tester = $this->runSync($this->makeXlsx([
+            ['Export date: 2-6-2026'],
+            $header,
+            ['100', 'Produit 100 maj', 'C'],
+        ]));
+        $tester->assertCommandIsSuccessful();
+
+        self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product'));
+        self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE is_active = TRUE'));
+        self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE idtf_number = 200 AND is_active = FALSE'));
+
+        $row100 = $this->connection->fetchAssociative('SELECT * FROM idtf_product WHERE idtf_number = 100');
+        self::assertNotFalse($row100);
+        self::assertSame('Produit 100 maj', $row100['name']);
+        self::assertSame('C', $row100['cleaning_regime']);
+
+        $log = $this->connection->fetchAssociative('SELECT * FROM idtf_sync_log ORDER BY id DESC LIMIT 1');
+        self::assertNotFalse($log);
+        self::assertSame(1, (int) $log['rows_upserted']);
+        self::assertSame(1, (int) $log['rows_deactivated']);
+    }
+
+    public function testInvalidSchemaIsRejected(): void
+    {
+        $path = $this->makeXlsx([
+            ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'],
+            ['1', 'X', 'A'],
+        ]);
+
+        $application = new Application(self::$kernel);
+        $tester      = new CommandTester($application->find('app:idtf:sync'));
+        $exitCode    = $tester->execute(['--file' => $path, '--schema' => 'air']);
+
+        @unlink($path);
+
+        self::assertSame(2, $exitCode); // Command::INVALID
+        self::assertSame(0, $this->countRows('SELECT COUNT(*) FROM idtf_product'));
+    }
+
+    /**
+     * @param array<int, array<int, mixed>> $matrix
+     */
+    private function makeXlsx(array $matrix): string
+    {
+        $spreadsheet = new Spreadsheet();
+        $spreadsheet->getActiveSheet()->fromArray($matrix, null, 'A1', true);
+
+        $path = tempnam(sys_get_temp_dir(), 'idtf_').'.xlsx';
+        new Xlsx($spreadsheet)->save($path);
+        $spreadsheet->disconnectWorksheets();
+
+        return $path;
+    }
+
+    private function runSync(string $path): CommandTester
+    {
+        $application = new Application(self::$kernel);
+        $tester      = new CommandTester($application->find('app:idtf:sync'));
+        $tester->execute(['--file' => $path, '--schema' => 'road']);
+
+        @unlink($path);
+
+        return $tester;
+    }
+
+    private function countRows(string $sql): int
+    {
+        return (int) $this->connection->fetchOne($sql);
+    }
+
+    private function purge(): void
+    {
+        $this->connection->executeStatement('DELETE FROM idtf_product');
+        $this->connection->executeStatement('DELETE FROM idtf_sync_log');
+    }
+}