From 5f3da7022b7cf8b2aca27ca0fd3c7c01254bcd0d Mon Sep 17 00:00:00 2001 From: tristan Date: Fri, 12 Jun 2026 14:35:16 +0200 Subject: [PATCH 1/7] =?UTF-8?q?feat(transport)=20:=20cr=C3=A9er=20le=20mod?= =?UTF-8?q?ule=20Transport=20(ERP-150)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Module Transport (ID transport, non requis) destiné à héberger les référentiels externes synchronisés par commandes console (codes IDTF ERP-149, transporteurs QUALIMAT ERP-39). - TransportModule.php avec permissions() vide à ce stade - activation dans config/modules.php - layer Nuxt front minimal (pas d'écran ni d'item sidebar) --- config/modules.php | 2 ++ frontend/modules/transport/nuxt.config.ts | 1 + src/Module/Transport/TransportModule.php | 29 +++++++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 frontend/modules/transport/nuxt.config.ts create mode 100644 src/Module/Transport/TransportModule.php diff --git a/config/modules.php b/config/modules.php index c4f8f54..e55d2ed 100644 --- a/config/modules.php +++ b/config/modules.php @@ -5,10 +5,12 @@ use App\Module\Catalog\CatalogModule; use App\Module\Commercial\CommercialModule; use App\Module\Core\CoreModule; use App\Module\Sites\SitesModule; +use App\Module\Transport\TransportModule; return [ CoreModule::class, CommercialModule::class, SitesModule::class, CatalogModule::class, + TransportModule::class, ]; diff --git a/frontend/modules/transport/nuxt.config.ts b/frontend/modules/transport/nuxt.config.ts new file mode 100644 index 0000000..268da7f --- /dev/null +++ b/frontend/modules/transport/nuxt.config.ts @@ -0,0 +1 @@ +export default defineNuxtConfig({}) diff --git a/src/Module/Transport/TransportModule.php b/src/Module/Transport/TransportModule.php new file mode 100644 index 0000000..f7248ba --- /dev/null +++ b/src/Module/Transport/TransportModule.php @@ -0,0 +1,29 @@ + + */ + public static function permissions(): array + { + return []; + } +} -- 2.39.5 From b4440612376396f057f0fc7800858ec9e06e34b8 Mon Sep 17 00:00:00 2001 From: tristan Date: Fri, 12 Jun 2026 15:03:28 +0200 Subject: [PATCH 2/7] =?UTF-8?q?feat(transport)=20:=20synchronisation=20du?= =?UTF-8?q?=20r=C3=A9f=C3=A9rentiel=20transporteurs=20QUALIMAT=20(ERP-39)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commande console app:qualimat:sync : récupère les opérateurs de transport agréés depuis l'API publique qualimat.org, normalise et synchronise une table référentielle (upsert sur le SIRET + soft-delete des absents + journal). Prévue pour un cron quotidien. - migration : tables qualimat_carrier + qualimat_sync_log (COMMENT ON COLUMN sur chaque colonne) - QualimatRowMapper : normalisation pure (SIRET sans espaces, date dd/mm/yyyy -> ISO, skip sans SIRET) + tests unitaires - SyncQualimatCommand : options --file / --ppp / --dry-run, upsert DBAL transactionnel - activation de framework.http_client - tests fonctionnels de la commande (upsert/normalisation/journal/soft-delete) --- config/packages/http_client.yaml | 9 + migrations/Version20260612150000.php | 111 ++++++++ .../Qualimat/QualimatRowMapper.php | 124 +++++++++ .../Console/SyncQualimatCommand.php | 250 ++++++++++++++++++ .../Qualimat/QualimatRowMapperTest.php | 90 +++++++ .../Console/SyncQualimatCommandTest.php | 138 ++++++++++ 6 files changed, 722 insertions(+) create mode 100644 config/packages/http_client.yaml create mode 100644 migrations/Version20260612150000.php create mode 100644 src/Module/Transport/Application/Qualimat/QualimatRowMapper.php create mode 100644 src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php create mode 100644 tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php create mode 100644 tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php diff --git a/config/packages/http_client.yaml b/config/packages/http_client.yaml new file mode 100644 index 0000000..66ac52b --- /dev/null +++ b/config/packages/http_client.yaml @@ -0,0 +1,9 @@ +# Active le composant HTTP Client (symfony/http-client) et enregistre +# l'autowiring de HttpClientInterface. Utilise par les commandes de +# synchronisation de referentiels externes (QUALIMAT, IDTF...). +framework: + http_client: + default_options: + timeout: 30 + headers: + User-Agent: 'Starseed-ERP (referentiel-sync)' diff --git a/migrations/Version20260612150000.php b/migrations/Version20260612150000.php new file mode 100644 index 0000000..f82f5d7 --- /dev/null +++ b/migrations/Version20260612150000.php @@ -0,0 +1,111 @@ +addSql(<<<'SQL' + CREATE TABLE qualimat_carrier ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + siret VARCHAR(20) NOT NULL, + name VARCHAR(255) NOT NULL, + address VARCHAR(255) DEFAULT NULL, + postal_code VARCHAR(10) DEFAULT NULL, + city VARCHAR(255) DEFAULT NULL, + phone VARCHAR(32) DEFAULT NULL, + department VARCHAR(64) DEFAULT NULL, + status VARCHAR(32) NOT NULL, + validity_date DATE DEFAULT NULL, + is_active BOOLEAN DEFAULT TRUE NOT NULL, + last_synced_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + PRIMARY KEY (id), + CONSTRAINT uq_qualimat_carrier_siret UNIQUE (siret) + ) + SQL); + $this->addSql('CREATE INDEX idx_qualimat_carrier_active ON qualimat_carrier (is_active)'); + + $this->comment('qualimat_carrier', '_table', "Referentiel des transporteurs agrees QUALIMAT, synchronise quotidiennement depuis l'API qualimat.org (type=operateur_transport)."); + $this->comment('qualimat_carrier', 'id', 'Cle technique auto-incrementee.'); + $this->comment('qualimat_carrier', 'siret', 'SIRET normalise (chiffres sans espaces). Cle naturelle de synchro (unique). Source parfois incomplete (longueur variable), non contrainte a 14.'); + $this->comment('qualimat_carrier', 'name', 'Raison sociale du transporteur (champs Nom = Societe de la source, identiques).'); + $this->comment('qualimat_carrier', 'address', 'Adresse postale (voie). Nullable.'); + $this->comment('qualimat_carrier', 'postal_code', 'Code postal. Nullable.'); + $this->comment('qualimat_carrier', 'city', 'Ville. Nullable.'); + $this->comment('qualimat_carrier', 'phone', 'Telephone au format source "indicatif|numero" (ex: +33|0608890316). Nullable.'); + $this->comment('qualimat_carrier', 'department', 'Departement au format source "code - libelle" (ex: 65 - Hautes-Pyrenees). Nullable.'); + $this->comment('qualimat_carrier', 'status', "Statut d'agrement QUALIMAT (valeurs connues : Audite, Valide, Suspendu). Valeur brute de la source, non contrainte."); + $this->comment('qualimat_carrier', 'validity_date', 'Date de fin de validite de la certification (convertie depuis dd/mm/yyyy). Nullable.'); + $this->comment('qualimat_carrier', 'is_active', 'Faux = transporteur absent du dernier import (soft-delete). Toute ligne non revue par le dernier run passe a FALSE.'); + $this->comment('qualimat_carrier', 'last_synced_at', 'Horodatage du run de synchro ayant vu cette ligne en dernier (soft-delete : last_synced_at < run courant).'); + + $this->addSql(<<<'SQL' + CREATE TABLE qualimat_sync_log ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + fetched_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + rows_total INT NOT NULL, + rows_upserted INT NOT NULL, + rows_skipped INT NOT NULL, + rows_deactivated INT NOT NULL, + created_at TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW() NOT NULL, + PRIMARY KEY (id) + ) + SQL); + + $this->comment('qualimat_sync_log', '_table', 'Journal des synchronisations QUALIMAT (une ligne par run de la commande app:qualimat:sync).'); + $this->comment('qualimat_sync_log', 'id', 'Cle technique auto-incrementee.'); + $this->comment('qualimat_sync_log', 'fetched_at', "Horodatage de l'appel a l'API source (= run de synchro)."); + $this->comment('qualimat_sync_log', 'rows_total', "Nombre d'items renvoyes par l'API."); + $this->comment('qualimat_sync_log', 'rows_upserted', 'Nombre de transporteurs inseres ou mis a jour.'); + $this->comment('qualimat_sync_log', 'rows_skipped', "Nombre d'items ignores (sans SIRET exploitable)."); + $this->comment('qualimat_sync_log', 'rows_deactivated', 'Nombre de transporteurs passes a is_active=false (absents de cet import).'); + $this->comment('qualimat_sync_log', 'created_at', 'Horodatage de fin du run (insertion du journal).'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS qualimat_sync_log'); + $this->addSql('DROP TABLE IF EXISTS qualimat_carrier'); + } + + /** + * Pose un COMMENT ON TABLE/COLUMN en dollar-quoting Postgres ($_$...$_$) + * pour eviter tout echappement d'apostrophes dans les descriptions. + */ + private function comment(string $table, string $column, string $description): void + { + $quotedTable = '"'.str_replace('"', '""', $table).'"'; + + if ('_table' === $column) { + $this->addSql(sprintf('COMMENT ON TABLE %s IS $_$%s$_$', $quotedTable, $description)); + + return; + } + + $this->addSql(sprintf( + 'COMMENT ON COLUMN %s.%s IS $_$%s$_$', + $quotedTable, + '"'.str_replace('"', '""', $column).'"', + $description, + )); + } +} diff --git a/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php new file mode 100644 index 0000000..91bb31a --- /dev/null +++ b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php @@ -0,0 +1,124 @@ +> $items + * + * @return array{rows: list>, skipped: int} + */ + public static function mapMany(array $items): array + { + $rows = []; + $skipped = 0; + + foreach ($items as $item) { + $row = self::mapOne($item); + + if (null === $row) { + ++$skipped; + + continue; + } + + $rows[] = $row; + } + + return ['rows' => $rows, 'skipped' => $skipped]; + } + + /** + * Mappe un item unique. Retourne null si le SIRET est absent ou vide + * (ligne inexploitable : pas de cle naturelle pour l'upsert). + * + * @param array $item + * + * @return null|array + */ + public static function mapOne(array $item): ?array + { + $siret = self::normalizeSiret(self::str($item['Siret'] ?? null)); + + if (null === $siret) { + return null; + } + + return [ + 'siret' => $siret, + // Nom et Societe sont identiques a la source : une seule colonne. + 'name' => self::str($item['Nom'] ?? null) ?? '', + 'address' => self::str($item['Adresse'] ?? null), + 'postal_code' => self::str($item['CodePostal'] ?? null), + 'city' => self::str($item['Ville'] ?? null), + 'phone' => self::str($item['Telephone_1'] ?? null), + 'department' => self::str($item['Departement'] ?? null), + // Statut conserve brut (feed externe, valeurs non contraintes). + 'status' => self::str($item['Statut'] ?? null) ?? '', + 'validity_date' => self::parseDate(self::str($item['Validite'] ?? null)), + ]; + } + + /** + * Normalise un SIRET : ne conserve que les chiffres. Null si vide. + * La source est "sale" (longueurs variables 7 a 14) : aucune contrainte + * de longueur, on stocke les chiffres tels quels. + */ + public static function normalizeSiret(?string $raw): ?string + { + if (null === $raw) { + return null; + } + + $digits = preg_replace('/\D+/', '', $raw) ?? ''; + + return '' === $digits ? null : $digits; + } + + /** + * Convertit une date "dd/mm/yyyy" en "yyyy-mm-dd". Null si le format ne + * correspond pas ou si la date n'est pas un jour calendaire valide + * (garde-fou : evite un INSERT en erreur sur une date impossible). + */ + public static function parseDate(?string $raw): ?string + { + if (null === $raw || !preg_match('#^(\d{2})/(\d{2})/(\d{4})$#', $raw, $m)) { + return null; + } + + $day = (int) $m[1]; + $month = (int) $m[2]; + $year = (int) $m[3]; + + if (!checkdate($month, $day, $year)) { + return null; + } + + return sprintf('%04d-%02d-%02d', $year, $month, $day); + } + + /** + * Trim d'une valeur scalaire ; null si la chaine resultante est vide. + */ + private static function str(mixed $value): ?string + { + if (null === $value) { + return null; + } + + $trimmed = trim((string) $value); + + return '' === $trimmed ? null : $trimmed; + } +} diff --git a/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php new file mode 100644 index 0000000..db1bde4 --- /dev/null +++ b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php @@ -0,0 +1,250 @@ +addOption('file', null, InputOption::VALUE_REQUIRED, "Chemin d'un JSON local (court-circuite l'appel HTTP, utile pour tests/rejeu).") + ->addOption('ppp', null, InputOption::VALUE_REQUIRED, "Taille de page demandee a l'API.", (string) self::DEFAULT_PPP) + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Analyse sans ecriture en base.') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $ppp = max(1, (int) $input->getOption('ppp')); + $dryRun = (bool) $input->getOption('dry-run'); + $file = $input->getOption('file'); + + // 1. Recuperation des items (fichier local ou API). + try { + $items = null !== $file ? $this->readLocal((string) $file) : $this->fetchRemote($ppp); + } catch (Throwable $e) { + $io->error('Recuperation impossible : '.$e->getMessage()); + + return Command::FAILURE; + } + + $total = count($items); + $io->section(sprintf('QUALIMAT — %d items recus', $total)); + + // Garde-fou troncature : un retour egal a ppp signale un dataset coupe. + if (null === $file && $total === $ppp) { + $io->warning(sprintf("Le nombre d'items recus (%d) egale --ppp : resultat potentiellement tronque, augmente --ppp.", $ppp)); + } + + // 2. Mapping / normalisation (les items sans SIRET sont ignores). + ['rows' => $rows, 'skipped' => $skipped] = QualimatRowMapper::mapMany($items); + $io->writeln(sprintf('%d lignes exploitables, %d ignorees (sans SIRET).', count($rows), $skipped)); + + if ($dryRun) { + $this->renderPreview($io, $rows); + $io->note(sprintf('Dry-run : aucune ecriture. (%d lignes au total)', count($rows))); + + return Command::SUCCESS; + } + + // 3. Sync transactionnelle : upsert -> soft-delete -> journal. + $run = new DateTimeImmutable()->format('Y-m-d H:i:s.u'); + + $this->connection->beginTransaction(); + + try { + $upserted = $this->upsertAll($rows, $run); + $deactivated = $this->deactivateMissing($run); + $this->log($run, $total, $upserted, $skipped, $deactivated); + $this->connection->commit(); + } catch (Throwable $e) { + $this->connection->rollBack(); + $io->error('Sync annulee (rollback) : '.$e->getMessage()); + + return Command::FAILURE; + } + + $io->success(sprintf('%d upsert, %d ignore(s), %d desactive(s).', $upserted, $skipped, $deactivated)); + + return Command::SUCCESS; + } + + /** + * Rejoue l'appel GET de l'API QUALIMAT et retourne le tableau d'items. + * + * @return array> + */ + private function fetchRemote(int $ppp): array + { + $response = $this->httpClient->request('GET', self::API_URL, [ + 'query' => ['type' => 'operateur_transport', 'ppp' => $ppp], + 'timeout' => 60, + ]); + + // toArray() leve une exception sur un statut non-2xx ou un corps non-JSON. + $data = $response->toArray(); + + return array_is_list($data) ? $data : []; + } + + /** + * Lit un export JSON local (tableau d'objets). + * + * @return array> + */ + private function readLocal(string $path): array + { + $raw = @file_get_contents($path); + + if (false === $raw) { + throw new RuntimeException(sprintf('Fichier illisible : %s', $path)); + } + + $data = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); + + if (!is_array($data) || !array_is_list($data)) { + throw new RuntimeException("Le JSON doit etre un tableau d'objets."); + } + + return $data; + } + + /** + * Upsert de toutes les lignes valides (cle naturelle = siret). Marque + * is_active=TRUE et tamponne last_synced_at avec le run courant. + * + * @param list> $rows + */ + private function upsertAll(array $rows, string $run): int + { + $sql = <<<'SQL' + INSERT INTO qualimat_carrier + (siret, name, address, postal_code, city, phone, department, status, validity_date, is_active, last_synced_at) + VALUES + (:siret, :name, :address, :postal_code, :city, :phone, :department, :status, :validity_date, TRUE, :run) + ON CONFLICT (siret) DO UPDATE SET + name = EXCLUDED.name, + address = EXCLUDED.address, + postal_code = EXCLUDED.postal_code, + city = EXCLUDED.city, + phone = EXCLUDED.phone, + department = EXCLUDED.department, + status = EXCLUDED.status, + validity_date = EXCLUDED.validity_date, + is_active = TRUE, + last_synced_at = EXCLUDED.last_synced_at + SQL; + + $count = 0; + + foreach ($rows as $r) { + $this->connection->executeStatement($sql, [ + 'siret' => $r['siret'], + 'name' => $r['name'], + 'address' => $r['address'], + 'postal_code' => $r['postal_code'], + 'city' => $r['city'], + 'phone' => $r['phone'], + 'department' => $r['department'], + 'status' => $r['status'], + 'validity_date' => $r['validity_date'], + 'run' => $run, + ]); + ++$count; + } + + return $count; + } + + /** + * Soft-delete : toute ligne active non revue par ce run (tampon anterieur) + * passe a is_active=false. + */ + private function deactivateMissing(string $run): int + { + return (int) $this->connection->executeStatement( + 'UPDATE qualimat_carrier SET is_active = FALSE WHERE is_active = TRUE AND last_synced_at < :run', + ['run' => $run], + ); + } + + private function log(string $run, int $total, int $upserted, int $skipped, int $deactivated): void + { + $this->connection->executeStatement( + <<<'SQL' + INSERT INTO qualimat_sync_log (fetched_at, rows_total, rows_upserted, rows_skipped, rows_deactivated) + VALUES (:run, :total, :upserted, :skipped, :deactivated) + SQL, + [ + 'run' => $run, + 'total' => $total, + 'upserted' => $upserted, + 'skipped' => $skipped, + 'deactivated' => $deactivated, + ], + ); + } + + /** + * @param list> $rows + */ + private function renderPreview(SymfonyStyle $io, array $rows): void + { + $io->table( + ['SIRET', 'Nom', 'CP', 'Ville', 'Statut', 'Validite'], + array_map(static fn (array $r): array => [ + (string) $r['siret'], + mb_strimwidth((string) $r['name'], 0, 40, '…'), + (string) ($r['postal_code'] ?? ''), + mb_strimwidth((string) ($r['city'] ?? ''), 0, 25, '…'), + (string) $r['status'], + (string) ($r['validity_date'] ?? ''), + ], array_slice($rows, 0, 15)), + ); + } +} diff --git a/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php new file mode 100644 index 0000000..f27d390 --- /dev/null +++ b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php @@ -0,0 +1,90 @@ + ' 2C TRANS ', + 'Societe' => '2C TRANS', + 'Adresse' => '66 Impasse Mendi', + 'CodePostal' => '65500', + 'Ville' => 'VIC EN BIGORRE', + 'Telephone_1' => '+33|0608890316', + 'Siret' => '444 156 285 000 25', + 'Validite' => '14/05/2027', + 'Statut' => 'Audité', + 'Departement' => '65 - Hautes-Pyrénées', + ]); + + self::assertNotNull($row); + self::assertSame('44415628500025', $row['siret']); + self::assertSame('2C TRANS', $row['name']); + self::assertSame('2027-05-14', $row['validity_date']); + self::assertSame('+33|0608890316', $row['phone']); + self::assertSame('Audité', $row['status']); + self::assertSame('65 - Hautes-Pyrénées', $row['department']); + } + + public function testMapOneReturnsNullWithoutSiret(): void + { + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X', 'Siret' => null])); + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X'])); + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X', 'Siret' => ' '])); + } + + public function testMapManyCountsSkipped(): void + { + $result = QualimatRowMapper::mapMany([ + ['Nom' => 'A', 'Siret' => '111 111 111 00011', 'Statut' => 'Audité', 'Validite' => '01/01/2030'], + ['Nom' => 'B', 'Siret' => null], + ['Nom' => 'C', 'Siret' => ' '], + ]); + + self::assertCount(1, $result['rows']); + self::assertSame(2, $result['skipped']); + } + + public function testEmptyOptionalFieldsBecomeNull(): void + { + $row = QualimatRowMapper::mapOne([ + 'Siret' => '111 111 111 00011', + 'Nom' => 'A', + 'Adresse' => '', + 'Ville' => ' ', + ]); + + self::assertNotNull($row); + self::assertNull($row['address']); + self::assertNull($row['city']); + self::assertNull($row['validity_date']); + } +} diff --git a/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php new file mode 100644 index 0000000..7084c00 --- /dev/null +++ b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php @@ -0,0 +1,138 @@ +get('doctrine.dbal.default_connection'); + $this->connection = $connection; + $this->purge(); + } + + protected function tearDown(): void + { + $this->purge(); + parent::tearDown(); + } + + public function testFirstSyncInsertsNormalizesAndLogs(): void + { + $tester = $this->runSync([ + [ + 'Nom' => '2C TRANS', + 'Societe' => '2C TRANS', + 'Adresse' => '66 Impasse Mendi', + 'CodePostal' => '65500', + 'Ville' => 'VIC EN BIGORRE', + 'Telephone_1' => '+33|0608890316', + 'Siret' => '444 156 285 000 25', + 'Validite' => '14/05/2027', + 'Statut' => 'Audité', + 'Departement' => '65 - Hautes-Pyrénées', + ], + // Item sans SIRET : doit etre ignore (compte dans rows_skipped). + ['Nom' => 'SANS SIRET', 'Siret' => null, 'Validite' => '01/01/2030', 'Statut' => 'Valide'], + ]); + + $tester->assertCommandIsSuccessful(); + + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier')); + + $row = $this->connection->fetchAssociative('SELECT * FROM qualimat_carrier'); + self::assertNotFalse($row); + self::assertSame('44415628500025', $row['siret']); + self::assertSame('2C TRANS', $row['name']); + self::assertSame('2027-05-14', $row['validity_date']); + self::assertSame('+33|0608890316', $row['phone']); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + + $log = $this->connection->fetchAssociative('SELECT * FROM qualimat_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame(2, (int) $log['rows_total']); + self::assertSame(1, (int) $log['rows_upserted']); + self::assertSame(1, (int) $log['rows_skipped']); + self::assertSame(0, (int) $log['rows_deactivated']); + } + + public function testSecondSyncUpdatesAndSoftDeletesMissing(): void + { + $a = ['Nom' => 'A', 'Siret' => '111 111 111 00011', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + $b = ['Nom' => 'B', 'Siret' => '222 222 222 00022', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + + $this->runSync([$a, $b])->assertCommandIsSuccessful(); + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + + // 2e run sans B et avec A renomme : A est mis a jour, B est soft-delete. + $aRenamed = ['Nom' => 'A BIS', 'Siret' => '111 111 111 00011', 'Validite' => '02/02/2031', 'Statut' => 'Valide']; + $tester = $this->runSync([$aRenamed]); + $tester->assertCommandIsSuccessful(); + + // Toujours 2 lignes en base, mais une seule active. + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier')); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + self::assertSame(1, $this->countRows("SELECT COUNT(*) FROM qualimat_carrier WHERE siret = '22222222200022' AND is_active = FALSE")); + + // A a bien ete mis a jour (nom + statut + date). + $a = $this->connection->fetchAssociative("SELECT * FROM qualimat_carrier WHERE siret = '11111111100011'"); + self::assertNotFalse($a); + self::assertSame('A BIS', $a['name']); + self::assertSame('Valide', $a['status']); + self::assertSame('2031-02-02', $a['validity_date']); + + $log = $this->connection->fetchAssociative('SELECT * FROM qualimat_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame(1, (int) $log['rows_upserted']); + self::assertSame(1, (int) $log['rows_deactivated']); + self::assertSame(0, (int) $log['rows_skipped']); + } + + /** + * @param array> $items + */ + private function runSync(array $items): CommandTester + { + $path = tempnam(sys_get_temp_dir(), 'qualimat_').'.json'; + file_put_contents($path, json_encode($items, JSON_THROW_ON_ERROR)); + + $application = new Application(self::$kernel); + $tester = new CommandTester($application->find('app:qualimat:sync')); + $tester->execute(['--file' => $path]); + + @unlink($path); + + return $tester; + } + + private function countRows(string $sql): int + { + return (int) $this->connection->fetchOne($sql); + } + + private function purge(): void + { + $this->connection->executeStatement('DELETE FROM qualimat_carrier'); + $this->connection->executeStatement('DELETE FROM qualimat_sync_log'); + } +} -- 2.39.5 From c8bff683733badf6629863fcdbdf68782c076748 Mon Sep 17 00:00:00 2001 From: tristan Date: Fri, 12 Jun 2026 15:19:13 +0200 Subject: [PATCH 3/7] chore(transport) : ajouter la cible make qualimat-sync (ERP-39) --- makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/makefile b/makefile index b749e4b..f07ca38 100644 --- a/makefile +++ b/makefile @@ -249,6 +249,14 @@ sync-permissions: seed-rbac: $(SYMFONY_CONSOLE) --no-interaction app:seed-rbac +# Synchronise le referentiel des transporteurs QUALIMAT (ERP-39) : upsert sur +# le SIRET + soft-delete des absents + journal. Idempotent (refresh complet), +# prevu pour un cron quotidien. +# Options : --dry-run (analyse sans ecriture), --file= (source +# locale au lieu de l'API), --ppp= (taille de page API, defaut 10000). +qualimat-sync: + $(SYMFONY_CONSOLE) --no-interaction app:qualimat:sync + # Attention, supprime votre bdd local db-reset: $(DOCKER_COMPOSE) down -v -- 2.39.5 From abe663d355818a20766643e020ebb186f6a3c495 Mon Sep 17 00:00:00 2001 From: tristan Date: Fri, 12 Jun 2026 15:49:28 +0200 Subject: [PATCH 4/7] =?UTF-8?q?feat(transport)=20:=20synchronisation=20du?= =?UTF-8?q?=20r=C3=A9f=C3=A9rentiel=20codes=20IDTF=20(ERP-149)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commande console app:idtf:sync : récupère l'export Excel des codes IDTF (régimes de nettoyage transport) depuis icrt-idtf.com, le parse et synchronise une table référentielle (upsert sur (schema, idtf_number) + soft-delete + journal). Scope road ; discriminant schema road/water conservé. - migration : tables idtf_product + idtf_sync_log (COMMENT ON COLUMN sur chaque colonne, unique (schema, idtf_number), cas_numbers JSONB) - IdtfSheetParser : parsing pur d'une matrice (détection dynamique de l'en-tête, mapping par libellé, CAS split, date dd-mm-yyyy -> ISO) + tests unitaires - SyncIdtfCommand : options --schema / --file / --dry-run, POST avec fields[] explicites (export 11 colonnes), upsert DBAL transactionnel - cible make idtf-sync - tests fonctionnels via .xlsx généré (parsing/upsert/journal/soft-delete) Réutilise framework.http_client (activé pour QUALIMAT, ERP-39). phpoffice/phpspreadsheet déjà présent. --- makefile | 8 + migrations/Version20260612160000.php | 120 +++++++ .../Application/Idtf/IdtfSheetParser.php | 219 ++++++++++++ .../Console/SyncIdtfCommand.php | 317 ++++++++++++++++++ .../Application/Idtf/IdtfSheetParserTest.php | 110 ++++++ .../Console/SyncIdtfCommandTest.php | 163 +++++++++ 6 files changed, 937 insertions(+) create mode 100644 migrations/Version20260612160000.php create mode 100644 src/Module/Transport/Application/Idtf/IdtfSheetParser.php create mode 100644 src/Module/Transport/Infrastructure/Console/SyncIdtfCommand.php create mode 100644 tests/Module/Transport/Application/Idtf/IdtfSheetParserTest.php create mode 100644 tests/Module/Transport/Infrastructure/Console/SyncIdtfCommandTest.php diff --git a/makefile b/makefile index f07ca38..8cac32b 100644 --- a/makefile +++ b/makefile @@ -257,6 +257,14 @@ seed-rbac: qualimat-sync: $(SYMFONY_CONSOLE) --no-interaction app:qualimat:sync +# Synchronise le referentiel des codes IDTF (ERP-149) depuis l'export Excel +# icrt-idtf.com : upsert sur (schema, idtf_number) + soft-delete + journal. +# Idempotent (refresh complet). +# Options : --schema=road|water (defaut road), --dry-run (analyse sans +# ecriture), --file= (source locale au lieu du telechargement). +idtf-sync: + $(SYMFONY_CONSOLE) --no-interaction app:idtf:sync + # Attention, supprime votre bdd local db-reset: $(DOCKER_COMPOSE) down -v diff --git a/migrations/Version20260612160000.php b/migrations/Version20260612160000.php new file mode 100644 index 0000000..6385a3e --- /dev/null +++ b/migrations/Version20260612160000.php @@ -0,0 +1,120 @@ +addSql(<<<'SQL' + CREATE TABLE idtf_product ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + idtf_number INTEGER NOT NULL, + schema VARCHAR(8) NOT NULL, + product_group VARCHAR(255) DEFAULT NULL, + name TEXT NOT NULL, + cleaning_regime VARCHAR(64) NOT NULL, + important_requirements TEXT DEFAULT NULL, + mandatory_date DATE DEFAULT NULL, + related_products TEXT DEFAULT NULL, + formula VARCHAR(255) DEFAULT NULL, + eural_code VARCHAR(64) DEFAULT NULL, + cas_numbers JSONB DEFAULT '[]' NOT NULL, + footnotes TEXT DEFAULT NULL, + source_export_date DATE NOT NULL, + is_active BOOLEAN DEFAULT TRUE NOT NULL, + last_synced_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + PRIMARY KEY (id), + CONSTRAINT uq_idtf_product_schema_number UNIQUE (schema, idtf_number), + CONSTRAINT chk_idtf_product_schema CHECK (schema IN ('road', 'water')) + ) + SQL); + $this->addSql('CREATE INDEX idx_idtf_product_active ON idtf_product (schema, is_active)'); + + $this->comment('idtf_product', '_table', "Referentiel des codes IDTF (marchandise + regime de nettoyage transport), synchronise depuis l'export Excel icrt-idtf.com."); + $this->comment('idtf_product', 'id', 'Cle technique auto-incrementee.'); + $this->comment('idtf_product', 'idtf_number', 'Numero IDTF de la marchandise (identifiant metier source). Unique par schema.'); + $this->comment('idtf_product', 'schema', "Mode de transport / schema IDTF : 'road' (routier) ou 'water' (fluvial). Discriminant d'unicite avec idtf_number."); + $this->comment('idtf_product', 'product_group', "Groupe de produit (colonne Product Group de l'export). Nullable."); + $this->comment('idtf_product', 'name', "Nom de la marchandise (libelle FR de l'export)."); + $this->comment('idtf_product', 'cleaning_regime', 'Regime de nettoyage minimal exige (A, B, C, Interdit, ...).'); + $this->comment('idtf_product', 'important_requirements', 'Exigences importantes associees. Nullable.'); + $this->comment('idtf_product', 'mandatory_date', "Date d'application obligatoire du regime (convertie depuis dd-mm-yyyy). Nullable."); + $this->comment('idtf_product', 'related_products', 'Produits apparentes (texte libre). Nullable.'); + $this->comment('idtf_product', 'formula', 'Formule chimique de la marchandise. Nullable.'); + $this->comment('idtf_product', 'eural_code', 'Code EURAL (dechet) associe. Nullable.'); + $this->comment('idtf_product', 'cas_numbers', 'Liste des numeros CAS (JSONB), eclatee depuis la cellule "Numero CAS" separee par ";". Tableau vide si absent.'); + $this->comment('idtf_product', 'footnotes', "Annotations / notes de bas de page de l'export. Nullable."); + $this->comment('idtf_product', 'source_export_date', 'Date d\'export du fichier source (preambule "Export date:").'); + $this->comment('idtf_product', 'is_active', 'Faux = ligne absente du dernier export (soft-delete). Toute ligne non revue par le dernier run passe a FALSE.'); + $this->comment('idtf_product', 'last_synced_at', 'Horodatage du run de synchro ayant vu cette ligne en dernier (soft-delete : last_synced_at < run courant).'); + + $this->addSql(<<<'SQL' + CREATE TABLE idtf_sync_log ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + schema VARCHAR(8) NOT NULL, + export_date DATE NOT NULL, + rows_total INT NOT NULL, + rows_upserted INT NOT NULL, + rows_deactivated INT NOT NULL, + created_at TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW() NOT NULL, + PRIMARY KEY (id) + ) + SQL); + + $this->comment('idtf_sync_log', '_table', 'Journal des synchronisations IDTF (une ligne par run de la commande app:idtf:sync).'); + $this->comment('idtf_sync_log', 'id', 'Cle technique auto-incrementee.'); + $this->comment('idtf_sync_log', 'schema', "Mode de transport synchronise : 'road' ou 'water'."); + $this->comment('idtf_sync_log', 'export_date', "Date d'export du fichier source traite par ce run."); + $this->comment('idtf_sync_log', 'rows_total', 'Nombre de lignes exploitables lues dans le fichier.'); + $this->comment('idtf_sync_log', 'rows_upserted', 'Nombre de lignes inserees ou mises a jour.'); + $this->comment('idtf_sync_log', 'rows_deactivated', 'Nombre de lignes passees a is_active=false (absentes de cet export).'); + $this->comment('idtf_sync_log', 'created_at', 'Horodatage de fin du run (insertion du journal).'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS idtf_sync_log'); + $this->addSql('DROP TABLE IF EXISTS idtf_product'); + } + + /** + * Pose un COMMENT ON TABLE/COLUMN en dollar-quoting Postgres ($_$...$_$) + * pour eviter tout echappement d'apostrophes dans les descriptions. + */ + private function comment(string $table, string $column, string $description): void + { + $quotedTable = '"'.str_replace('"', '""', $table).'"'; + + if ('_table' === $column) { + $this->addSql(sprintf('COMMENT ON TABLE %s IS $_$%s$_$', $quotedTable, $description)); + + return; + } + + $this->addSql(sprintf( + 'COMMENT ON COLUMN %s.%s IS $_$%s$_$', + $quotedTable, + '"'.str_replace('"', '""', $column).'"', + $description, + )); + } +} diff --git a/src/Module/Transport/Application/Idtf/IdtfSheetParser.php b/src/Module/Transport/Application/Idtf/IdtfSheetParser.php new file mode 100644 index 0000000..6ff36a0 --- /dev/null +++ b/src/Module/Transport/Application/Idtf/IdtfSheetParser.php @@ -0,0 +1,219 @@ +> $matrix + * + * @return array{exportDate: null|string, rows: list>} + */ + public static function parse(array $matrix): array + { + $exportDate = self::extractExportDate($matrix); + $headerIndex = self::findHeaderIndex($matrix); + + if (null === $headerIndex) { + throw new RuntimeException("Ligne d'en-tete introuvable (colonne 'Numero IDTF')."); + } + + $map = self::buildColumnMap($matrix[$headerIndex]); + + if (!isset($map['idtf_number'])) { + throw new RuntimeException("Colonne 'Numero IDTF' introuvable dans l'en-tete."); + } + + $rows = []; + + foreach (array_slice($matrix, $headerIndex + 1) as $row) { + $idtf = trim((string) ($row[$map['idtf_number']] ?? '')); + + // Ligne vide / non exploitable : pas d'identifiant numerique. + if ('' === $idtf || !ctype_digit($idtf)) { + continue; + } + + $rows[] = [ + 'idtf_number' => (int) $idtf, + 'product_group' => self::val($row, $map['product_group'] ?? null), + 'name' => self::val($row, $map['name'] ?? null) ?? '', + 'cleaning_regime' => self::val($row, $map['cleaning_regime'] ?? null) ?? '', + 'important_requirements' => self::val($row, $map['important_requirements'] ?? null), + 'mandatory_date' => self::parseDate(self::val($row, $map['mandatory_date'] ?? null)), + 'related_products' => self::val($row, $map['related_products'] ?? null), + 'formula' => self::val($row, $map['formula'] ?? null), + 'eural_code' => self::val($row, $map['eural_code'] ?? null), + 'cas_numbers' => self::splitCas(self::val($row, $map['cas'] ?? null)), + 'footnotes' => self::val($row, $map['footnotes'] ?? null), + ]; + } + + return ['exportDate' => $exportDate, 'rows' => $rows]; + } + + /** + * Cherche une date "d-m-Y" dans les premieres lignes (preambule + * "Export date: 12-6-2026") et la convertit en "Y-m-d". Null si absente. + * + * @param array> $matrix + */ + public static function extractExportDate(array $matrix): ?string + { + foreach (array_slice($matrix, 0, 5) as $row) { + $line = implode(' ', array_map(static fn (mixed $c): string => (string) $c, $row)); + + if (preg_match('/(\d{1,2})-(\d{1,2})-(\d{4})/', $line, $m)) { + $day = (int) $m[1]; + $month = (int) $m[2]; + $year = (int) $m[3]; + + if (checkdate($month, $day, $year)) { + return sprintf('%04d-%02d-%02d', $year, $month, $day); + } + } + } + + return null; + } + + /** + * Index de la ligne d'en-tete : premiere ligne contenant une cellule dont + * le libelle normalise contient "numero idtf". + * + * @param array> $matrix + */ + private static function findHeaderIndex(array $matrix): ?int + { + foreach ($matrix as $i => $row) { + foreach ($row as $cell) { + if (str_contains(self::normalize((string) $cell), 'numero idtf')) { + return $i; + } + } + } + + return null; + } + + /** + * Construit le mapping logique -> index de colonne a partir de la ligne + * d'en-tete (resiste au reordonnancement via fields[]). + * + * @param array $header + * + * @return array + */ + private static function buildColumnMap(array $header): array + { + $map = []; + + foreach ($header as $col => $label) { + $n = self::normalize((string) $label); + + $key = match (true) { + str_contains($n, 'numero idtf') => 'idtf_number', + str_contains($n, 'product group'), + str_contains($n, 'groupe') => 'product_group', + str_contains($n, 'nom de la marchandise') => 'name', + str_contains($n, 'regime de nettoyage') => 'cleaning_regime', + str_contains($n, 'exigences importantes') => 'important_requirements', + str_contains($n, 'date d application') => 'mandatory_date', + str_contains($n, 'produits apparentes') => 'related_products', + str_contains($n, 'formule') => 'formula', + str_contains($n, 'code eural') => 'eural_code', + str_contains($n, 'numero cas') => 'cas', + str_contains($n, 'annotations') => 'footnotes', + default => null, + }; + + if (null !== $key && !isset($map[$key])) { + $map[$key] = (int) $col; + } + } + + return $map; + } + + /** + * Convertit une date "dd-mm-yyyy" en "yyyy-mm-dd". Null si format invalide + * ou date calendaire impossible. + */ + private static function parseDate(?string $raw): ?string + { + if (null === $raw || !preg_match('/^(\d{1,2})-(\d{1,2})-(\d{4})$/', $raw, $m)) { + return null; + } + + $day = (int) $m[1]; + $month = (int) $m[2]; + $year = (int) $m[3]; + + if (!checkdate($month, $day, $year)) { + return null; + } + + return sprintf('%04d-%02d-%02d', $year, $month, $day); + } + + /** + * Eclate une cellule "Numero CAS" sur ';' en liste de chaines non vides. + * + * @return list + */ + private static function splitCas(?string $raw): array + { + if (null === $raw) { + return []; + } + + $parts = array_map('trim', explode(';', $raw)); + + return array_values(array_filter($parts, static fn (string $v): bool => '' !== $v)); + } + + /** + * Valeur d'une cellule par index : trim, null si absente/vide. + * + * @param array $row + */ + private static function val(array $row, ?int $col): ?string + { + if (null === $col) { + return null; + } + + $v = trim((string) ($row[$col] ?? '')); + + return '' === $v ? null : $v; + } + + /** + * Normalise un libelle d'en-tete : minuscules, sans accents ni apostrophes, + * espaces compresses (pour un matching robuste). + */ + private static function normalize(string $s): string + { + $s = str_replace(['’', "'"], ' ', $s); + $s = (string) iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $s); + $s = mb_strtolower($s); + + return trim((string) preg_replace('/\s+/', ' ', $s)); + } +} diff --git a/src/Module/Transport/Infrastructure/Console/SyncIdtfCommand.php b/src/Module/Transport/Infrastructure/Console/SyncIdtfCommand.php new file mode 100644 index 0000000..54400b4 --- /dev/null +++ b/src/Module/Transport/Infrastructure/Console/SyncIdtfCommand.php @@ -0,0 +1,317 @@ + + */ + private const array EXPORT_FIELDS = [ + 'product_number_idtf', + 'product_name', + 'minimum_cleaning_regime', + 'important_requirements', + 'date_mandatory', + 'related_products', + 'formula', + 'product_number_eural', + 'product_number_cas', + 'footnotes', + ]; + + public function __construct( + private readonly Connection $connection, + private readonly HttpClientInterface $httpClient, + ) { + parent::__construct(); + } + + protected function configure(): void + { + $this + ->addOption('schema', null, InputOption::VALUE_REQUIRED, "Module IDTF : 'road' (routier) ou 'water' (fluvial).", 'road') + ->addOption('file', null, InputOption::VALUE_REQUIRED, "Chemin d'un .xlsx local (court-circuite le telechargement, utile pour tests/rejeu).") + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Analyse sans ecriture en base.') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $schema = (string) $input->getOption('schema'); + $dryRun = (bool) $input->getOption('dry-run'); + $file = $input->getOption('file'); + + if (!in_array($schema, ['road', 'water'], true)) { + $io->error("--schema doit valoir 'road' ou 'water'."); + + return Command::INVALID; + } + + // 1. Recuperation du binaire xlsx (local ou via POST). + try { + $xlsx = null !== $file ? $this->readLocal((string) $file) : $this->downloadExport($schema); + } catch (Throwable $e) { + $io->error('Telechargement/lecture impossible : '.$e->getMessage()); + + return Command::FAILURE; + } + + // 2. Parsing (xlsx -> matrice -> lignes normalisees). + try { + $parsed = IdtfSheetParser::parse($this->toMatrix($xlsx)); + } catch (Throwable $e) { + $io->error('Parsing impossible : '.$e->getMessage()); + + return Command::FAILURE; + } + + $rows = $parsed['rows']; + $exportDate = $parsed['exportDate'] ?? new DateTimeImmutable()->format('Y-m-d'); + + $io->section(sprintf('IDTF %s — export du %s', mb_strtoupper($schema), $exportDate)); + $io->writeln(sprintf('%d lignes exploitables lues.', count($rows))); + + if ($dryRun) { + $this->renderPreview($io, $rows); + $io->note(sprintf('Dry-run : aucune ecriture. (%d lignes au total)', count($rows))); + + return Command::SUCCESS; + } + + // 3. Sync transactionnelle : upsert -> soft-delete -> journal. + $run = new DateTimeImmutable()->format('Y-m-d H:i:s.u'); + + $this->connection->beginTransaction(); + + try { + $upserted = $this->upsertAll($schema, $exportDate, $rows, $run); + $deactivated = $this->deactivateMissing($schema, $run); + $this->log($schema, $exportDate, count($rows), $upserted, $deactivated); + $this->connection->commit(); + } catch (Throwable $e) { + $this->connection->rollBack(); + $io->error('Sync annulee (rollback) : '.$e->getMessage()); + + return Command::FAILURE; + } + + $io->success(sprintf('%d upsert, %d desactive(s).', $upserted, $deactivated)); + + return Command::SUCCESS; + } + + /** + * Rejoue le POST du generateur pour recuperer le binaire xlsx complet. + * Le formulaire poste sur lui-meme ; pas besoin de GET/cookies prealables. + */ + private function downloadExport(string $schema): string + { + // Corps construit a la main : http-client encoderait fields[] en + // indices numerotes, on veut bien des "fields[]=..." repetes. + $pairs = [ + 'schema='.$schema, + 'type%5B%5D='.$schema, + 'roadRegime%5B%5D=all', + 'waterRegime%5B%5D=all', + 'groups%5B%5D=all', + 'products%5B%5D=all', + ]; + + foreach (self::EXPORT_FIELDS as $field) { + $pairs[] = 'fields%5B%5D='.$field; + } + + $pairs[] = 'generateExcel='; + + $response = $this->httpClient->request('POST', self::GENERATOR_URL, [ + 'headers' => ['Content-Type' => 'application/x-www-form-urlencoded'], + 'body' => implode('&', $pairs), + 'timeout' => 90, + ]); + + $content = $response->getContent(); + $contentType = $response->getHeaders(false)['content-type'][0] ?? ''; + + // Garde-fou : un HTML signifie un POST rejete (filtres/payload). + if (!str_contains($contentType, 'spreadsheet') && !str_starts_with($content, "PK\x03\x04")) { + throw new RuntimeException(sprintf('Reponse non-xlsx (content-type: %s). Verifie le payload.', $contentType)); + } + + return $content; + } + + private function readLocal(string $path): string + { + $raw = @file_get_contents($path); + + if (false === $raw) { + throw new RuntimeException(sprintf('Fichier illisible : %s', $path)); + } + + return $raw; + } + + /** + * Charge le binaire xlsx via PhpSpreadsheet et retourne la feuille active + * sous forme de matrice 0-indexee (lignes/colonnes). + * + * @return array> + */ + private function toMatrix(string $xlsx): array + { + $tmp = tempnam(sys_get_temp_dir(), 'idtf_').'.xlsx'; + file_put_contents($tmp, $xlsx); + + try { + // toArray(null, true, true, false) : colonnes 0-indexees. + return IOFactory::load($tmp)->getActiveSheet()->toArray(null, true, true, false); + } finally { + @unlink($tmp); + } + } + + /** + * Upsert de toutes les lignes (cle naturelle = schema + idtf_number). + * + * @param list> $rows + */ + private function upsertAll(string $schema, string $exportDate, array $rows, string $run): int + { + $sql = <<<'SQL' + INSERT INTO idtf_product + (idtf_number, schema, product_group, name, cleaning_regime, important_requirements, + mandatory_date, related_products, formula, eural_code, cas_numbers, footnotes, + source_export_date, is_active, last_synced_at) + VALUES + (:idtf, :schema, :grp, :name, :regime, :req, :mdate, :related, :formula, :eural, + CAST(:cas AS JSONB), :foot, :export, TRUE, :run) + ON CONFLICT (schema, idtf_number) DO UPDATE SET + product_group = EXCLUDED.product_group, + name = EXCLUDED.name, + cleaning_regime = EXCLUDED.cleaning_regime, + important_requirements = EXCLUDED.important_requirements, + mandatory_date = EXCLUDED.mandatory_date, + related_products = EXCLUDED.related_products, + formula = EXCLUDED.formula, + eural_code = EXCLUDED.eural_code, + cas_numbers = EXCLUDED.cas_numbers, + footnotes = EXCLUDED.footnotes, + source_export_date = EXCLUDED.source_export_date, + is_active = TRUE, + last_synced_at = EXCLUDED.last_synced_at + SQL; + + $count = 0; + + foreach ($rows as $r) { + $this->connection->executeStatement($sql, [ + 'idtf' => $r['idtf_number'], + 'schema' => $schema, + 'grp' => $r['product_group'], + 'name' => $r['name'], + 'regime' => $r['cleaning_regime'], + 'req' => $r['important_requirements'], + 'mdate' => $r['mandatory_date'], + 'related' => $r['related_products'], + 'formula' => $r['formula'], + 'eural' => $r['eural_code'], + 'cas' => json_encode($r['cas_numbers'], JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR), + 'foot' => $r['footnotes'], + 'export' => $exportDate, + 'run' => $run, + ]); + ++$count; + } + + return $count; + } + + /** + * Soft-delete : toute ligne du schema active non revue par ce run passe a + * is_active=false. + */ + private function deactivateMissing(string $schema, string $run): int + { + return (int) $this->connection->executeStatement( + 'UPDATE idtf_product SET is_active = FALSE WHERE schema = :schema AND is_active = TRUE AND last_synced_at < :run', + ['schema' => $schema, 'run' => $run], + ); + } + + private function log(string $schema, string $exportDate, int $total, int $upserted, int $deactivated): void + { + $this->connection->executeStatement( + <<<'SQL' + INSERT INTO idtf_sync_log (schema, export_date, rows_total, rows_upserted, rows_deactivated) + VALUES (:schema, :export, :total, :upserted, :deactivated) + SQL, + [ + 'schema' => $schema, + 'export' => $exportDate, + 'total' => $total, + 'upserted' => $upserted, + 'deactivated' => $deactivated, + ], + ); + } + + /** + * @param list> $rows + */ + private function renderPreview(SymfonyStyle $io, array $rows): void + { + $io->table( + ['IDTF', 'Nom', 'Regime', 'CAS'], + array_map(static fn (array $r): array => [ + (string) $r['idtf_number'], + mb_strimwidth((string) $r['name'], 0, 50, '…'), + (string) $r['cleaning_regime'], + implode(', ', $r['cas_numbers']), + ], array_slice($rows, 0, 15)), + ); + } +} diff --git a/tests/Module/Transport/Application/Idtf/IdtfSheetParserTest.php b/tests/Module/Transport/Application/Idtf/IdtfSheetParserTest.php new file mode 100644 index 0000000..2348eaa --- /dev/null +++ b/tests/Module/Transport/Application/Idtf/IdtfSheetParserTest.php @@ -0,0 +1,110 @@ +sampleMatrix()); + self::assertSame('2026-06-12', $parsed['exportDate']); + } + + public function testParsesAndNormalizesFirstRow(): void + { + $parsed = IdtfSheetParser::parse($this->sampleMatrix()); + $row = $parsed['rows'][0]; + + self::assertSame(30748, $row['idtf_number']); + self::assertSame('Argiles avec régime de nettoyage C', $row['name']); + self::assertSame('C', $row['cleaning_regime']); + self::assertSame('2026-04-02', $row['mandatory_date']); + self::assertSame('Al2O3', $row['formula']); + self::assertSame('01 01 01', $row['eural_code']); + self::assertSame(['7631-86-9', '1344-28-1'], $row['cas_numbers']); + self::assertSame('Note 1', $row['footnotes']); + } + + public function testSkipsEmptyAndNonNumericRows(): void + { + $parsed = IdtfSheetParser::parse($this->sampleMatrix()); + + // 2 lignes exploitables (30748 et 30744) ; vide + "abc" ignorees. + self::assertCount(2, $parsed['rows']); + self::assertSame(30744, $parsed['rows'][1]['idtf_number']); + } + + public function testEmptyOptionalCellsBecomeNullAndCasEmpty(): void + { + $parsed = IdtfSheetParser::parse($this->sampleMatrix()); + $row = $parsed['rows'][1]; // 30744 + + self::assertNull($row['mandatory_date']); + self::assertNull($row['formula']); + self::assertNull($row['product_group']); + self::assertSame([], $row['cas_numbers']); + } + + public function testColumnOrderIsResolvedByLabel(): void + { + // En-tete dans un ordre different : le mapping doit suivre les libelles. + $matrix = [ + ['Export date: 1-1-2026'], + ['Numéro CAS', 'Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'], + ['7440-44-0', '99', 'Carbone', 'B'], + ]; + + $parsed = IdtfSheetParser::parse($matrix); + $row = $parsed['rows'][0]; + + self::assertSame(99, $row['idtf_number']); + self::assertSame('Carbone', $row['name']); + self::assertSame('B', $row['cleaning_regime']); + self::assertSame(['7440-44-0'], $row['cas_numbers']); + } + + public function testThrowsWhenHeaderMissing(): void + { + $this->expectException(RuntimeException::class); + IdtfSheetParser::parse([['foo', 'bar'], ['1', '2']]); + } + + public function testExportDateNullWhenAbsent(): void + { + $matrix = [ + ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'], + ['1', 'X', 'A'], + ]; + + self::assertNull(IdtfSheetParser::parse($matrix)['exportDate']); + } + + /** + * Matrice representative de l'export reel : preambule (lignes 0-1), ligne + * vide (2), en-tete (3) puis donnees. + * + * @return array> + */ + private function sampleMatrix(): array + { + return [ + ['Export date: 12-6-2026'], + ['Changes in the database after this date...'], + [], + ['Numéro IDTF', 'Product Group', 'Nom de la marchandise', 'Régime de nettoyage', 'Exigences importantes', 'Date d’application obligatoire', 'Produits apparentés', 'Formule', 'Code EURAL', 'Numéro CAS', 'Annotations'], + ['30748', 'Substances inorganiques', 'Argiles avec régime de nettoyage C', 'C', 'Exigence X', '02-04-2026', 'Poudre argile', 'Al2O3', '01 01 01', '7631-86-9 ; 1344-28-1', 'Note 1'], + ['', '', '', '', '', '', '', '', '', '', ''], + ['abc', 'ligne non numerique a ignorer', '', '', '', '', '', '', '', '', ''], + ['30744', '', 'Additifs alimentaires', 'A', '', '', '', '', '', '', ''], + ]; + } +} diff --git a/tests/Module/Transport/Infrastructure/Console/SyncIdtfCommandTest.php b/tests/Module/Transport/Infrastructure/Console/SyncIdtfCommandTest.php new file mode 100644 index 0000000..c01e530 --- /dev/null +++ b/tests/Module/Transport/Infrastructure/Console/SyncIdtfCommandTest.php @@ -0,0 +1,163 @@ + parser -> DBAL). + * + * @internal + */ +final class SyncIdtfCommandTest extends KernelTestCase +{ + private Connection $connection; + + protected function setUp(): void + { + self::bootKernel(); + + /** @var Connection $connection */ + $connection = self::getContainer()->get('doctrine.dbal.default_connection'); + $this->connection = $connection; + $this->purge(); + } + + protected function tearDown(): void + { + $this->purge(); + parent::tearDown(); + } + + public function testSyncParsesXlsxUpsertsAndLogs(): void + { + $path = $this->makeXlsx([ + ['Export date: 12-6-2026'], + ['Avertissement preambule'], + [], + ['Numéro IDTF', 'Product Group', 'Nom de la marchandise', 'Régime de nettoyage', 'Exigences importantes', 'Date d’application obligatoire', 'Produits apparentés', 'Formule', 'Code EURAL', 'Numéro CAS', 'Annotations'], + ['30748', 'Inorganiques', 'Argiles régime C', 'C', 'Exig X', '02-04-2026', 'Poudre', 'Al2O3', '01 01 01', '7631-86-9 ; 1344-28-1', 'Note'], + ['', '', '', '', '', '', '', '', '', '', ''], + ['30744', '', 'Additifs', 'A', '', '', '', '', '', '', ''], + ]); + + $tester = $this->runSync($path); + $tester->assertCommandIsSuccessful(); + + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product')); + + $row = $this->connection->fetchAssociative("SELECT * FROM idtf_product WHERE idtf_number = 30748 AND schema = 'road'"); + self::assertNotFalse($row); + self::assertSame('Argiles régime C', $row['name']); + self::assertSame('C', $row['cleaning_regime']); + self::assertSame('2026-04-02', $row['mandatory_date']); + self::assertSame('2026-06-12', $row['source_export_date']); + self::assertSame(['7631-86-9', '1344-28-1'], json_decode((string) $row['cas_numbers'], true)); + + $log = $this->connection->fetchAssociative('SELECT * FROM idtf_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame('road', $log['schema']); + self::assertSame('2026-06-12', $log['export_date']); + self::assertSame(2, (int) $log['rows_total']); + self::assertSame(2, (int) $log['rows_upserted']); + self::assertSame(0, (int) $log['rows_deactivated']); + } + + public function testSecondSyncSoftDeletesMissing(): void + { + $header = ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage']; + + $this->runSync($this->makeXlsx([ + ['Export date: 1-6-2026'], + $header, + ['100', 'Produit 100', 'A'], + ['200', 'Produit 200', 'B'], + ]))->assertCommandIsSuccessful(); + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE is_active = TRUE')); + + // 2e export sans 200 -> soft-delete de 200, mise a jour de 100. + $tester = $this->runSync($this->makeXlsx([ + ['Export date: 2-6-2026'], + $header, + ['100', 'Produit 100 maj', 'C'], + ])); + $tester->assertCommandIsSuccessful(); + + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM idtf_product')); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE is_active = TRUE')); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM idtf_product WHERE idtf_number = 200 AND is_active = FALSE')); + + $row100 = $this->connection->fetchAssociative('SELECT * FROM idtf_product WHERE idtf_number = 100'); + self::assertNotFalse($row100); + self::assertSame('Produit 100 maj', $row100['name']); + self::assertSame('C', $row100['cleaning_regime']); + + $log = $this->connection->fetchAssociative('SELECT * FROM idtf_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame(1, (int) $log['rows_upserted']); + self::assertSame(1, (int) $log['rows_deactivated']); + } + + public function testInvalidSchemaIsRejected(): void + { + $path = $this->makeXlsx([ + ['Numéro IDTF', 'Nom de la marchandise', 'Régime de nettoyage'], + ['1', 'X', 'A'], + ]); + + $application = new Application(self::$kernel); + $tester = new CommandTester($application->find('app:idtf:sync')); + $exitCode = $tester->execute(['--file' => $path, '--schema' => 'air']); + + @unlink($path); + + self::assertSame(2, $exitCode); // Command::INVALID + self::assertSame(0, $this->countRows('SELECT COUNT(*) FROM idtf_product')); + } + + /** + * @param array> $matrix + */ + private function makeXlsx(array $matrix): string + { + $spreadsheet = new Spreadsheet(); + $spreadsheet->getActiveSheet()->fromArray($matrix, null, 'A1', true); + + $path = tempnam(sys_get_temp_dir(), 'idtf_').'.xlsx'; + new Xlsx($spreadsheet)->save($path); + $spreadsheet->disconnectWorksheets(); + + return $path; + } + + private function runSync(string $path): CommandTester + { + $application = new Application(self::$kernel); + $tester = new CommandTester($application->find('app:idtf:sync')); + $tester->execute(['--file' => $path, '--schema' => 'road']); + + @unlink($path); + + return $tester; + } + + private function countRows(string $sql): int + { + return (int) $this->connection->fetchOne($sql); + } + + private function purge(): void + { + $this->connection->executeStatement('DELETE FROM idtf_product'); + $this->connection->executeStatement('DELETE FROM idtf_sync_log'); + } +} -- 2.39.5 From c888a45cc4d60ff2aebb350a59dc730a44715c8b Mon Sep 17 00:00:00 2001 From: tristan Date: Fri, 12 Jun 2026 15:57:59 +0200 Subject: [PATCH 5/7] chore(transport) : User-Agent navigateur neutre pour les syncs externes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remplace l'UA 'Starseed-ERP' par un UA navigateur neutre : évite les filtres anti-bot des sources (qualimat.org WordPress/WAF, icrt-idtf.com) sans révéler l'application. --- config/packages/http_client.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/packages/http_client.yaml b/config/packages/http_client.yaml index 66ac52b..1476bf6 100644 --- a/config/packages/http_client.yaml +++ b/config/packages/http_client.yaml @@ -1,9 +1,13 @@ # Active le composant HTTP Client (symfony/http-client) et enregistre # l'autowiring de HttpClientInterface. Utilise par les commandes de # synchronisation de referentiels externes (QUALIMAT, IDTF...). +# +# User-Agent navigateur neutre : les sources (qualimat.org sous WordPress/WAF, +# icrt-idtf.com) filtrent souvent les UA de bibliotheque/vides ; un UA de type +# navigateur evite les blocages anti-bot sans reveler l'application. framework: http_client: default_options: timeout: 30 headers: - User-Agent: 'Starseed-ERP (referentiel-sync)' + User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' -- 2.39.5 From 0b9aaef38eaeb89b508dc64157c1032169617d03 Mon Sep 17 00:00:00 2001 From: tristan Date: Mon, 15 Jun 2026 10:20:53 +0200 Subject: [PATCH 6/7] fix(transport) : securiser la synchro QUALIMAT (revue ERP-39) - garde-fou anti-desactivation de masse : fetchRemote leve sur un payload non-list (2xx inattendu) et la commande abandonne sans ecriture si aucune ligne exploitable, au lieu de soft-delete tout le referentiel - verrou consultatif pg_try_advisory_lock pour serialiser les runs (anti-overlap) - deduplication par SIRET dans le mapper (rows_upserted = transporteurs distincts) - upsert par paquets (INSERT groupe) au lieu d'un aller-retour par ligne - migration des tables qualimat deplacee vers le namespace modulaire Transport (+ enregistrement du path dans doctrine_migrations.yaml) - tests : deduplication + abandon sur source vide --- config/packages/doctrine_migrations.yaml | 1 + .../Qualimat/QualimatRowMapper.php | 14 +- .../Console/SyncQualimatCommand.php | 151 +++++++++++++----- .../Migrations}/Version20260612150000.php | 7 +- .../Qualimat/QualimatRowMapperTest.php | 15 ++ .../Console/SyncQualimatCommandTest.php | 25 +++ 6 files changed, 169 insertions(+), 44 deletions(-) rename {migrations => src/Module/Transport/Infrastructure/Doctrine/Migrations}/Version20260612150000.php (95%) diff --git a/config/packages/doctrine_migrations.yaml b/config/packages/doctrine_migrations.yaml index c7bb23a..ef45b16 100644 --- a/config/packages/doctrine_migrations.yaml +++ b/config/packages/doctrine_migrations.yaml @@ -2,4 +2,5 @@ doctrine_migrations: migrations_paths: 'DoctrineMigrations': '%kernel.project_dir%/migrations' 'App\Module\Core\Infrastructure\Doctrine\Migrations': '%kernel.project_dir%/src/Module/Core/Infrastructure/Doctrine/Migrations' + 'App\Module\Transport\Infrastructure\Doctrine\Migrations': '%kernel.project_dir%/src/Module/Transport/Infrastructure/Doctrine/Migrations' enable_profiler: false diff --git a/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php index 91bb31a..c0e4943 100644 --- a/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php +++ b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php @@ -13,7 +13,11 @@ final class QualimatRowMapper { /** * Mappe un lot d'items. Les items sans SIRET exploitable sont ignores et - * comptes a part (cf. `rows_skipped` du journal). + * comptes a part (cf. `rows_skipped` du journal). Les doublons de SIRET + * (source "sale" : memes chiffres a separateurs pres) sont fusionnes, + * derniere occurrence gagnante — l'upsert ne verrait qu'une ligne de toute + * facon, et le compte `rows_upserted` reflete ainsi les transporteurs + * distincts. * * @param array> $items * @@ -21,7 +25,7 @@ final class QualimatRowMapper */ public static function mapMany(array $items): array { - $rows = []; + $bySiret = []; $skipped = 0; foreach ($items as $item) { @@ -33,10 +37,12 @@ final class QualimatRowMapper continue; } - $rows[] = $row; + // Cle = SIRET normalise : une occurrence ulterieure ecrase la + // precedente (derniere gagnante). + $bySiret[$row['siret']] = $row; } - return ['rows' => $rows, 'skipped' => $skipped]; + return ['rows' => array_values($bySiret), 'skipped' => $skipped]; } /** diff --git a/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php index db1bde4..d03d2a4 100644 --- a/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php +++ b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php @@ -41,6 +41,14 @@ final class SyncQualimatCommand extends Command private const string API_URL = 'https://www.qualimat.org/wp-json/qualimat/v1/getOperateurs'; private const int DEFAULT_PPP = 10000; + // Cle arbitraire (mais stable) du verrou consultatif Postgres serialisant + // les runs de `app:qualimat:sync` entre eux. Propre a cette commande. + private const int ADVISORY_LOCK_KEY = 3_900_000_039; + + // Nombre de lignes par INSERT groupe. 10 parametres/ligne, large marge sous + // la limite Postgres de 65535 parametres par requete. + private const int UPSERT_CHUNK = 1000; + public function __construct( private readonly Connection $connection, private readonly HttpClientInterface $httpClient, @@ -64,9 +72,30 @@ final class SyncQualimatCommand extends Command $dryRun = (bool) $input->getOption('dry-run'); $file = $input->getOption('file'); + // Verrou consultatif (session) : empeche deux runs de se chevaucher + // (cron qui deborde, invocation manuelle parallele). Sans lui, le run le + // plus tardif desactiverait les lignes que l'autre vient d'inserer. + if (!$this->acquireLock()) { + $io->error('Une synchronisation QUALIMAT est deja en cours (verrou non disponible).'); + + return Command::FAILURE; + } + + try { + return $this->doSync($io, $ppp, $dryRun, $file); + } finally { + $this->releaseLock(); + } + } + + /** + * Coeur de la synchronisation, execute sous verrou consultatif. + */ + private function doSync(SymfonyStyle $io, int $ppp, bool $dryRun, ?string $file): int + { // 1. Recuperation des items (fichier local ou API). try { - $items = null !== $file ? $this->readLocal((string) $file) : $this->fetchRemote($ppp); + $items = null !== $file ? $this->readLocal($file) : $this->fetchRemote($ppp); } catch (Throwable $e) { $io->error('Recuperation impossible : '.$e->getMessage()); @@ -81,7 +110,8 @@ final class SyncQualimatCommand extends Command $io->warning(sprintf("Le nombre d'items recus (%d) egale --ppp : resultat potentiellement tronque, augmente --ppp.", $ppp)); } - // 2. Mapping / normalisation (les items sans SIRET sont ignores). + // 2. Mapping / normalisation (les items sans SIRET sont ignores, les + // doublons de SIRET sont fusionnes : derniere occurrence gagnante). ['rows' => $rows, 'skipped' => $skipped] = QualimatRowMapper::mapMany($items); $io->writeln(sprintf('%d lignes exploitables, %d ignorees (sans SIRET).', count($rows), $skipped)); @@ -92,6 +122,15 @@ final class SyncQualimatCommand extends Command return Command::SUCCESS; } + // Garde-fou « zero ligne » : une source vide (incident amont, liste [] + // legitime) ne doit JAMAIS atteindre le soft-delete, qui desactiverait + // tout le referentiel. On abandonne sans rien ecrire. + if ([] === $rows) { + $io->error('Aucune ligne exploitable : synchronisation abandonnee (desactivation de masse evitee).'); + + return Command::FAILURE; + } + // 3. Sync transactionnelle : upsert -> soft-delete -> journal. $run = new DateTimeImmutable()->format('Y-m-d H:i:s.u'); @@ -114,6 +153,23 @@ final class SyncQualimatCommand extends Command return Command::SUCCESS; } + /** + * Tente de prendre le verrou consultatif de session. Retourne false si un + * autre run le detient deja (Postgres `pg_try_advisory_lock`, non bloquant). + */ + private function acquireLock(): bool + { + return (bool) $this->connection->fetchOne('SELECT pg_try_advisory_lock(:key)', ['key' => self::ADVISORY_LOCK_KEY]); + } + + /** + * Relache le verrou consultatif pris par acquireLock(). + */ + private function releaseLock(): void + { + $this->connection->executeStatement('SELECT pg_advisory_unlock(:key)', ['key' => self::ADVISORY_LOCK_KEY]); + } + /** * Rejoue l'appel GET de l'API QUALIMAT et retourne le tableau d'items. * @@ -129,7 +185,15 @@ final class SyncQualimatCommand extends Command // toArray() leve une exception sur un statut non-2xx ou un corps non-JSON. $data = $response->toArray(); - return array_is_list($data) ? $data : []; + // Un 2xx au corps inattendu (objet d'erreur, enveloppe {"data":[...]}, etc.) + // ne doit PAS etre interprete comme « 0 transporteur » : ce serait masquer + // un changement de contrat de l'API et declencher la desactivation de masse + // (cf. garde-fou « zero ligne » dans execute()). On echoue franchement. + if (!array_is_list($data)) { + throw new RuntimeException("Reponse inattendue de l'API QUALIMAT : un tableau d'items etait attendu."); + } + + return $data; } /** @@ -155,47 +219,60 @@ final class SyncQualimatCommand extends Command } /** - * Upsert de toutes les lignes valides (cle naturelle = siret). Marque - * is_active=TRUE et tamponne last_synced_at avec le run courant. + * Upsert de toutes les lignes valides (cle naturelle = siret) par paquets + * (INSERT groupe), au lieu d'un aller-retour par ligne. Marque is_active=TRUE + * et tamponne last_synced_at avec le run courant. Les lignes etant deja + * dedoublonnees par SIRET en amont, le compte retourne = transporteurs + * distincts effectivement synchronises. * * @param list> $rows */ private function upsertAll(array $rows, string $run): int { - $sql = <<<'SQL' - INSERT INTO qualimat_carrier - (siret, name, address, postal_code, city, phone, department, status, validity_date, is_active, last_synced_at) - VALUES - (:siret, :name, :address, :postal_code, :city, :phone, :department, :status, :validity_date, TRUE, :run) - ON CONFLICT (siret) DO UPDATE SET - name = EXCLUDED.name, - address = EXCLUDED.address, - postal_code = EXCLUDED.postal_code, - city = EXCLUDED.city, - phone = EXCLUDED.phone, - department = EXCLUDED.department, - status = EXCLUDED.status, - validity_date = EXCLUDED.validity_date, - is_active = TRUE, - last_synced_at = EXCLUDED.last_synced_at - SQL; - $count = 0; - foreach ($rows as $r) { - $this->connection->executeStatement($sql, [ - 'siret' => $r['siret'], - 'name' => $r['name'], - 'address' => $r['address'], - 'postal_code' => $r['postal_code'], - 'city' => $r['city'], - 'phone' => $r['phone'], - 'department' => $r['department'], - 'status' => $r['status'], - 'validity_date' => $r['validity_date'], - 'run' => $run, - ]); - ++$count; + foreach (array_chunk($rows, self::UPSERT_CHUNK) as $chunk) { + $placeholders = []; + $params = []; + + foreach ($chunk as $r) { + // 10 valeurs liees + is_active force a TRUE (litteral). + $placeholders[] = '(?, ?, ?, ?, ?, ?, ?, ?, ?, TRUE, ?)'; + $params[] = $r['siret']; + $params[] = $r['name']; + $params[] = $r['address']; + $params[] = $r['postal_code']; + $params[] = $r['city']; + $params[] = $r['phone']; + $params[] = $r['department']; + $params[] = $r['status']; + $params[] = $r['validity_date']; + $params[] = $run; + } + + $sql = sprintf( + <<<'SQL' + INSERT INTO qualimat_carrier + (siret, name, address, postal_code, city, phone, department, status, validity_date, is_active, last_synced_at) + VALUES + %s + ON CONFLICT (siret) DO UPDATE SET + name = EXCLUDED.name, + address = EXCLUDED.address, + postal_code = EXCLUDED.postal_code, + city = EXCLUDED.city, + phone = EXCLUDED.phone, + department = EXCLUDED.department, + status = EXCLUDED.status, + validity_date = EXCLUDED.validity_date, + is_active = TRUE, + last_synced_at = EXCLUDED.last_synced_at + SQL, + implode(",\n ", $placeholders), + ); + + $this->connection->executeStatement($sql, $params); + $count += count($chunk); } return $count; diff --git a/migrations/Version20260612150000.php b/src/Module/Transport/Infrastructure/Doctrine/Migrations/Version20260612150000.php similarity index 95% rename from migrations/Version20260612150000.php rename to src/Module/Transport/Infrastructure/Doctrine/Migrations/Version20260612150000.php index f82f5d7..17a5c8f 100644 --- a/migrations/Version20260612150000.php +++ b/src/Module/Transport/Infrastructure/Doctrine/Migrations/Version20260612150000.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace DoctrineMigrations; +namespace App\Module\Transport\Infrastructure\Doctrine\Migrations; use Doctrine\DBAL\Schema\Schema; use Doctrine\Migrations\AbstractMigration; @@ -12,8 +12,9 @@ use Doctrine\Migrations\AbstractMigration; * * Tables alimentees par la commande de synchronisation `app:qualimat:sync` * (upsert sur le SIRET + soft-delete des absents + journal). Aucune FK - * cross-module (referentiel autonome) : migration posee au namespace racine - * `DoctrineMigrations`, comme les autres migrations de creation de tables. + * cross-module (referentiel autonome) : migration au namespace modulaire + * Transport. Tables autonomes, sans dependance d'ordre vis-a-vis des autres + * migrations, donc insensible au tri cross-namespace de Doctrine Migrations. */ final class Version20260612150000 extends AbstractMigration { diff --git a/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php index f27d390..44ea13c 100644 --- a/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php +++ b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php @@ -73,6 +73,21 @@ final class QualimatRowMapperTest extends TestCase self::assertSame(2, $result['skipped']); } + public function testMapManyDeduplicatesBySiretLastWins(): void + { + // Memes chiffres a separateurs pres : un seul transporteur, derniere + // occurrence gagnante (le compte ne doit pas surcompter les doublons). + $result = QualimatRowMapper::mapMany([ + ['Nom' => 'PREMIER', 'Siret' => '111 111 111 00011', 'Statut' => 'Audité'], + ['Nom' => 'DERNIER', 'Siret' => '11111111100011', 'Statut' => 'Valide'], + ]); + + self::assertCount(1, $result['rows']); + self::assertSame(0, $result['skipped']); + self::assertSame('DERNIER', $result['rows'][0]['name']); + self::assertSame('Valide', $result['rows'][0]['status']); + } + public function testEmptyOptionalFieldsBecomeNull(): void { $row = QualimatRowMapper::mapOne([ diff --git a/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php index 7084c00..ad3d497 100644 --- a/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php +++ b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php @@ -7,6 +7,7 @@ namespace App\Tests\Module\Transport\Infrastructure\Console; use Doctrine\DBAL\Connection; use Symfony\Bundle\FrameworkBundle\Console\Application; use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase; +use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Tester\CommandTester; use const JSON_THROW_ON_ERROR; @@ -108,6 +109,30 @@ final class SyncQualimatCommandTest extends KernelTestCase self::assertSame(0, (int) $log['rows_skipped']); } + public function testEmptySourceAbortsWithoutMassDeactivation(): void + { + // Premier run : 2 transporteurs actifs. + $a = ['Nom' => 'A', 'Siret' => '111 111 111 00011', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + $b = ['Nom' => 'B', 'Siret' => '222 222 222 00022', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + $this->runSync([$a, $b])->assertCommandIsSuccessful(); + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + + // Source ne contenant que des items inexploitables (zero ligne mappee) : + // la commande doit ECHOUER sans toucher le referentiel (pas de soft-delete + // de masse) et sans journaliser de run. + $logsBefore = $this->countRows('SELECT COUNT(*) FROM qualimat_sync_log'); + $tester = $this->runSync([ + ['Nom' => 'SANS SIRET 1', 'Siret' => null], + ['Nom' => 'SANS SIRET 2', 'Siret' => ' '], + ]); + + self::assertSame(Command::FAILURE, $tester->getStatusCode()); + // Les 2 transporteurs restent ACTIFS (aucune desactivation de masse). + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + // Aucun journal supplementaire (abandon avant la transaction). + self::assertSame($logsBefore, $this->countRows('SELECT COUNT(*) FROM qualimat_sync_log')); + } + /** * @param array> $items */ -- 2.39.5 From b51c3821a8d5664b6329d86d3e445528bb37f947 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jun 2026 17:18:29 +0200 Subject: [PATCH 7/7] fix(transport) : exclure les tables IDTF du schema_filter Doctrine (ERP-149) Les tables idtf_product / idtf_sync_log sont des referentiels DBAL bruts (sans entite ORM, synchronises par app:idtf:sync). Sans exclusion du schema_filter, doctrine:schema:update --force les droppe juste apres la migration dans test-db-setup, cassant SyncIdtfCommandTest (relation "idtf_product" does not exist). Meme correctif que QUALIMAT (#99). --- config/packages/doctrine.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 9b53c1d..d8a832a 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -17,13 +17,15 @@ doctrine: # - `qualimat_carrier` / `qualimat_sync_log` : referentiel # transporteurs synchronise en DBAL brut (upsert `ON CONFLICT`) # par `app:qualimat:sync`, hors ORM. + # - `idtf_product` / `idtf_sync_log` : referentiel codes IDTF + # synchronise en DBAL brut par `app:idtf:sync`, hors ORM. # Sans ce filtre, schema:update les considere comme "orphelines" et # genere un `DROP TABLE` qui casse la base de test apres chaque # `make test-db-setup` (la migration les a creees, schema:update les # supprime juste apres). Creation / suppression restent pilotees par # les migrations (audit_log : Version20260420202749 ; qualimat : - # Version20260612150000). - schema_filter: '~^(?!(?:audit_log|qualimat_carrier|qualimat_sync_log)$).+~' + # Version20260612150000 ; idtf : Version20260612160000). + schema_filter: '~^(?!(?:audit_log|qualimat_carrier|qualimat_sync_log|idtf_product|idtf_sync_log)$).+~' audit: url: '%env(resolve:DATABASE_URL)%' orm: -- 2.39.5