diff --git a/config/packages/http_client.yaml b/config/packages/http_client.yaml new file mode 100644 index 0000000..66ac52b --- /dev/null +++ b/config/packages/http_client.yaml @@ -0,0 +1,9 @@ +# Active le composant HTTP Client (symfony/http-client) et enregistre +# l'autowiring de HttpClientInterface. Utilise par les commandes de +# synchronisation de referentiels externes (QUALIMAT, IDTF...). +framework: + http_client: + default_options: + timeout: 30 + headers: + User-Agent: 'Starseed-ERP (referentiel-sync)' diff --git a/migrations/Version20260612150000.php b/migrations/Version20260612150000.php new file mode 100644 index 0000000..f82f5d7 --- /dev/null +++ b/migrations/Version20260612150000.php @@ -0,0 +1,111 @@ +addSql(<<<'SQL' + CREATE TABLE qualimat_carrier ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + siret VARCHAR(20) NOT NULL, + name VARCHAR(255) NOT NULL, + address VARCHAR(255) DEFAULT NULL, + postal_code VARCHAR(10) DEFAULT NULL, + city VARCHAR(255) DEFAULT NULL, + phone VARCHAR(32) DEFAULT NULL, + department VARCHAR(64) DEFAULT NULL, + status VARCHAR(32) NOT NULL, + validity_date DATE DEFAULT NULL, + is_active BOOLEAN DEFAULT TRUE NOT NULL, + last_synced_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + PRIMARY KEY (id), + CONSTRAINT uq_qualimat_carrier_siret UNIQUE (siret) + ) + SQL); + $this->addSql('CREATE INDEX idx_qualimat_carrier_active ON qualimat_carrier (is_active)'); + + $this->comment('qualimat_carrier', '_table', "Referentiel des transporteurs agrees QUALIMAT, synchronise quotidiennement depuis l'API qualimat.org (type=operateur_transport)."); + $this->comment('qualimat_carrier', 'id', 'Cle technique auto-incrementee.'); + $this->comment('qualimat_carrier', 'siret', 'SIRET normalise (chiffres sans espaces). Cle naturelle de synchro (unique). Source parfois incomplete (longueur variable), non contrainte a 14.'); + $this->comment('qualimat_carrier', 'name', 'Raison sociale du transporteur (champs Nom = Societe de la source, identiques).'); + $this->comment('qualimat_carrier', 'address', 'Adresse postale (voie). Nullable.'); + $this->comment('qualimat_carrier', 'postal_code', 'Code postal. Nullable.'); + $this->comment('qualimat_carrier', 'city', 'Ville. Nullable.'); + $this->comment('qualimat_carrier', 'phone', 'Telephone au format source "indicatif|numero" (ex: +33|0608890316). Nullable.'); + $this->comment('qualimat_carrier', 'department', 'Departement au format source "code - libelle" (ex: 65 - Hautes-Pyrenees). Nullable.'); + $this->comment('qualimat_carrier', 'status', "Statut d'agrement QUALIMAT (valeurs connues : Audite, Valide, Suspendu). Valeur brute de la source, non contrainte."); + $this->comment('qualimat_carrier', 'validity_date', 'Date de fin de validite de la certification (convertie depuis dd/mm/yyyy). Nullable.'); + $this->comment('qualimat_carrier', 'is_active', 'Faux = transporteur absent du dernier import (soft-delete). Toute ligne non revue par le dernier run passe a FALSE.'); + $this->comment('qualimat_carrier', 'last_synced_at', 'Horodatage du run de synchro ayant vu cette ligne en dernier (soft-delete : last_synced_at < run courant).'); + + $this->addSql(<<<'SQL' + CREATE TABLE qualimat_sync_log ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + fetched_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + rows_total INT NOT NULL, + rows_upserted INT NOT NULL, + rows_skipped INT NOT NULL, + rows_deactivated INT NOT NULL, + created_at TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW() NOT NULL, + PRIMARY KEY (id) + ) + SQL); + + $this->comment('qualimat_sync_log', '_table', 'Journal des synchronisations QUALIMAT (une ligne par run de la commande app:qualimat:sync).'); + $this->comment('qualimat_sync_log', 'id', 'Cle technique auto-incrementee.'); + $this->comment('qualimat_sync_log', 'fetched_at', "Horodatage de l'appel a l'API source (= run de synchro)."); + $this->comment('qualimat_sync_log', 'rows_total', "Nombre d'items renvoyes par l'API."); + $this->comment('qualimat_sync_log', 'rows_upserted', 'Nombre de transporteurs inseres ou mis a jour.'); + $this->comment('qualimat_sync_log', 'rows_skipped', "Nombre d'items ignores (sans SIRET exploitable)."); + $this->comment('qualimat_sync_log', 'rows_deactivated', 'Nombre de transporteurs passes a is_active=false (absents de cet import).'); + $this->comment('qualimat_sync_log', 'created_at', 'Horodatage de fin du run (insertion du journal).'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS qualimat_sync_log'); + $this->addSql('DROP TABLE IF EXISTS qualimat_carrier'); + } + + /** + * Pose un COMMENT ON TABLE/COLUMN en dollar-quoting Postgres ($_$...$_$) + * pour eviter tout echappement d'apostrophes dans les descriptions. + */ + private function comment(string $table, string $column, string $description): void + { + $quotedTable = '"'.str_replace('"', '""', $table).'"'; + + if ('_table' === $column) { + $this->addSql(sprintf('COMMENT ON TABLE %s IS $_$%s$_$', $quotedTable, $description)); + + return; + } + + $this->addSql(sprintf( + 'COMMENT ON COLUMN %s.%s IS $_$%s$_$', + $quotedTable, + '"'.str_replace('"', '""', $column).'"', + $description, + )); + } +} diff --git a/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php new file mode 100644 index 0000000..91bb31a --- /dev/null +++ b/src/Module/Transport/Application/Qualimat/QualimatRowMapper.php @@ -0,0 +1,124 @@ +> $items + * + * @return array{rows: list>, skipped: int} + */ + public static function mapMany(array $items): array + { + $rows = []; + $skipped = 0; + + foreach ($items as $item) { + $row = self::mapOne($item); + + if (null === $row) { + ++$skipped; + + continue; + } + + $rows[] = $row; + } + + return ['rows' => $rows, 'skipped' => $skipped]; + } + + /** + * Mappe un item unique. Retourne null si le SIRET est absent ou vide + * (ligne inexploitable : pas de cle naturelle pour l'upsert). + * + * @param array $item + * + * @return null|array + */ + public static function mapOne(array $item): ?array + { + $siret = self::normalizeSiret(self::str($item['Siret'] ?? null)); + + if (null === $siret) { + return null; + } + + return [ + 'siret' => $siret, + // Nom et Societe sont identiques a la source : une seule colonne. + 'name' => self::str($item['Nom'] ?? null) ?? '', + 'address' => self::str($item['Adresse'] ?? null), + 'postal_code' => self::str($item['CodePostal'] ?? null), + 'city' => self::str($item['Ville'] ?? null), + 'phone' => self::str($item['Telephone_1'] ?? null), + 'department' => self::str($item['Departement'] ?? null), + // Statut conserve brut (feed externe, valeurs non contraintes). + 'status' => self::str($item['Statut'] ?? null) ?? '', + 'validity_date' => self::parseDate(self::str($item['Validite'] ?? null)), + ]; + } + + /** + * Normalise un SIRET : ne conserve que les chiffres. Null si vide. + * La source est "sale" (longueurs variables 7 a 14) : aucune contrainte + * de longueur, on stocke les chiffres tels quels. + */ + public static function normalizeSiret(?string $raw): ?string + { + if (null === $raw) { + return null; + } + + $digits = preg_replace('/\D+/', '', $raw) ?? ''; + + return '' === $digits ? null : $digits; + } + + /** + * Convertit une date "dd/mm/yyyy" en "yyyy-mm-dd". Null si le format ne + * correspond pas ou si la date n'est pas un jour calendaire valide + * (garde-fou : evite un INSERT en erreur sur une date impossible). + */ + public static function parseDate(?string $raw): ?string + { + if (null === $raw || !preg_match('#^(\d{2})/(\d{2})/(\d{4})$#', $raw, $m)) { + return null; + } + + $day = (int) $m[1]; + $month = (int) $m[2]; + $year = (int) $m[3]; + + if (!checkdate($month, $day, $year)) { + return null; + } + + return sprintf('%04d-%02d-%02d', $year, $month, $day); + } + + /** + * Trim d'une valeur scalaire ; null si la chaine resultante est vide. + */ + private static function str(mixed $value): ?string + { + if (null === $value) { + return null; + } + + $trimmed = trim((string) $value); + + return '' === $trimmed ? null : $trimmed; + } +} diff --git a/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php new file mode 100644 index 0000000..db1bde4 --- /dev/null +++ b/src/Module/Transport/Infrastructure/Console/SyncQualimatCommand.php @@ -0,0 +1,250 @@ +addOption('file', null, InputOption::VALUE_REQUIRED, "Chemin d'un JSON local (court-circuite l'appel HTTP, utile pour tests/rejeu).") + ->addOption('ppp', null, InputOption::VALUE_REQUIRED, "Taille de page demandee a l'API.", (string) self::DEFAULT_PPP) + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Analyse sans ecriture en base.') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $ppp = max(1, (int) $input->getOption('ppp')); + $dryRun = (bool) $input->getOption('dry-run'); + $file = $input->getOption('file'); + + // 1. Recuperation des items (fichier local ou API). + try { + $items = null !== $file ? $this->readLocal((string) $file) : $this->fetchRemote($ppp); + } catch (Throwable $e) { + $io->error('Recuperation impossible : '.$e->getMessage()); + + return Command::FAILURE; + } + + $total = count($items); + $io->section(sprintf('QUALIMAT — %d items recus', $total)); + + // Garde-fou troncature : un retour egal a ppp signale un dataset coupe. + if (null === $file && $total === $ppp) { + $io->warning(sprintf("Le nombre d'items recus (%d) egale --ppp : resultat potentiellement tronque, augmente --ppp.", $ppp)); + } + + // 2. Mapping / normalisation (les items sans SIRET sont ignores). + ['rows' => $rows, 'skipped' => $skipped] = QualimatRowMapper::mapMany($items); + $io->writeln(sprintf('%d lignes exploitables, %d ignorees (sans SIRET).', count($rows), $skipped)); + + if ($dryRun) { + $this->renderPreview($io, $rows); + $io->note(sprintf('Dry-run : aucune ecriture. (%d lignes au total)', count($rows))); + + return Command::SUCCESS; + } + + // 3. Sync transactionnelle : upsert -> soft-delete -> journal. + $run = new DateTimeImmutable()->format('Y-m-d H:i:s.u'); + + $this->connection->beginTransaction(); + + try { + $upserted = $this->upsertAll($rows, $run); + $deactivated = $this->deactivateMissing($run); + $this->log($run, $total, $upserted, $skipped, $deactivated); + $this->connection->commit(); + } catch (Throwable $e) { + $this->connection->rollBack(); + $io->error('Sync annulee (rollback) : '.$e->getMessage()); + + return Command::FAILURE; + } + + $io->success(sprintf('%d upsert, %d ignore(s), %d desactive(s).', $upserted, $skipped, $deactivated)); + + return Command::SUCCESS; + } + + /** + * Rejoue l'appel GET de l'API QUALIMAT et retourne le tableau d'items. + * + * @return array> + */ + private function fetchRemote(int $ppp): array + { + $response = $this->httpClient->request('GET', self::API_URL, [ + 'query' => ['type' => 'operateur_transport', 'ppp' => $ppp], + 'timeout' => 60, + ]); + + // toArray() leve une exception sur un statut non-2xx ou un corps non-JSON. + $data = $response->toArray(); + + return array_is_list($data) ? $data : []; + } + + /** + * Lit un export JSON local (tableau d'objets). + * + * @return array> + */ + private function readLocal(string $path): array + { + $raw = @file_get_contents($path); + + if (false === $raw) { + throw new RuntimeException(sprintf('Fichier illisible : %s', $path)); + } + + $data = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); + + if (!is_array($data) || !array_is_list($data)) { + throw new RuntimeException("Le JSON doit etre un tableau d'objets."); + } + + return $data; + } + + /** + * Upsert de toutes les lignes valides (cle naturelle = siret). Marque + * is_active=TRUE et tamponne last_synced_at avec le run courant. + * + * @param list> $rows + */ + private function upsertAll(array $rows, string $run): int + { + $sql = <<<'SQL' + INSERT INTO qualimat_carrier + (siret, name, address, postal_code, city, phone, department, status, validity_date, is_active, last_synced_at) + VALUES + (:siret, :name, :address, :postal_code, :city, :phone, :department, :status, :validity_date, TRUE, :run) + ON CONFLICT (siret) DO UPDATE SET + name = EXCLUDED.name, + address = EXCLUDED.address, + postal_code = EXCLUDED.postal_code, + city = EXCLUDED.city, + phone = EXCLUDED.phone, + department = EXCLUDED.department, + status = EXCLUDED.status, + validity_date = EXCLUDED.validity_date, + is_active = TRUE, + last_synced_at = EXCLUDED.last_synced_at + SQL; + + $count = 0; + + foreach ($rows as $r) { + $this->connection->executeStatement($sql, [ + 'siret' => $r['siret'], + 'name' => $r['name'], + 'address' => $r['address'], + 'postal_code' => $r['postal_code'], + 'city' => $r['city'], + 'phone' => $r['phone'], + 'department' => $r['department'], + 'status' => $r['status'], + 'validity_date' => $r['validity_date'], + 'run' => $run, + ]); + ++$count; + } + + return $count; + } + + /** + * Soft-delete : toute ligne active non revue par ce run (tampon anterieur) + * passe a is_active=false. + */ + private function deactivateMissing(string $run): int + { + return (int) $this->connection->executeStatement( + 'UPDATE qualimat_carrier SET is_active = FALSE WHERE is_active = TRUE AND last_synced_at < :run', + ['run' => $run], + ); + } + + private function log(string $run, int $total, int $upserted, int $skipped, int $deactivated): void + { + $this->connection->executeStatement( + <<<'SQL' + INSERT INTO qualimat_sync_log (fetched_at, rows_total, rows_upserted, rows_skipped, rows_deactivated) + VALUES (:run, :total, :upserted, :skipped, :deactivated) + SQL, + [ + 'run' => $run, + 'total' => $total, + 'upserted' => $upserted, + 'skipped' => $skipped, + 'deactivated' => $deactivated, + ], + ); + } + + /** + * @param list> $rows + */ + private function renderPreview(SymfonyStyle $io, array $rows): void + { + $io->table( + ['SIRET', 'Nom', 'CP', 'Ville', 'Statut', 'Validite'], + array_map(static fn (array $r): array => [ + (string) $r['siret'], + mb_strimwidth((string) $r['name'], 0, 40, '…'), + (string) ($r['postal_code'] ?? ''), + mb_strimwidth((string) ($r['city'] ?? ''), 0, 25, '…'), + (string) $r['status'], + (string) ($r['validity_date'] ?? ''), + ], array_slice($rows, 0, 15)), + ); + } +} diff --git a/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php new file mode 100644 index 0000000..f27d390 --- /dev/null +++ b/tests/Module/Transport/Application/Qualimat/QualimatRowMapperTest.php @@ -0,0 +1,90 @@ + ' 2C TRANS ', + 'Societe' => '2C TRANS', + 'Adresse' => '66 Impasse Mendi', + 'CodePostal' => '65500', + 'Ville' => 'VIC EN BIGORRE', + 'Telephone_1' => '+33|0608890316', + 'Siret' => '444 156 285 000 25', + 'Validite' => '14/05/2027', + 'Statut' => 'Audité', + 'Departement' => '65 - Hautes-Pyrénées', + ]); + + self::assertNotNull($row); + self::assertSame('44415628500025', $row['siret']); + self::assertSame('2C TRANS', $row['name']); + self::assertSame('2027-05-14', $row['validity_date']); + self::assertSame('+33|0608890316', $row['phone']); + self::assertSame('Audité', $row['status']); + self::assertSame('65 - Hautes-Pyrénées', $row['department']); + } + + public function testMapOneReturnsNullWithoutSiret(): void + { + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X', 'Siret' => null])); + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X'])); + self::assertNull(QualimatRowMapper::mapOne(['Nom' => 'X', 'Siret' => ' '])); + } + + public function testMapManyCountsSkipped(): void + { + $result = QualimatRowMapper::mapMany([ + ['Nom' => 'A', 'Siret' => '111 111 111 00011', 'Statut' => 'Audité', 'Validite' => '01/01/2030'], + ['Nom' => 'B', 'Siret' => null], + ['Nom' => 'C', 'Siret' => ' '], + ]); + + self::assertCount(1, $result['rows']); + self::assertSame(2, $result['skipped']); + } + + public function testEmptyOptionalFieldsBecomeNull(): void + { + $row = QualimatRowMapper::mapOne([ + 'Siret' => '111 111 111 00011', + 'Nom' => 'A', + 'Adresse' => '', + 'Ville' => ' ', + ]); + + self::assertNotNull($row); + self::assertNull($row['address']); + self::assertNull($row['city']); + self::assertNull($row['validity_date']); + } +} diff --git a/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php new file mode 100644 index 0000000..7084c00 --- /dev/null +++ b/tests/Module/Transport/Infrastructure/Console/SyncQualimatCommandTest.php @@ -0,0 +1,138 @@ +get('doctrine.dbal.default_connection'); + $this->connection = $connection; + $this->purge(); + } + + protected function tearDown(): void + { + $this->purge(); + parent::tearDown(); + } + + public function testFirstSyncInsertsNormalizesAndLogs(): void + { + $tester = $this->runSync([ + [ + 'Nom' => '2C TRANS', + 'Societe' => '2C TRANS', + 'Adresse' => '66 Impasse Mendi', + 'CodePostal' => '65500', + 'Ville' => 'VIC EN BIGORRE', + 'Telephone_1' => '+33|0608890316', + 'Siret' => '444 156 285 000 25', + 'Validite' => '14/05/2027', + 'Statut' => 'Audité', + 'Departement' => '65 - Hautes-Pyrénées', + ], + // Item sans SIRET : doit etre ignore (compte dans rows_skipped). + ['Nom' => 'SANS SIRET', 'Siret' => null, 'Validite' => '01/01/2030', 'Statut' => 'Valide'], + ]); + + $tester->assertCommandIsSuccessful(); + + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier')); + + $row = $this->connection->fetchAssociative('SELECT * FROM qualimat_carrier'); + self::assertNotFalse($row); + self::assertSame('44415628500025', $row['siret']); + self::assertSame('2C TRANS', $row['name']); + self::assertSame('2027-05-14', $row['validity_date']); + self::assertSame('+33|0608890316', $row['phone']); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + + $log = $this->connection->fetchAssociative('SELECT * FROM qualimat_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame(2, (int) $log['rows_total']); + self::assertSame(1, (int) $log['rows_upserted']); + self::assertSame(1, (int) $log['rows_skipped']); + self::assertSame(0, (int) $log['rows_deactivated']); + } + + public function testSecondSyncUpdatesAndSoftDeletesMissing(): void + { + $a = ['Nom' => 'A', 'Siret' => '111 111 111 00011', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + $b = ['Nom' => 'B', 'Siret' => '222 222 222 00022', 'Validite' => '01/01/2030', 'Statut' => 'Audité']; + + $this->runSync([$a, $b])->assertCommandIsSuccessful(); + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + + // 2e run sans B et avec A renomme : A est mis a jour, B est soft-delete. + $aRenamed = ['Nom' => 'A BIS', 'Siret' => '111 111 111 00011', 'Validite' => '02/02/2031', 'Statut' => 'Valide']; + $tester = $this->runSync([$aRenamed]); + $tester->assertCommandIsSuccessful(); + + // Toujours 2 lignes en base, mais une seule active. + self::assertSame(2, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier')); + self::assertSame(1, $this->countRows('SELECT COUNT(*) FROM qualimat_carrier WHERE is_active = TRUE')); + self::assertSame(1, $this->countRows("SELECT COUNT(*) FROM qualimat_carrier WHERE siret = '22222222200022' AND is_active = FALSE")); + + // A a bien ete mis a jour (nom + statut + date). + $a = $this->connection->fetchAssociative("SELECT * FROM qualimat_carrier WHERE siret = '11111111100011'"); + self::assertNotFalse($a); + self::assertSame('A BIS', $a['name']); + self::assertSame('Valide', $a['status']); + self::assertSame('2031-02-02', $a['validity_date']); + + $log = $this->connection->fetchAssociative('SELECT * FROM qualimat_sync_log ORDER BY id DESC LIMIT 1'); + self::assertNotFalse($log); + self::assertSame(1, (int) $log['rows_upserted']); + self::assertSame(1, (int) $log['rows_deactivated']); + self::assertSame(0, (int) $log['rows_skipped']); + } + + /** + * @param array> $items + */ + private function runSync(array $items): CommandTester + { + $path = tempnam(sys_get_temp_dir(), 'qualimat_').'.json'; + file_put_contents($path, json_encode($items, JSON_THROW_ON_ERROR)); + + $application = new Application(self::$kernel); + $tester = new CommandTester($application->find('app:qualimat:sync')); + $tester->execute(['--file' => $path]); + + @unlink($path); + + return $tester; + } + + private function countRows(string $sql): int + { + return (int) $this->connection->fetchOne($sql); + } + + private function purge(): void + { + $this->connection->executeStatement('DELETE FROM qualimat_carrier'); + $this->connection->executeStatement('DELETE FROM qualimat_sync_log'); + } +}