From 96a37a0cb0400d2f3c5699b40552f4728d28a80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Mon, 5 Jan 2026 22:41:40 +0100 Subject: [PATCH 01/11] Implemented proof of concept to convert between database types --- config/packages/doctrine.yaml | 91 +++++++------ src/Command/Migrations/DBMigrationCommand.php | 121 ++++++++++++++++++ .../PartKeeprImporter/PKImportHelper.php | 4 +- 3 files changed, 166 insertions(+), 50 deletions(-) create mode 100644 src/Command/Migrations/DBMigrationCommand.php diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 5261c295..2952e516 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -1,61 +1,56 @@ doctrine: dbal: - url: '%env(resolve:DATABASE_URL)%' - - # Required for DAMA doctrine test bundle - use_savepoints: true - - # IMPORTANT: You MUST configure your server version, - # either here or in the DATABASE_URL env var (see .env file) - + # 1. GLOBAL SETTINGS (Apply to all connections) types: - # UTC datetimes - datetime: - class: App\Doctrine\Types\UTCDateTimeType - date: - class: App\Doctrine\Types\UTCDateTimeType + datetime: App\Doctrine\Types\UTCDateTimeType + date: App\Doctrine\Types\UTCDateTimeType + datetime_immutable: App\Doctrine\Types\UTCDateTimeImmutableType + date_immutable: App\Doctrine\Types\UTCDateTimeImmutableType + big_decimal: App\Doctrine\Types\BigDecimalType + tinyint: App\Doctrine\Types\TinyIntType - datetime_immutable: - class: App\Doctrine\Types\UTCDateTimeImmutableType - date_immutable: - class: App\Doctrine\Types\UTCDateTimeImmutableType + connections: + default: + use_savepoints: true + schema_filter: ~^(?!internal)~ + url: '%env(resolve:DATABASE_URL)%' - big_decimal: - class: App\Doctrine\Types\BigDecimalType - tinyint: - class: App\Doctrine\Types\TinyIntType - - schema_filter: ~^(?!internal)~ - # Only enable this when needed - profiling_collect_backtrace: false + migration_source: + use_savepoints: true + schema_filter: ~^(?!internal)~ + url: '%env(resolve:DB_MIGRATION_SOURCE)%' orm: auto_generate_proxy_classes: true enable_lazy_ghost_objects: true - report_fields_where_declared: true - validate_xml_mapping: true - naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware - identity_generation_preferences: - Doctrine\DBAL\Platforms\PostgreSQLPlatform: identity - auto_mapping: true - controller_resolver: - auto_mapping: true - mappings: - App: - type: attribute - is_bundle: false - dir: '%kernel.project_dir%/src/Entity' - prefix: 'App\Entity' - alias: App - dql: - string_functions: - regexp: App\Doctrine\Functions\Regexp - field: DoctrineExtensions\Query\Mysql\Field - field2: App\Doctrine\Functions\Field2 - natsort: App\Doctrine\Functions\Natsort - array_position: App\Doctrine\Functions\ArrayPosition - ilike: App\Doctrine\Functions\ILike + entity_managers: + default: &common_orm_settings + report_fields_where_declared: true + validate_xml_mapping: true + naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware + identity_generation_preferences: + Doctrine\DBAL\Platforms\PostgreSQLPlatform: identity + mappings: + App: + type: attribute + is_bundle: false + dir: '%kernel.project_dir%/src/Entity' + prefix: 'App\Entity' + alias: App + dql: + string_functions: + regexp: App\Doctrine\Functions\Regexp + field: DoctrineExtensions\Query\Mysql\Field + field2: App\Doctrine\Functions\Field2 + natsort: App\Doctrine\Functions\Natsort + array_position: App\Doctrine\Functions\ArrayPosition + ilike: App\Doctrine\Functions\ILike + connection: default + + migration_source: + <<: *common_orm_settings + connection: migration_source when@test: doctrine: diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBMigrationCommand.php new file mode 100644 index 00000000..5cd5808e --- /dev/null +++ b/src/Command/Migrations/DBMigrationCommand.php @@ -0,0 +1,121 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Command\Migrations; + +use App\DataTables\Helpers\ColumnSortHelper; +use App\Entity\Parts\Manufacturer; +use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; +use Doctrine\ORM\EntityManagerInterface; + +use Doctrine\ORM\Id\AssignedGenerator; +use Doctrine\ORM\Mapping\ClassMetadata; +use Doctrine\Persistence\ManagerRegistry; +use Doctrine\Persistence\ObjectManager; +use Symfony\Component\Console\Attribute\AsCommand; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +#[AsCommand('partdb:migrate-db', 'Migrate the database to a different platform')] +class DBMigrationCommand extends Command +{ + private readonly EntityManagerInterface $sourceEM; + private readonly EntityManagerInterface $targetEM; + + public function __construct(private readonly ManagerRegistry $managerRegistry, + private readonly PKImportHelper $importHelper, + ) + { + $this->sourceEM = $this->managerRegistry->getManager('migration_source'); + $this->targetEM = $this->managerRegistry->getManager('default'); + + parent::__construct(); + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + // Example migration logic (to be replaced with actual migration code) + $output->writeln('Starting database migration...'); + + //Disable all event listeners on target EM to avoid unwanted side effects + $eventManager = $this->targetEM->getEventManager(); + foreach ($eventManager->getAllListeners() as $event => $listeners) { + foreach ($listeners as $listener) { + $eventManager->removeEventListener($event, $listener); + } + } + + $output->writeln('Clear target database...'); + $this->importHelper->purgeDatabaseForImport($this->targetEM, ['internal', 'migration_versions']); + + $metadata = $this->targetEM->getMetadataFactory()->getAllMetadata(); + + //First we modify each entity metadata to have an persist cascade on all relations + foreach ($metadata as $metadatum) { + $entityClass = $metadatum->getName(); + $output->writeln('Modifying cascade and ID settings for entity: ' . $entityClass); + + foreach ($metadatum->getAssociationNames() as $fieldName) { + $mapping = $metadatum->getAssociationMapping($fieldName); + $mapping->cascade = array_unique(array_merge($mapping->cascade, ['persist'])); + + $metadatum->setIdGeneratorType(ClassMetadata::GENERATOR_TYPE_NONE); + $metadatum->setIdGenerator(new AssignedGenerator()); + } + } + + + //Afterwards we migrate all entities + foreach ($metadata as $metadatum) { + //skip all superclasses + if ($metadatum->isMappedSuperclass) { + continue; + } + + $entityClass = $metadatum->getName(); + + $output->writeln('Migrating entity: ' . $entityClass); + + $repo = $this->sourceEM->getRepository($entityClass); + $items = $repo->findAll(); + foreach ($items as $item) { + $this->targetEM->persist($item); + } + $this->targetEM->flush(); + } + + //Migrate all manufacturers from source to target + /*$manufacturerRepo = $this->sourceEM->getRepository(Manufacturer::class); + $manufacturers = $manufacturerRepo->findAll(); + foreach ($manufacturers as $manufacturer) { + $this->targetEM->persist($manufacturer); + } + $this->targetEM->flush(); + */ + + $output->writeln('Database migration completed successfully.'); + + return Command::SUCCESS; + } +} diff --git a/src/Services/ImportExportSystem/PartKeeprImporter/PKImportHelper.php b/src/Services/ImportExportSystem/PartKeeprImporter/PKImportHelper.php index f36e48ce..880d77be 100644 --- a/src/Services/ImportExportSystem/PartKeeprImporter/PKImportHelper.php +++ b/src/Services/ImportExportSystem/PartKeeprImporter/PKImportHelper.php @@ -39,10 +39,10 @@ class PKImportHelper * Existing users and groups are not purged. * This is needed to avoid ID collisions. */ - public function purgeDatabaseForImport(): void + public function purgeDatabaseForImport(?EntityManagerInterface $entityManager = null, array $excluded_tables = ['users', 'groups', 'u2f_keys', 'internal', 'migration_versions']): void { //We use the ResetAutoIncrementORMPurger to reset the auto increment values of the tables. Also it normalizes table names before checking for exclusion. - $purger = new ResetAutoIncrementORMPurger($this->em, ['users', 'groups', 'u2f_keys', 'internal', 'migration_versions']); + $purger = new ResetAutoIncrementORMPurger($entityManager ?? $this->em, $excluded_tables); $purger->purge(); } From 3f0e4b09ce9cf318469d61c2d3fc933213b6f039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Mon, 5 Jan 2026 23:14:40 +0100 Subject: [PATCH 02/11] Added a progress bar --- src/Command/Migrations/DBMigrationCommand.php | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBMigrationCommand.php index 5cd5808e..c22f3db8 100644 --- a/src/Command/Migrations/DBMigrationCommand.php +++ b/src/Command/Migrations/DBMigrationCommand.php @@ -36,6 +36,7 @@ use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand('partdb:migrate-db', 'Migrate the database to a different platform')] class DBMigrationCommand extends Command @@ -55,8 +56,10 @@ class DBMigrationCommand extends Command public function execute(InputInterface $input, OutputInterface $output): int { + $io = new SymfonyStyle($input, $output); + // Example migration logic (to be replaced with actual migration code) - $output->writeln('Starting database migration...'); + $io->info('Starting database migration...'); //Disable all event listeners on target EM to avoid unwanted side effects $eventManager = $this->targetEM->getEventManager(); @@ -66,15 +69,16 @@ class DBMigrationCommand extends Command } } - $output->writeln('Clear target database...'); + $io->info('Clear target database...'); $this->importHelper->purgeDatabaseForImport($this->targetEM, ['internal', 'migration_versions']); $metadata = $this->targetEM->getMetadataFactory()->getAllMetadata(); + $io->info('Modifying entity metadata for migration...'); //First we modify each entity metadata to have an persist cascade on all relations foreach ($metadata as $metadatum) { $entityClass = $metadatum->getName(); - $output->writeln('Modifying cascade and ID settings for entity: ' . $entityClass); + $io->writeln('Modifying cascade and ID settings for entity: ' . $entityClass, OutputInterface::VERBOSITY_VERBOSE); foreach ($metadatum->getAssociationNames() as $fieldName) { $mapping = $metadatum->getAssociationMapping($fieldName); @@ -86,6 +90,8 @@ class DBMigrationCommand extends Command } + $io->progressStart(count($metadata)); + //Afterwards we migrate all entities foreach ($metadata as $metadatum) { //skip all superclasses @@ -95,16 +101,20 @@ class DBMigrationCommand extends Command $entityClass = $metadatum->getName(); - $output->writeln('Migrating entity: ' . $entityClass); + $io->note('Migrating entity: ' . $entityClass); $repo = $this->sourceEM->getRepository($entityClass); $items = $repo->findAll(); - foreach ($items as $item) { + foreach ($items as $index => $item) { $this->targetEM->persist($item); } $this->targetEM->flush(); + + $io-> } + $io->progressFinish(); + //Migrate all manufacturers from source to target /*$manufacturerRepo = $this->sourceEM->getRepository(Manufacturer::class); $manufacturers = $manufacturerRepo->findAll(); @@ -116,6 +126,10 @@ class DBMigrationCommand extends Command $output->writeln('Database migration completed successfully.'); + if ($io->isVerbose()) { + $io->info('Process took peak memory: ' . round(memory_get_peak_usage(true) / 1024 / 1024, 2) . ' MB'); + } + return Command::SUCCESS; } } From e0a25009d9b8df6706e6a7d48cbb48d1d260bcf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Mon, 5 Jan 2026 23:16:33 +0100 Subject: [PATCH 03/11] fixed --- src/Command/Migrations/DBMigrationCommand.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBMigrationCommand.php index c22f3db8..06aa3e00 100644 --- a/src/Command/Migrations/DBMigrationCommand.php +++ b/src/Command/Migrations/DBMigrationCommand.php @@ -109,8 +109,6 @@ class DBMigrationCommand extends Command $this->targetEM->persist($item); } $this->targetEM->flush(); - - $io-> } $io->progressFinish(); From 00b35e3306579ef42a45912aedc1507fe6678a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Mon, 5 Jan 2026 23:25:53 +0100 Subject: [PATCH 04/11] Fix sequences of postgres after migration --- src/Command/Migrations/DBMigrationCommand.php | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBMigrationCommand.php index 06aa3e00..0c58e6b9 100644 --- a/src/Command/Migrations/DBMigrationCommand.php +++ b/src/Command/Migrations/DBMigrationCommand.php @@ -26,6 +26,8 @@ namespace App\Command\Migrations; use App\DataTables\Helpers\ColumnSortHelper; use App\Entity\Parts\Manufacturer; use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; +use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; @@ -122,6 +124,10 @@ class DBMigrationCommand extends Command $this->targetEM->flush(); */ + //Fix sequences / auto increment values on target database + $io->info('Fixing sequences / auto increment values on target database...'); + $this->fixAutoIncrements($this->targetEM); + $output->writeln('Database migration completed successfully.'); if ($io->isVerbose()) { @@ -130,4 +136,33 @@ class DBMigrationCommand extends Command return Command::SUCCESS; } + + private function fixAutoIncrements(EntityManagerInterface $em): void + { + $connection = $em->getConnection(); + $platform = $connection->getDatabasePlatform(); + + if ($platform instanceof PostgreSQLPlatform) { + $connection->executeStatement( + //From: https://wiki.postgresql.org/wiki/Fixing_Sequences + << Date: Thu, 8 Jan 2026 21:03:38 +0100 Subject: [PATCH 05/11] Made DBMigrationCommand take a DB url so we do not need a special doctrine config --- config/packages/doctrine.yaml | 91 ++++++++++--------- config/reference.php | 5 +- src/Command/Migrations/DBMigrationCommand.php | 33 +++++-- 3 files changed, 79 insertions(+), 50 deletions(-) diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 2952e516..5261c295 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -1,56 +1,61 @@ doctrine: dbal: - # 1. GLOBAL SETTINGS (Apply to all connections) + url: '%env(resolve:DATABASE_URL)%' + + # Required for DAMA doctrine test bundle + use_savepoints: true + + # IMPORTANT: You MUST configure your server version, + # either here or in the DATABASE_URL env var (see .env file) + types: - datetime: App\Doctrine\Types\UTCDateTimeType - date: App\Doctrine\Types\UTCDateTimeType - datetime_immutable: App\Doctrine\Types\UTCDateTimeImmutableType - date_immutable: App\Doctrine\Types\UTCDateTimeImmutableType - big_decimal: App\Doctrine\Types\BigDecimalType - tinyint: App\Doctrine\Types\TinyIntType + # UTC datetimes + datetime: + class: App\Doctrine\Types\UTCDateTimeType + date: + class: App\Doctrine\Types\UTCDateTimeType - connections: - default: - use_savepoints: true - schema_filter: ~^(?!internal)~ - url: '%env(resolve:DATABASE_URL)%' + datetime_immutable: + class: App\Doctrine\Types\UTCDateTimeImmutableType + date_immutable: + class: App\Doctrine\Types\UTCDateTimeImmutableType - migration_source: - use_savepoints: true - schema_filter: ~^(?!internal)~ - url: '%env(resolve:DB_MIGRATION_SOURCE)%' + big_decimal: + class: App\Doctrine\Types\BigDecimalType + tinyint: + class: App\Doctrine\Types\TinyIntType + + schema_filter: ~^(?!internal)~ + # Only enable this when needed + profiling_collect_backtrace: false orm: auto_generate_proxy_classes: true enable_lazy_ghost_objects: true + report_fields_where_declared: true + validate_xml_mapping: true + naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware + identity_generation_preferences: + Doctrine\DBAL\Platforms\PostgreSQLPlatform: identity + auto_mapping: true + controller_resolver: + auto_mapping: true + mappings: + App: + type: attribute + is_bundle: false + dir: '%kernel.project_dir%/src/Entity' + prefix: 'App\Entity' + alias: App - entity_managers: - default: &common_orm_settings - report_fields_where_declared: true - validate_xml_mapping: true - naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware - identity_generation_preferences: - Doctrine\DBAL\Platforms\PostgreSQLPlatform: identity - mappings: - App: - type: attribute - is_bundle: false - dir: '%kernel.project_dir%/src/Entity' - prefix: 'App\Entity' - alias: App - dql: - string_functions: - regexp: App\Doctrine\Functions\Regexp - field: DoctrineExtensions\Query\Mysql\Field - field2: App\Doctrine\Functions\Field2 - natsort: App\Doctrine\Functions\Natsort - array_position: App\Doctrine\Functions\ArrayPosition - ilike: App\Doctrine\Functions\ILike - connection: default - - migration_source: - <<: *common_orm_settings - connection: migration_source + dql: + string_functions: + regexp: App\Doctrine\Functions\Regexp + field: DoctrineExtensions\Query\Mysql\Field + field2: App\Doctrine\Functions\Field2 + natsort: App\Doctrine\Functions\Natsort + array_position: App\Doctrine\Functions\ArrayPosition + ilike: App\Doctrine\Functions\ILike when@test: doctrine: diff --git a/config/reference.php b/config/reference.php index 3ed46fd1..756dc446 100644 --- a/config/reference.php +++ b/config/reference.php @@ -1622,6 +1622,7 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param; * flysystem?: array{ * filesystem_service: scalar|null|Param, * }, + * asset_mapper?: array, * chain?: array{ * loaders: list, * }, @@ -2301,11 +2302,13 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param; * controllers_json?: scalar|null|Param, // Default: "%kernel.project_dir%/assets/controllers.json" * } * @psalm-type UxTranslatorConfig = array{ - * dump_directory?: scalar|null|Param, // Default: "%kernel.project_dir%/var/translations" + * dump_directory?: scalar|null|Param, // The directory where translations and TypeScript types are dumped. // Default: "%kernel.project_dir%/var/translations" + * dump_typescript?: bool|Param, // Control whether TypeScript types are dumped alongside translations. Disable this if you do not use TypeScript (e.g. in production when using AssetMapper). // Default: true * domains?: string|array{ // List of domains to include/exclude from the generated translations. Prefix with a `!` to exclude a domain. * type?: scalar|null|Param, * elements?: list, * }, + * keys_patterns?: list, * } * @psalm-type DompdfFontLoaderConfig = array{ * autodiscovery?: bool|array{ diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBMigrationCommand.php index 0c58e6b9..812d1c36 100644 --- a/src/Command/Migrations/DBMigrationCommand.php +++ b/src/Command/Migrations/DBMigrationCommand.php @@ -26,8 +26,11 @@ namespace App\Command\Migrations; use App\DataTables\Helpers\ColumnSortHelper; use App\Entity\Parts\Manufacturer; use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; +use Doctrine\Bundle\DoctrineBundle\ConnectionFactory; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Tools\DsnParser; +use Doctrine\ORM\EntityManager; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; @@ -36,6 +39,7 @@ use Doctrine\Persistence\ManagerRegistry; use Doctrine\Persistence\ObjectManager; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; @@ -43,23 +47,40 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand('partdb:migrate-db', 'Migrate the database to a different platform')] class DBMigrationCommand extends Command { - private readonly EntityManagerInterface $sourceEM; - private readonly EntityManagerInterface $targetEM; + private ?EntityManagerInterface $sourceEM = null; - public function __construct(private readonly ManagerRegistry $managerRegistry, + public function __construct( + private readonly EntityManagerInterface $targetEM, private readonly PKImportHelper $importHelper, ) { - $this->sourceEM = $this->managerRegistry->getManager('migration_source'); - $this->targetEM = $this->managerRegistry->getManager('default'); - parent::__construct(); } + public function configure(): void + { + $this-> + addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); + } + + /** + * Construct a source EntityManager based on the given connection URL + * @param string $url + * @return EntityManagerInterface + */ + private function getSourceEm(string $url): EntityManagerInterface + { + $connectionFactory = new ConnectionFactory(); + $connection = $connectionFactory->createConnection(['url' => $url]); + return new EntityManager($connection, $this->targetEM->getConfiguration()); + } + public function execute(InputInterface $input, OutputInterface $output): int { $io = new SymfonyStyle($input, $output); + $this->sourceEM = $this->getSourceEm($input->getArgument('url')); + // Example migration logic (to be replaced with actual migration code) $io->info('Starting database migration...'); From 343ad6beff4cfa492cb5941b0bbba856525ba903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Thu, 8 Jan 2026 22:16:38 +0100 Subject: [PATCH 06/11] Check that databases are up to date --- ...mmand.php => DBPlatformConvertCommand.php} | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) rename src/Command/Migrations/{DBMigrationCommand.php => DBPlatformConvertCommand.php} (70%) diff --git a/src/Command/Migrations/DBMigrationCommand.php b/src/Command/Migrations/DBPlatformConvertCommand.php similarity index 70% rename from src/Command/Migrations/DBMigrationCommand.php rename to src/Command/Migrations/DBPlatformConvertCommand.php index 812d1c36..61a34c31 100644 --- a/src/Command/Migrations/DBMigrationCommand.php +++ b/src/Command/Migrations/DBPlatformConvertCommand.php @@ -23,20 +23,17 @@ declare(strict_types=1); namespace App\Command\Migrations; -use App\DataTables\Helpers\ColumnSortHelper; -use App\Entity\Parts\Manufacturer; use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; use Doctrine\Bundle\DoctrineBundle\ConnectionFactory; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; -use Doctrine\DBAL\Tools\DsnParser; +use Doctrine\Migrations\Configuration\EntityManager\ExistingEntityManager; +use Doctrine\Migrations\Configuration\Migration\ExistingConfiguration; use Doctrine\ORM\EntityManager; use Doctrine\ORM\EntityManagerInterface; - +use Doctrine\Migrations\DependencyFactory; use Doctrine\ORM\Id\AssignedGenerator; use Doctrine\ORM\Mapping\ClassMetadata; -use Doctrine\Persistence\ManagerRegistry; -use Doctrine\Persistence\ObjectManager; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -44,14 +41,14 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; -#[AsCommand('partdb:migrate-db', 'Migrate the database to a different platform')] -class DBMigrationCommand extends Command +#[AsCommand('partdb:migrations:convert-db-platform', 'Convert the database to a different platform')] +class DBPlatformConvertCommand extends Command { - private ?EntityManagerInterface $sourceEM = null; public function __construct( private readonly EntityManagerInterface $targetEM, private readonly PKImportHelper $importHelper, + private readonly DependencyFactory $dependencyFactory, ) { parent::__construct(); @@ -60,26 +57,23 @@ class DBMigrationCommand extends Command public function configure(): void { $this-> - addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); - } - - /** - * Construct a source EntityManager based on the given connection URL - * @param string $url - * @return EntityManagerInterface - */ - private function getSourceEm(string $url): EntityManagerInterface - { - $connectionFactory = new ConnectionFactory(); - $connection = $connectionFactory->createConnection(['url' => $url]); - return new EntityManager($connection, $this->targetEM->getConfiguration()); + addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); } public function execute(InputInterface $input, OutputInterface $output): int { $io = new SymfonyStyle($input, $output); - $this->sourceEM = $this->getSourceEm($input->getArgument('url')); + $sourceEM = $this->getSourceEm($input->getArgument('url')); + + //Check that both databases are not using the same driver + if ($sourceEM->getConnection()->getDatabasePlatform()::class === $this->targetEM->getConnection()->getDatabasePlatform()::class) { + $io->error('Source and target database are using the same database platform / driver. This command is only intended to migrate between different database platforms (e.g. from MySQL to PostgreSQL).'); + return 1; + } + + + $this->ensureVersionUpToDate($sourceEM); // Example migration logic (to be replaced with actual migration code) $io->info('Starting database migration...'); @@ -115,7 +109,7 @@ class DBMigrationCommand extends Command $io->progressStart(count($metadata)); - //Afterwards we migrate all entities + //Afterward we migrate all entities foreach ($metadata as $metadatum) { //skip all superclasses if ($metadatum->isMappedSuperclass) { @@ -126,7 +120,7 @@ class DBMigrationCommand extends Command $io->note('Migrating entity: ' . $entityClass); - $repo = $this->sourceEM->getRepository($entityClass); + $repo = $sourceEM->getRepository($entityClass); $items = $repo->findAll(); foreach ($items as $index => $item) { $this->targetEM->persist($item); @@ -136,20 +130,12 @@ class DBMigrationCommand extends Command $io->progressFinish(); - //Migrate all manufacturers from source to target - /*$manufacturerRepo = $this->sourceEM->getRepository(Manufacturer::class); - $manufacturers = $manufacturerRepo->findAll(); - foreach ($manufacturers as $manufacturer) { - $this->targetEM->persist($manufacturer); - } - $this->targetEM->flush(); - */ //Fix sequences / auto increment values on target database $io->info('Fixing sequences / auto increment values on target database...'); $this->fixAutoIncrements($this->targetEM); - $output->writeln('Database migration completed successfully.'); + $io->success('Database migration completed successfully.'); if ($io->isVerbose()) { $io->info('Process took peak memory: ' . round(memory_get_peak_usage(true) / 1024 / 1024, 2) . ' MB'); @@ -158,6 +144,35 @@ class DBMigrationCommand extends Command return Command::SUCCESS; } + /** + * Construct a source EntityManager based on the given connection URL + * @param string $url + * @return EntityManagerInterface + */ + private function getSourceEm(string $url): EntityManagerInterface + { + $connectionFactory = new ConnectionFactory(); + $connection = $connectionFactory->createConnection(['url' => $url]); + return new EntityManager($connection, $this->targetEM->getConfiguration()); + } + + private function ensureVersionUpToDate(EntityManagerInterface $sourceEM): void + { + //Ensure that target database is up to date + $migrationStatusCalculator = $this->dependencyFactory->getMigrationStatusCalculator(); + $newMigrations = $migrationStatusCalculator->getNewMigrations(); + if (count($newMigrations->getItems()) > 0) { + throw new \RuntimeException("Target database is not up to date. Please run all migrations (with doctrine:migrations:migrate) before starting the migration process."); + } + + $sourceDependencyLoader = DependencyFactory::fromEntityManager(new ExistingConfiguration($this->dependencyFactory->getConfiguration()), new ExistingEntityManager($sourceEM)); + $sourceMigrationStatusCalculator = $sourceDependencyLoader->getMigrationStatusCalculator(); + $sourceNewMigrations = $sourceMigrationStatusCalculator->getNewMigrations(); + if (count($sourceNewMigrations->getItems()) > 0) { + throw new \RuntimeException("Source database is not up to date. Please run all migrations (with doctrine:migrations:migrate) on the source database before starting the migration process."); + } + } + private function fixAutoIncrements(EntityManagerInterface $em): void { $connection = $em->getConnection(); @@ -165,7 +180,7 @@ class DBMigrationCommand extends Command if ($platform instanceof PostgreSQLPlatform) { $connection->executeStatement( - //From: https://wiki.postgresql.org/wiki/Fixing_Sequences + //From: https://wiki.postgresql.org/wiki/Fixing_Sequences << Date: Thu, 8 Jan 2026 22:22:07 +0100 Subject: [PATCH 07/11] Support %kernel.project_dir% in db conversion tool --- .../Migrations/DBPlatformConvertCommand.php | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Command/Migrations/DBPlatformConvertCommand.php b/src/Command/Migrations/DBPlatformConvertCommand.php index 61a34c31..91172920 100644 --- a/src/Command/Migrations/DBPlatformConvertCommand.php +++ b/src/Command/Migrations/DBPlatformConvertCommand.php @@ -40,6 +40,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; +use Symfony\Component\DependencyInjection\Attribute\Autowire; #[AsCommand('partdb:migrations:convert-db-platform', 'Convert the database to a different platform')] class DBPlatformConvertCommand extends Command @@ -49,6 +50,8 @@ class DBPlatformConvertCommand extends Command private readonly EntityManagerInterface $targetEM, private readonly PKImportHelper $importHelper, private readonly DependencyFactory $dependencyFactory, + #[Autowire('%kernel.project_dir%')] + private readonly string $kernelProjectDir, ) { parent::__construct(); @@ -57,7 +60,7 @@ class DBPlatformConvertCommand extends Command public function configure(): void { $this-> - addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); + addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); } public function execute(InputInterface $input, OutputInterface $output): int @@ -75,6 +78,19 @@ class DBPlatformConvertCommand extends Command $this->ensureVersionUpToDate($sourceEM); + $io->note('This command is still in development. If you encounter any problems, please report them to the issue tracker on GitHub.'); + $io->warning(sprintf('This command will delete all existing data in the target database "%s". Make sure that you have no important data in the database before you continue!', + $this->targetEM->getConnection()->getDatabase() ?? 'unknown' + )); + + $io->ask('Please type "DELETE ALL DATA" to continue.', '', function ($answer) { + if (strtoupper($answer) !== 'DELETE ALL DATA') { + throw new \RuntimeException('You did not type "DELETE ALL DATA"!'); + } + return $answer; + }); + + // Example migration logic (to be replaced with actual migration code) $io->info('Starting database migration...'); @@ -151,6 +167,9 @@ class DBPlatformConvertCommand extends Command */ private function getSourceEm(string $url): EntityManagerInterface { + //Replace any %kernel.project_dir% placeholders + $url = str_replace('%kernel.project_dir%', $this->kernelProjectDir, $url); + $connectionFactory = new ConnectionFactory(); $connection = $connectionFactory->createConnection(['url' => $url]); return new EntityManager($connection, $this->targetEM->getConfiguration()); From ddbfc87ce16c86977c0355d99cf5f7aca1726baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Thu, 8 Jan 2026 22:22:47 +0100 Subject: [PATCH 08/11] Set help for DBPlatformConvertCommand --- src/Command/Migrations/DBPlatformConvertCommand.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Command/Migrations/DBPlatformConvertCommand.php b/src/Command/Migrations/DBPlatformConvertCommand.php index 91172920..80dc332e 100644 --- a/src/Command/Migrations/DBPlatformConvertCommand.php +++ b/src/Command/Migrations/DBPlatformConvertCommand.php @@ -59,8 +59,9 @@ class DBPlatformConvertCommand extends Command public function configure(): void { - $this-> - addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); + $this + ->setHelp('This command allows you to migrate the database from one database platform to another (e.g. from MySQL to PostgreSQL).') + ->addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); } public function execute(InputInterface $input, OutputInterface $output): int From 300ee33be27f08c3c0678a7c32fd154a9972e0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Fri, 9 Jan 2026 19:46:09 +0100 Subject: [PATCH 09/11] Allow to continue even if source and target db platform are the same --- src/Command/Migrations/DBPlatformConvertCommand.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Command/Migrations/DBPlatformConvertCommand.php b/src/Command/Migrations/DBPlatformConvertCommand.php index 80dc332e..256a4db1 100644 --- a/src/Command/Migrations/DBPlatformConvertCommand.php +++ b/src/Command/Migrations/DBPlatformConvertCommand.php @@ -72,8 +72,11 @@ class DBPlatformConvertCommand extends Command //Check that both databases are not using the same driver if ($sourceEM->getConnection()->getDatabasePlatform()::class === $this->targetEM->getConnection()->getDatabasePlatform()::class) { - $io->error('Source and target database are using the same database platform / driver. This command is only intended to migrate between different database platforms (e.g. from MySQL to PostgreSQL).'); - return 1; + $io->warning('Source and target database are using the same database platform / driver. This command is only intended to migrate between different database platforms (e.g. from MySQL to PostgreSQL).'); + if (!$io->confirm('Do you want to continue anyway?', false)) { + $io->info('Aborting migration process.'); + return Command::SUCCESS; + } } @@ -219,6 +222,6 @@ class DBPlatformConvertCommand extends Command AND T.relname = PGT.tablename ORDER BY S.relname; SQL); - }; + } } } From aefb69c51e625576534d82add8f6e73cd4a1f108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Fri, 9 Jan 2026 21:17:51 +0100 Subject: [PATCH 10/11] Fixed error that users could not be converted due to settings and backupCodes not allowed as nullable --- .../Migrations/DBPlatformConvertCommand.php | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Command/Migrations/DBPlatformConvertCommand.php b/src/Command/Migrations/DBPlatformConvertCommand.php index 256a4db1..86052bf7 100644 --- a/src/Command/Migrations/DBPlatformConvertCommand.php +++ b/src/Command/Migrations/DBPlatformConvertCommand.php @@ -23,6 +23,7 @@ declare(strict_types=1); namespace App\Command\Migrations; +use App\Entity\UserSystem\User; use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; use Doctrine\Bundle\DoctrineBundle\ConnectionFactory; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; @@ -87,6 +88,9 @@ class DBPlatformConvertCommand extends Command $this->targetEM->getConnection()->getDatabase() ?? 'unknown' )); + //$users = $sourceEM->getRepository(User::class)->findAll(); + //dump($users); + $io->ask('Please type "DELETE ALL DATA" to continue.', '', function ($answer) { if (strtoupper($answer) !== 'DELETE ALL DATA') { throw new \RuntimeException('You did not type "DELETE ALL DATA"!'); @@ -120,6 +124,7 @@ class DBPlatformConvertCommand extends Command foreach ($metadatum->getAssociationNames() as $fieldName) { $mapping = $metadatum->getAssociationMapping($fieldName); $mapping->cascade = array_unique(array_merge($mapping->cascade, ['persist'])); + $mapping->fetch = ClassMetadata::FETCH_EAGER; //Avoid lazy loading issues during migration $metadatum->setIdGeneratorType(ClassMetadata::GENERATOR_TYPE_NONE); $metadatum->setIdGenerator(new AssignedGenerator()); @@ -129,6 +134,10 @@ class DBPlatformConvertCommand extends Command $io->progressStart(count($metadata)); + //First we migrate users to avoid foreign key constraint issues + $io->info('Migrating users first to avoid foreign key constraint issues...'); + $this->fixUsers($sourceEM); + //Afterward we migrate all entities foreach ($metadata as $metadatum) { //skip all superclasses @@ -196,6 +205,23 @@ class DBPlatformConvertCommand extends Command } } + private function fixUsers(EntityManagerInterface $sourceEM): void + { + //To avoid a problem with (Column 'settings' cannot be null) in MySQL we need to migrate the user entities first + //and fix the settings and backupCodes fields + + $reflClass = new \ReflectionClass(User::class); + foreach ($sourceEM->getRepository(User::class)->findAll() as $user) { + foreach (['settings', 'backupCodes'] as $field) { + $property = $reflClass->getProperty($field); + if (!$property->isInitialized($user) || $property->getValue($user) === null) { + $property->setValue($user, []); + } + } + $this->targetEM->persist($user); + } + } + private function fixAutoIncrements(EntityManagerInterface $em): void { $connection = $em->getConnection(); From 6c3e4d788055af049a30bdb3fc7f901e0bb1b001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 10 Jan 2026 21:14:27 +0100 Subject: [PATCH 11/11] Added documentation about the database conversion command --- docs/installation/choosing_database.md | 24 ++++++++++++++++++++++-- docs/usage/console_commands.md | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/installation/choosing_database.md b/docs/installation/choosing_database.md index 8a070120..27d70e54 100644 --- a/docs/installation/choosing_database.md +++ b/docs/installation/choosing_database.md @@ -21,8 +21,8 @@ differences between them, which might be important for you. Therefore the pros a are listed here. {: .important } -You have to choose between the database types before you start using Part-DB and **you can not change it (easily) after -you have started creating data**. So you should choose the database type for your use case (and possible future uses). +While you can change the database platform later (see below), it is still experimental and not recommended. +So you should choose the database type for your use case (and possible future uses). ## Comparison @@ -180,3 +180,23 @@ and it is automatically used if available. For SQLite and MySQL < 10.7 it has to be emulated if wanted, which is pretty slow. Therefore it has to be explicitly enabled by setting the `DATABASE_EMULATE_NATURAL_SORT` environment variable to `1`. If it is 0 the classical binary sorting is used, on these databases. The emulations might have some quirks and issues, so it is recommended to use a database which supports natural sorting natively, if you want to use it. + +## Converting between database platforms + +{: .important } +The database conversion is still experimental. Therefore it is recommended to backup your database before performing a conversion, and check if everything works as expected afterwards. + +If you want to change the database platform of your Part-DB installation (e.g. from SQLite to MySQL/MariaDB or PostgreSQL, or vice versa), there is the `partdb:migrations:convert-db-platform` console command, which can help you with that: + +1. Make a backup of your current database to be safe if something goes wrong (see the backup documentation). +2. Ensure that your database is at the latest schema by running the migrations: `php bin/console doctrine:migrations:migrate` +3. Change the `DATABASE_URL` environment variable to the new database platform and connection information. Copy the old `DATABASE_URL` as you will need it later. +4. Run `php bin/console doctrine:migrations:migrate` again to create the database schema in the new database. You will not need the admin password, that is shown when running the migrations. +5. Run the conversion command, where you have to provide the old `DATABASE_URL` as parameter: `php bin/console partdb:migrations:convert-db-platform ` + Replace `