diff --git a/config/reference.php b/config/reference.php index 3ed46fd1..756dc446 100644 --- a/config/reference.php +++ b/config/reference.php @@ -1622,6 +1622,7 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param; * flysystem?: array{ * filesystem_service: scalar|null|Param, * }, + * asset_mapper?: array, * chain?: array{ * loaders: list, * }, @@ -2301,11 +2302,13 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param; * controllers_json?: scalar|null|Param, // Default: "%kernel.project_dir%/assets/controllers.json" * } * @psalm-type UxTranslatorConfig = array{ - * dump_directory?: scalar|null|Param, // Default: "%kernel.project_dir%/var/translations" + * dump_directory?: scalar|null|Param, // The directory where translations and TypeScript types are dumped. // Default: "%kernel.project_dir%/var/translations" + * dump_typescript?: bool|Param, // Control whether TypeScript types are dumped alongside translations. Disable this if you do not use TypeScript (e.g. in production when using AssetMapper). // Default: true * domains?: string|array{ // List of domains to include/exclude from the generated translations. Prefix with a `!` to exclude a domain. * type?: scalar|null|Param, * elements?: list, * }, + * keys_patterns?: list, * } * @psalm-type DompdfFontLoaderConfig = array{ * autodiscovery?: bool|array{ diff --git a/docs/installation/choosing_database.md b/docs/installation/choosing_database.md index 8a070120..27d70e54 100644 --- a/docs/installation/choosing_database.md +++ b/docs/installation/choosing_database.md @@ -21,8 +21,8 @@ differences between them, which might be important for you. Therefore the pros a are listed here. {: .important } -You have to choose between the database types before you start using Part-DB and **you can not change it (easily) after -you have started creating data**. So you should choose the database type for your use case (and possible future uses). +While you can change the database platform later (see below), it is still experimental and not recommended. +So you should choose the database type for your use case (and possible future uses). ## Comparison @@ -180,3 +180,23 @@ and it is automatically used if available. For SQLite and MySQL < 10.7 it has to be emulated if wanted, which is pretty slow. Therefore it has to be explicitly enabled by setting the `DATABASE_EMULATE_NATURAL_SORT` environment variable to `1`. If it is 0 the classical binary sorting is used, on these databases. The emulations might have some quirks and issues, so it is recommended to use a database which supports natural sorting natively, if you want to use it. + +## Converting between database platforms + +{: .important } +The database conversion is still experimental. Therefore it is recommended to backup your database before performing a conversion, and check if everything works as expected afterwards. + +If you want to change the database platform of your Part-DB installation (e.g. from SQLite to MySQL/MariaDB or PostgreSQL, or vice versa), there is the `partdb:migrations:convert-db-platform` console command, which can help you with that: + +1. Make a backup of your current database to be safe if something goes wrong (see the backup documentation). +2. Ensure that your database is at the latest schema by running the migrations: `php bin/console doctrine:migrations:migrate` +3. Change the `DATABASE_URL` environment variable to the new database platform and connection information. Copy the old `DATABASE_URL` as you will need it later. +4. Run `php bin/console doctrine:migrations:migrate` again to create the database schema in the new database. You will not need the admin password, that is shown when running the migrations. +5. Run the conversion command, where you have to provide the old `DATABASE_URL` as parameter: `php bin/console partdb:migrations:convert-db-platform ` + Replace `. + */ + +declare(strict_types=1); + + +namespace App\Command\Migrations; + +use App\Entity\UserSystem\User; +use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper; +use Doctrine\Bundle\DoctrineBundle\ConnectionFactory; +use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\Migrations\Configuration\EntityManager\ExistingEntityManager; +use Doctrine\Migrations\Configuration\Migration\ExistingConfiguration; +use Doctrine\ORM\EntityManager; +use Doctrine\ORM\EntityManagerInterface; +use Doctrine\Migrations\DependencyFactory; +use Doctrine\ORM\Id\AssignedGenerator; +use Doctrine\ORM\Mapping\ClassMetadata; +use Symfony\Component\Console\Attribute\AsCommand; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; +use Symfony\Component\DependencyInjection\Attribute\Autowire; + +#[AsCommand('partdb:migrations:convert-db-platform', 'Convert the database to a different platform')] +class DBPlatformConvertCommand extends Command +{ + + public function __construct( + private readonly EntityManagerInterface $targetEM, + private readonly PKImportHelper $importHelper, + private readonly DependencyFactory $dependencyFactory, + #[Autowire('%kernel.project_dir%')] + private readonly string $kernelProjectDir, + ) + { + parent::__construct(); + } + + public function configure(): void + { + $this + ->setHelp('This command allows you to migrate the database from one database platform to another (e.g. from MySQL to PostgreSQL).') + ->addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from'); + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $sourceEM = $this->getSourceEm($input->getArgument('url')); + + //Check that both databases are not using the same driver + if ($sourceEM->getConnection()->getDatabasePlatform()::class === $this->targetEM->getConnection()->getDatabasePlatform()::class) { + $io->warning('Source and target database are using the same database platform / driver. This command is only intended to migrate between different database platforms (e.g. from MySQL to PostgreSQL).'); + if (!$io->confirm('Do you want to continue anyway?', false)) { + $io->info('Aborting migration process.'); + return Command::SUCCESS; + } + } + + + $this->ensureVersionUpToDate($sourceEM); + + $io->note('This command is still in development. If you encounter any problems, please report them to the issue tracker on GitHub.'); + $io->warning(sprintf('This command will delete all existing data in the target database "%s". Make sure that you have no important data in the database before you continue!', + $this->targetEM->getConnection()->getDatabase() ?? 'unknown' + )); + + //$users = $sourceEM->getRepository(User::class)->findAll(); + //dump($users); + + $io->ask('Please type "DELETE ALL DATA" to continue.', '', function ($answer) { + if (strtoupper($answer) !== 'DELETE ALL DATA') { + throw new \RuntimeException('You did not type "DELETE ALL DATA"!'); + } + return $answer; + }); + + + // Example migration logic (to be replaced with actual migration code) + $io->info('Starting database migration...'); + + //Disable all event listeners on target EM to avoid unwanted side effects + $eventManager = $this->targetEM->getEventManager(); + foreach ($eventManager->getAllListeners() as $event => $listeners) { + foreach ($listeners as $listener) { + $eventManager->removeEventListener($event, $listener); + } + } + + $io->info('Clear target database...'); + $this->importHelper->purgeDatabaseForImport($this->targetEM, ['internal', 'migration_versions']); + + $metadata = $this->targetEM->getMetadataFactory()->getAllMetadata(); + + $io->info('Modifying entity metadata for migration...'); + //First we modify each entity metadata to have an persist cascade on all relations + foreach ($metadata as $metadatum) { + $entityClass = $metadatum->getName(); + $io->writeln('Modifying cascade and ID settings for entity: ' . $entityClass, OutputInterface::VERBOSITY_VERBOSE); + + foreach ($metadatum->getAssociationNames() as $fieldName) { + $mapping = $metadatum->getAssociationMapping($fieldName); + $mapping->cascade = array_unique(array_merge($mapping->cascade, ['persist'])); + $mapping->fetch = ClassMetadata::FETCH_EAGER; //Avoid lazy loading issues during migration + + $metadatum->setIdGeneratorType(ClassMetadata::GENERATOR_TYPE_NONE); + $metadatum->setIdGenerator(new AssignedGenerator()); + } + } + + + $io->progressStart(count($metadata)); + + //First we migrate users to avoid foreign key constraint issues + $io->info('Migrating users first to avoid foreign key constraint issues...'); + $this->fixUsers($sourceEM); + + //Afterward we migrate all entities + foreach ($metadata as $metadatum) { + //skip all superclasses + if ($metadatum->isMappedSuperclass) { + continue; + } + + $entityClass = $metadatum->getName(); + + $io->note('Migrating entity: ' . $entityClass); + + $repo = $sourceEM->getRepository($entityClass); + $items = $repo->findAll(); + foreach ($items as $index => $item) { + $this->targetEM->persist($item); + } + $this->targetEM->flush(); + } + + $io->progressFinish(); + + + //Fix sequences / auto increment values on target database + $io->info('Fixing sequences / auto increment values on target database...'); + $this->fixAutoIncrements($this->targetEM); + + $io->success('Database migration completed successfully.'); + + if ($io->isVerbose()) { + $io->info('Process took peak memory: ' . round(memory_get_peak_usage(true) / 1024 / 1024, 2) . ' MB'); + } + + return Command::SUCCESS; + } + + /** + * Construct a source EntityManager based on the given connection URL + * @param string $url + * @return EntityManagerInterface + */ + private function getSourceEm(string $url): EntityManagerInterface + { + //Replace any %kernel.project_dir% placeholders + $url = str_replace('%kernel.project_dir%', $this->kernelProjectDir, $url); + + $connectionFactory = new ConnectionFactory(); + $connection = $connectionFactory->createConnection(['url' => $url]); + return new EntityManager($connection, $this->targetEM->getConfiguration()); + } + + private function ensureVersionUpToDate(EntityManagerInterface $sourceEM): void + { + //Ensure that target database is up to date + $migrationStatusCalculator = $this->dependencyFactory->getMigrationStatusCalculator(); + $newMigrations = $migrationStatusCalculator->getNewMigrations(); + if (count($newMigrations->getItems()) > 0) { + throw new \RuntimeException("Target database is not up to date. Please run all migrations (with doctrine:migrations:migrate) before starting the migration process."); + } + + $sourceDependencyLoader = DependencyFactory::fromEntityManager(new ExistingConfiguration($this->dependencyFactory->getConfiguration()), new ExistingEntityManager($sourceEM)); + $sourceMigrationStatusCalculator = $sourceDependencyLoader->getMigrationStatusCalculator(); + $sourceNewMigrations = $sourceMigrationStatusCalculator->getNewMigrations(); + if (count($sourceNewMigrations->getItems()) > 0) { + throw new \RuntimeException("Source database is not up to date. Please run all migrations (with doctrine:migrations:migrate) on the source database before starting the migration process."); + } + } + + private function fixUsers(EntityManagerInterface $sourceEM): void + { + //To avoid a problem with (Column 'settings' cannot be null) in MySQL we need to migrate the user entities first + //and fix the settings and backupCodes fields + + $reflClass = new \ReflectionClass(User::class); + foreach ($sourceEM->getRepository(User::class)->findAll() as $user) { + foreach (['settings', 'backupCodes'] as $field) { + $property = $reflClass->getProperty($field); + if (!$property->isInitialized($user) || $property->getValue($user) === null) { + $property->setValue($user, []); + } + } + $this->targetEM->persist($user); + } + } + + private function fixAutoIncrements(EntityManagerInterface $em): void + { + $connection = $em->getConnection(); + $platform = $connection->getDatabasePlatform(); + + if ($platform instanceof PostgreSQLPlatform) { + $connection->executeStatement( + //From: https://wiki.postgresql.org/wiki/Fixing_Sequences + <<em, ['users', 'groups', 'u2f_keys', 'internal', 'migration_versions']); + $purger = new ResetAutoIncrementORMPurger($entityManager ?? $this->em, $excluded_tables); $purger->purge(); }