Merge branch 'db_converter'

config/reference.php

@@ -1622,6 +1622,7 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param;
  *         flysystem?: array{
  *             filesystem_service: scalar|null|Param,
  *         },
+ *         asset_mapper?: array<mixed>,
  *         chain?: array{
  *             loaders: list<scalar|null|Param>,
  *         },
@@ -2301,11 +2302,13 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param;
  *     controllers_json?: scalar|null|Param, // Default: "%kernel.project_dir%/assets/controllers.json"
  * }
  * @psalm-type UxTranslatorConfig = array{
- *     dump_directory?: scalar|null|Param, // Default: "%kernel.project_dir%/var/translations"
+ *     dump_directory?: scalar|null|Param, // The directory where translations and TypeScript types are dumped. // Default: "%kernel.project_dir%/var/translations"
+ *     dump_typescript?: bool|Param, // Control whether TypeScript types are dumped alongside translations. Disable this if you do not use TypeScript (e.g. in production when using AssetMapper). // Default: true
  *     domains?: string|array{ // List of domains to include/exclude from the generated translations. Prefix with a `!` to exclude a domain.
  *         type?: scalar|null|Param,
  *         elements?: list<scalar|null|Param>,
  *     },
+ *     keys_patterns?: list<scalar|null|Param>,
  * }
  * @psalm-type DompdfFontLoaderConfig = array{
  *     autodiscovery?: bool|array{

docs/installation/choosing_database.md

@@ -21,8 +21,8 @@ differences between them, which might be important for you. Therefore the pros a
 are listed here.
 
 {: .important }
-You have to choose between the database types before you start using Part-DB and **you can not change it (easily) after
-you have started creating data**. So you should choose the database type for your use case (and possible future uses).
+While you can change the database platform later (see below), it is still experimental and not recommended. 
+So you should choose the database type for your use case (and possible future uses).
 
 ## Comparison
 
@@ -180,3 +180,23 @@ and it is automatically used if available.
 For SQLite and MySQL < 10.7 it has to be emulated if wanted, which is pretty slow. Therefore it has to be explicitly enabled by setting the
 `DATABASE_EMULATE_NATURAL_SORT` environment variable to `1`. If it is 0 the classical binary sorting is used, on these databases. The emulations
 might have some quirks and issues, so it is recommended to use a database which supports natural sorting natively, if you want to use it.
+
+## Converting between database platforms
+
+{: .important }
+The database conversion is still experimental. Therefore it is recommended to backup your database before performing a conversion, and check if everything works as expected afterwards.
+
+If you want to change the database platform of your Part-DB installation (e.g. from SQLite to MySQL/MariaDB or PostgreSQL, or vice versa), there is the `partdb:migrations:convert-db-platform` console command, which can help you with that:
+
+1. Make a backup of your current database to be safe if something goes wrong (see the backup documentation).
+2. Ensure that your database is at the latest schema by running the migrations: `php bin/console doctrine:migrations:migrate`
+3. Change the `DATABASE_URL` environment variable to the new database platform and connection information. Copy the old `DATABASE_URL` as you will need it later.
+4. Run `php bin/console doctrine:migrations:migrate` again to create the database schema in the new database. You will not need the admin password, that is shown when running the migrations.
+5. Run the conversion command, where you have to provide the old `DATABASE_URL` as parameter: `php bin/console partdb:migrations:convert-db-platform <OLD_DATABASE_URL>`
+   Replace `<OLD_DATABASE_URL` with the actual old `DATABASE_URL` value (e.g. `sqlite:///%kernel.project_dir%/var/app.db`):
+   ```bash
+   php bin/console partdb:migrations:convert-db-platform sqlite:///%kernel.project_dir%/var/app.db
+   ```
+6. The command will purge all data in the new database and copy all data from the old database to the new one. This might take some time and memory depending on the size of your database.
+7. Clear the cache: `php bin/console partdb:cache:clear`
+8. You can login with your existing user accounts in the new database now. Check if everything works as expected.

docs/usage/console_commands.md

@@ -68,6 +68,7 @@ docker exec --user=www-data partdb php bin/console cache:clear
   deleted!*
 * `settings:migrate-env-to-settings`: Migrate configuration from environment variables to the settings interface.
 The value of the environment variable is copied to the settings database, so the environment variable can be removed afterwards without losing the configuration.
+* `partdb:migrations:convert-db-platform`: Convert the database platform (e.g. from SQLite to MySQL/MariaDB or PostgreSQL, or vice versa).
 
 ## Database commands
 

src/Command/Migrations/DBPlatformConvertCommand.php

@@ -0,0 +1,253 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2026 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\Command\Migrations;
+
+use App\Entity\UserSystem\User;
+use App\Services\ImportExportSystem\PartKeeprImporter\PKImportHelper;
+use Doctrine\Bundle\DoctrineBundle\ConnectionFactory;
+use Doctrine\DBAL\Platforms\AbstractMySQLPlatform;
+use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
+use Doctrine\Migrations\Configuration\EntityManager\ExistingEntityManager;
+use Doctrine\Migrations\Configuration\Migration\ExistingConfiguration;
+use Doctrine\ORM\EntityManager;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\Migrations\DependencyFactory;
+use Doctrine\ORM\Id\AssignedGenerator;
+use Doctrine\ORM\Mapping\ClassMetadata;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+
+#[AsCommand('partdb:migrations:convert-db-platform', 'Convert the database to a different platform')]
+class DBPlatformConvertCommand extends Command
+{
+
+    public function __construct(
+        private readonly EntityManagerInterface $targetEM,
+        private readonly PKImportHelper $importHelper,
+        private readonly DependencyFactory $dependencyFactory,
+        #[Autowire('%kernel.project_dir%')]
+        private readonly string $kernelProjectDir,
+    )
+    {
+        parent::__construct();
+    }
+
+    public function configure(): void
+    {
+        $this
+            ->setHelp('This command allows you to migrate the database from one database platform to another (e.g. from MySQL to PostgreSQL).')
+            ->addArgument('url', InputArgument::REQUIRED, 'The database connection URL of the source database to migrate from');
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $io = new SymfonyStyle($input, $output);
+
+        $sourceEM = $this->getSourceEm($input->getArgument('url'));
+
+        //Check that both databases are not using the same driver
+        if ($sourceEM->getConnection()->getDatabasePlatform()::class === $this->targetEM->getConnection()->getDatabasePlatform()::class) {
+            $io->warning('Source and target database are using the same database platform / driver. This command is only intended to migrate between different database platforms (e.g. from MySQL to PostgreSQL).');
+            if (!$io->confirm('Do you want to continue anyway?', false)) {
+                $io->info('Aborting migration process.');
+                return Command::SUCCESS;
+            }
+        }
+
+
+        $this->ensureVersionUpToDate($sourceEM);
+
+        $io->note('This command is still in development. If you encounter any problems, please report them to the issue tracker on GitHub.');
+        $io->warning(sprintf('This command will delete all existing data in the target database "%s". Make sure that you have no important data in the database before you continue!',
+            $this->targetEM->getConnection()->getDatabase() ?? 'unknown'
+        ));
+
+        //$users = $sourceEM->getRepository(User::class)->findAll();
+        //dump($users);
+
+        $io->ask('Please type "DELETE ALL DATA" to continue.', '', function ($answer) {
+            if (strtoupper($answer) !== 'DELETE ALL DATA') {
+                throw new \RuntimeException('You did not type "DELETE ALL DATA"!');
+            }
+            return $answer;
+        });
+
+
+        // Example migration logic (to be replaced with actual migration code)
+        $io->info('Starting database migration...');
+
+        //Disable all event listeners on target EM to avoid unwanted side effects
+        $eventManager = $this->targetEM->getEventManager();
+        foreach ($eventManager->getAllListeners() as $event => $listeners) {
+            foreach ($listeners as $listener) {
+                $eventManager->removeEventListener($event, $listener);
+            }
+        }
+
+        $io->info('Clear target database...');
+        $this->importHelper->purgeDatabaseForImport($this->targetEM, ['internal', 'migration_versions']);
+
+        $metadata = $this->targetEM->getMetadataFactory()->getAllMetadata();
+
+        $io->info('Modifying entity metadata for migration...');
+        //First we modify each entity metadata to have an persist cascade on all relations
+        foreach ($metadata as $metadatum) {
+            $entityClass = $metadatum->getName();
+            $io->writeln('Modifying cascade and ID settings for entity: ' . $entityClass, OutputInterface::VERBOSITY_VERBOSE);
+
+            foreach ($metadatum->getAssociationNames() as $fieldName) {
+                $mapping = $metadatum->getAssociationMapping($fieldName);
+                $mapping->cascade = array_unique(array_merge($mapping->cascade, ['persist']));
+                $mapping->fetch = ClassMetadata::FETCH_EAGER; //Avoid lazy loading issues during migration
+
+                $metadatum->setIdGeneratorType(ClassMetadata::GENERATOR_TYPE_NONE);
+                $metadatum->setIdGenerator(new AssignedGenerator());
+            }
+        }
+
+
+        $io->progressStart(count($metadata));
+
+        //First we migrate users to avoid foreign key constraint issues
+        $io->info('Migrating users first to avoid foreign key constraint issues...');
+        $this->fixUsers($sourceEM);
+
+        //Afterward we migrate all entities
+        foreach ($metadata as $metadatum) {
+            //skip all superclasses
+            if ($metadatum->isMappedSuperclass) {
+                continue;
+            }
+
+            $entityClass = $metadatum->getName();
+
+            $io->note('Migrating entity: ' . $entityClass);
+
+            $repo = $sourceEM->getRepository($entityClass);
+            $items = $repo->findAll();
+            foreach ($items as $index => $item) {
+                $this->targetEM->persist($item);
+            }
+            $this->targetEM->flush();
+        }
+
+        $io->progressFinish();
+
+
+        //Fix sequences / auto increment values on target database
+        $io->info('Fixing sequences / auto increment values on target database...');
+        $this->fixAutoIncrements($this->targetEM);
+
+        $io->success('Database migration completed successfully.');
+
+        if ($io->isVerbose()) {
+            $io->info('Process took peak memory: ' . round(memory_get_peak_usage(true) / 1024 / 1024, 2) . ' MB');
+        }
+
+        return Command::SUCCESS;
+    }
+
+    /**
+     * Construct a source EntityManager based on the given connection URL
+     * @param  string  $url
+     * @return EntityManagerInterface
+     */
+    private function getSourceEm(string $url): EntityManagerInterface
+    {
+        //Replace any %kernel.project_dir% placeholders
+        $url = str_replace('%kernel.project_dir%', $this->kernelProjectDir, $url);
+
+        $connectionFactory = new ConnectionFactory();
+        $connection = $connectionFactory->createConnection(['url' => $url]);
+        return new EntityManager($connection, $this->targetEM->getConfiguration());
+    }
+
+    private function ensureVersionUpToDate(EntityManagerInterface $sourceEM): void
+    {
+        //Ensure that target database is up to date
+        $migrationStatusCalculator = $this->dependencyFactory->getMigrationStatusCalculator();
+        $newMigrations = $migrationStatusCalculator->getNewMigrations();
+        if (count($newMigrations->getItems()) > 0) {
+            throw new \RuntimeException("Target database is not up to date. Please run all migrations (with doctrine:migrations:migrate) before starting the migration process.");
+        }
+
+        $sourceDependencyLoader = DependencyFactory::fromEntityManager(new ExistingConfiguration($this->dependencyFactory->getConfiguration()), new ExistingEntityManager($sourceEM));
+        $sourceMigrationStatusCalculator = $sourceDependencyLoader->getMigrationStatusCalculator();
+        $sourceNewMigrations = $sourceMigrationStatusCalculator->getNewMigrations();
+        if (count($sourceNewMigrations->getItems()) > 0) {
+            throw new \RuntimeException("Source database is not up to date. Please run all migrations (with doctrine:migrations:migrate) on the source database before starting the migration process.");
+        }
+    }
+
+    private function fixUsers(EntityManagerInterface $sourceEM): void
+    {
+        //To avoid a problem with (Column 'settings' cannot be null) in MySQL we need to migrate the user entities first
+        //and fix the settings and backupCodes fields
+
+        $reflClass = new \ReflectionClass(User::class);
+        foreach ($sourceEM->getRepository(User::class)->findAll() as $user) {
+            foreach (['settings', 'backupCodes'] as $field) {
+                $property = $reflClass->getProperty($field);
+                if (!$property->isInitialized($user) || $property->getValue($user) === null) {
+                    $property->setValue($user, []);
+                }
+            }
+            $this->targetEM->persist($user);
+        }
+    }
+
+    private function fixAutoIncrements(EntityManagerInterface $em): void
+    {
+        $connection = $em->getConnection();
+        $platform = $connection->getDatabasePlatform();
+
+        if ($platform instanceof PostgreSQLPlatform) {
+            $connection->executeStatement(
+            //From: https://wiki.postgresql.org/wiki/Fixing_Sequences
+                <<<SQL
+                SELECT 'SELECT SETVAL(' ||
+                    quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
+                    ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
+                    quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
+                FROM pg_class AS S,
+                     pg_depend AS D,
+                     pg_class AS T,
+                     pg_attribute AS C,
+                     pg_tables AS PGT
+                WHERE S.relkind = 'S'
+                    AND S.oid = D.objid
+                    AND D.refobjid = T.oid
+                    AND D.refobjid = C.attrelid
+                    AND D.refobjsubid = C.attnum
+                    AND T.relname = PGT.tablename
+                ORDER BY S.relname;
+                SQL);
+        }
+    }
+}

src/Services/ImportExportSystem/PartKeeprImporter/PKImportHelper.php

@@ -39,10 +39,10 @@ class PKImportHelper
      * Existing users and groups are not purged.
      * This is needed to avoid ID collisions.
      */
-    public function purgeDatabaseForImport(): void
+    public function purgeDatabaseForImport(?EntityManagerInterface $entityManager = null, array $excluded_tables = ['users', 'groups', 'u2f_keys', 'internal', 'migration_versions']): void
     {
         //We use the ResetAutoIncrementORMPurger to reset the auto increment values of the tables. Also it normalizes table names before checking for exclusion.
-        $purger = new ResetAutoIncrementORMPurger($this->em, ['users', 'groups', 'u2f_keys', 'internal', 'migration_versions']);
+        $purger = new ResetAutoIncrementORMPurger($entityManager ?? $this->em, $excluded_tables);
         $purger->purge();
     }