From 4277f4228530879f4dd3848b863afba470840194 Mon Sep 17 00:00:00 2001 From: barisgit Date: Sun, 3 Aug 2025 16:14:27 +0200 Subject: [PATCH 01/10] Fix same error as in other branch and add makefile --- makefile | 108 ++++++++++++++++++ .../TimestampableElementProviderTest.php | 19 +-- 2 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 makefile diff --git a/makefile b/makefile new file mode 100644 index 00000000..3f389638 --- /dev/null +++ b/makefile @@ -0,0 +1,108 @@ +# PartDB Makefile for Test Environment Management + +.PHONY: help test-setup test-clean test-db-create test-db-migrate test-cache-clear test-fixtures test-run dev-setup dev-clean dev-db-create dev-db-migrate dev-cache-clear dev-warmup dev-reset deps-install + +# Default target +help: + @echo "PartDB Test Environment Management" + @echo "==================================" + @echo "" + @echo "Available targets:" + @echo " deps-install - Install PHP dependencies with unlimited memory" + @echo "" + @echo "Development Environment:" + @echo " dev-setup - Complete development environment setup (clean, create DB, migrate, warmup)" + @echo " dev-clean - Clean development cache and database files" + @echo " dev-db-create - Create development database (if not exists)" + @echo " dev-db-migrate - Run database migrations for development environment" + @echo " dev-cache-clear - Clear development cache" + @echo " dev-warmup - Warm up development cache" + @echo " dev-reset - Quick development reset (clean + migrate)" + @echo "" + @echo "Test Environment:" + @echo " test-setup - Complete test environment setup (clean, create DB, migrate, load fixtures)" + @echo " test-clean - Clean test cache and database files" + @echo " test-db-create - Create test database (if not exists)" + @echo " test-db-migrate - Run database migrations for test environment" + @echo " test-cache-clear- Clear test cache" + @echo " test-fixtures - Load test fixtures" + @echo " test-run - Run PHPUnit tests" + @echo "" + @echo " help - Show this help message" + +# Install PHP dependencies with unlimited memory +deps-install: + @echo "📦 Installing PHP dependencies..." + COMPOSER_MEMORY_LIMIT=-1 composer install + @echo "✅ Dependencies installed" + +# Complete test environment setup +test-setup: deps-install test-clean test-db-create test-db-migrate test-fixtures + @echo "✅ Test environment setup complete!" + +# Clean test environment +test-clean: + @echo "🧹 Cleaning test environment..." + rm -rf var/cache/test + rm -f var/app_test.db + @echo "✅ Test environment cleaned" + +# Create test database +test-db-create: + @echo "🗄️ Creating test database..." + -php bin/console doctrine:database:create --if-not-exists --env test || echo "⚠️ Database creation failed (expected for SQLite) - continuing..." + +# Run database migrations for test environment +test-db-migrate: + @echo "🔄 Running database migrations..." + php -d memory_limit=1G bin/console doctrine:migrations:migrate -n --env test + +# Clear test cache +test-cache-clear: + @echo "🗑️ Clearing test cache..." + rm -rf var/cache/test + @echo "✅ Test cache cleared" + +# Load test fixtures +test-fixtures: + @echo "📦 Loading test fixtures..." + php bin/console partdb:fixtures:load -n --env test + +# Run PHPUnit tests +test-run: + @echo "🧪 Running tests..." + php bin/phpunit + +# Quick test reset (clean + migrate + fixtures, skip DB creation) +test-reset: test-cache-clear test-db-migrate test-fixtures + @echo "✅ Test environment reset complete!" + +# Development helpers +dev-setup: deps-install dev-clean dev-db-create dev-db-migrate dev-warmup + @echo "✅ Development environment setup complete!" + +dev-clean: + @echo "🧹 Cleaning development environment..." + rm -rf var/cache/dev + rm -f var/app_dev.db + @echo "✅ Development environment cleaned" + +dev-db-create: + @echo "🗄️ Creating development database..." + -php bin/console doctrine:database:create --if-not-exists --env dev || echo "⚠️ Database creation failed (expected for SQLite) - continuing..." + +dev-db-migrate: + @echo "🔄 Running database migrations..." + php -d memory_limit=1G bin/console doctrine:migrations:migrate -n --env dev + +dev-cache-clear: + @echo "🗑️ Clearing development cache..." + rm -rf var/cache/dev + @echo "✅ Development cache cleared" + +dev-warmup: + @echo "🔥 Warming up development cache..." + php -d memory_limit=1G bin/console cache:warmup --env dev -n + +dev-reset: dev-cache-clear dev-db-migrate + @echo "✅ Development environment reset complete!" \ No newline at end of file diff --git a/tests/Services/LabelSystem/PlaceholderProviders/TimestampableElementProviderTest.php b/tests/Services/LabelSystem/PlaceholderProviders/TimestampableElementProviderTest.php index a72f06df..6aa152b9 100644 --- a/tests/Services/LabelSystem/PlaceholderProviders/TimestampableElementProviderTest.php +++ b/tests/Services/LabelSystem/PlaceholderProviders/TimestampableElementProviderTest.php @@ -60,26 +60,29 @@ class TimestampableElementProviderTest extends WebTestCase protected function setUp(): void { self::bootKernel(); - \Locale::setDefault('en'); + \Locale::setDefault('en_US'); $this->service = self::getContainer()->get(TimestampableElementProvider::class); - $this->target = new class() implements TimeStampableInterface { + $this->target = new class () implements TimeStampableInterface { public function getLastModified(): ?DateTime { - return new \DateTime('2000-01-01'); + return new DateTime('2000-01-01'); } public function getAddedDate(): ?DateTime { - return new \DateTime('2000-01-01'); + return new DateTime('2000-01-01'); } }; } public static function dataProvider(): \Iterator { - \Locale::setDefault('en'); - yield ['1/1/00, 12:00 AM', '[[LAST_MODIFIED]]']; - yield ['1/1/00, 12:00 AM', '[[CREATION_DATE]]']; + \Locale::setDefault('en_US'); + // Use IntlDateFormatter like the actual service does + $formatter = new \IntlDateFormatter(\Locale::getDefault(), \IntlDateFormatter::SHORT, \IntlDateFormatter::SHORT); + $expectedFormat = $formatter->format(new DateTime('2000-01-01')); + yield [$expectedFormat, '[[LAST_MODIFIED]]']; + yield [$expectedFormat, '[[CREATION_DATE]]']; } #[DataProvider('dataProvider')] @@ -87,4 +90,4 @@ class TimestampableElementProviderTest extends WebTestCase { $this->assertSame($expected, $this->service->replace($placeholder, $this->target)); } -} +} \ No newline at end of file From d0f2422e0dee9e9c36e14ab98207732cb4c58283 Mon Sep 17 00:00:00 2001 From: barisgit Date: Sun, 3 Aug 2025 18:46:46 +0200 Subject: [PATCH 02/10] Implement functionality to import schematic csv (or any other csv for that matter), with ability to map input columns to output columns with input validation and error handling --- makefile | 2 +- src/Controller/ProjectController.php | 300 ++++++++- .../ImportExportSystem/BOMImporter.php | 593 +++++++++++++++++- .../BOMValidationService.php | 476 ++++++++++++++ .../_bom_validation_results.html.twig | 186 ++++++ .../projects/import_bom_map_fields.html.twig | 204 ++++++ 6 files changed, 1733 insertions(+), 28 deletions(-) create mode 100644 src/Services/ImportExportSystem/BOMValidationService.php create mode 100644 templates/projects/_bom_validation_results.html.twig create mode 100644 templates/projects/import_bom_map_fields.html.twig diff --git a/makefile b/makefile index 3f389638..6b5ac61f 100644 --- a/makefile +++ b/makefile @@ -97,7 +97,7 @@ dev-db-migrate: dev-cache-clear: @echo "🗑️ Clearing development cache..." - rm -rf var/cache/dev + php -d memory_limit=1G bin/console cache:clear --env dev -n @echo "✅ Development cache cleared" dev-warmup: diff --git a/src/Controller/ProjectController.php b/src/Controller/ProjectController.php index a64c1851..444ff5b3 100644 --- a/src/Controller/ProjectController.php +++ b/src/Controller/ProjectController.php @@ -36,6 +36,7 @@ use Doctrine\Common\Collections\ArrayCollection; use Doctrine\ORM\EntityManagerInterface; use League\Csv\SyntaxError; use Omines\DataTablesBundle\DataTableFactory; +use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\Form\Extension\Core\Type\CheckboxType; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; @@ -102,9 +103,14 @@ class ProjectController extends AbstractController $this->addFlash('success', 'project.build.flash.success'); return $this->redirect( - $request->get('_redirect', - $this->generateUrl('project_info', ['id' => $project->getID()] - ))); + $request->get( + '_redirect', + $this->generateUrl( + 'project_info', + ['id' => $project->getID()] + ) + ) + ); } $this->addFlash('error', 'project.build.flash.invalid_input'); @@ -120,9 +126,13 @@ class ProjectController extends AbstractController } #[Route(path: '/{id}/import_bom', name: 'project_import_bom', requirements: ['id' => '\d+'])] - public function importBOM(Request $request, EntityManagerInterface $entityManager, Project $project, - BOMImporter $BOMImporter, ValidatorInterface $validator): Response - { + public function importBOM( + Request $request, + EntityManagerInterface $entityManager, + Project $project, + BOMImporter $BOMImporter, + ValidatorInterface $validator + ): Response { $this->denyAccessUnlessGranted('edit', $project); $builder = $this->createFormBuilder(); @@ -138,6 +148,7 @@ class ProjectController extends AbstractController 'required' => true, 'choices' => [ 'project.bom_import.type.kicad_pcbnew' => 'kicad_pcbnew', + 'project.bom_import.type.kicad_schematic' => 'kicad_schematic', ] ]); $builder->add('clear_existing_bom', CheckboxType::class, [ @@ -161,25 +172,40 @@ class ProjectController extends AbstractController $entityManager->flush(); } + $import_type = $form->get('type')->getData(); + try { + // For schematic imports, redirect to field mapping step + if ($import_type === 'kicad_schematic') { + // Store file content and options in session for field mapping step + $file_content = $form->get('file')->getData()->getContent(); + $clear_existing = $form->get('clear_existing_bom')->getData(); + + $request->getSession()->set('bom_import_data', $file_content); + $request->getSession()->set('bom_import_clear', $clear_existing); + + return $this->redirectToRoute('project_import_bom_map_fields', ['id' => $project->getID()]); + } + + // For PCB imports, proceed directly $entries = $BOMImporter->importFileIntoProject($form->get('file')->getData(), $project, [ - 'type' => $form->get('type')->getData(), + 'type' => $import_type, ]); - //Validate the project entries + // Validate the project entries $errors = $validator->validateProperty($project, 'bom_entries'); - //If no validation errors occured, save the changes and redirect to edit page - if (count ($errors) === 0) { + // If no validation errors occurred, save the changes and redirect to edit page + if (count($errors) === 0) { $this->addFlash('success', t('project.bom_import.flash.success', ['%count%' => count($entries)])); $entityManager->flush(); return $this->redirectToRoute('project_edit', ['id' => $project->getID()]); } - //When we get here, there were validation errors + // When we get here, there were validation errors $this->addFlash('error', t('project.bom_import.flash.invalid_entries')); - } catch (\UnexpectedValueException|SyntaxError $e) { + } catch (\UnexpectedValueException | SyntaxError $e) { $this->addFlash('error', t('project.bom_import.flash.invalid_file', ['%message%' => $e->getMessage()])); } } @@ -191,11 +217,257 @@ class ProjectController extends AbstractController ]); } + #[Route(path: '/{id}/import_bom/map_fields', name: 'project_import_bom_map_fields', requirements: ['id' => '\d+'])] + public function importBOMMapFields( + Request $request, + EntityManagerInterface $entityManager, + Project $project, + BOMImporter $BOMImporter, + ValidatorInterface $validator, + LoggerInterface $logger + ): Response { + $this->denyAccessUnlessGranted('edit', $project); + + // Get stored data from session + $file_content = $request->getSession()->get('bom_import_data'); + $clear_existing = $request->getSession()->get('bom_import_clear', false); + + + if (!$file_content) { + $this->addFlash('error', 'project.bom_import.flash.session_expired'); + return $this->redirectToRoute('project_import_bom', ['id' => $project->getID()]); + } + + // Detect fields and get suggestions + $detected_fields = $BOMImporter->detectFields($file_content); + $suggested_mapping = $BOMImporter->getSuggestedFieldMapping($detected_fields); + + // Create mapping of original field names to sanitized field names for template + $field_name_mapping = []; + foreach ($detected_fields as $field) { + $sanitized_field = preg_replace('/[^a-zA-Z0-9_-]/', '_', $field); + $field_name_mapping[$field] = $sanitized_field; + } + + // Create form for field mapping + $builder = $this->createFormBuilder(); + + // Add delimiter selection + $builder->add('delimiter', ChoiceType::class, [ + 'label' => 'project.bom_import.delimiter', + 'required' => true, + 'data' => ',', + 'choices' => [ + 'project.bom_import.delimiter.comma' => ',', + 'project.bom_import.delimiter.semicolon' => ';', + 'project.bom_import.delimiter.tab' => "\t", + ] + ]); + + // Get dynamic field mapping targets from BOMImporter + $available_targets = $BOMImporter->getAvailableFieldTargets(); + $target_fields = ['project.bom_import.field_mapping.ignore' => '']; + + foreach ($available_targets as $target_key => $target_info) { + $target_fields[$target_info['label']] = $target_key; + } + + foreach ($detected_fields as $field) { + // Sanitize field name for form use - replace invalid characters with underscores + $sanitized_field = preg_replace('/[^a-zA-Z0-9_-]/', '_', $field); + $builder->add('mapping_' . $sanitized_field, ChoiceType::class, [ + 'label' => $field, + 'required' => false, + 'choices' => $target_fields, + 'data' => $suggested_mapping[$field] ?? '', + ]); + } + + $builder->add('submit', SubmitType::class, [ + 'label' => 'project.bom_import.preview', + ]); + + $form = $builder->getForm(); + $form->handleRequest($request); + + if ($form->isSubmitted() && $form->isValid()) { + // Build field mapping array with priority support + $field_mapping = []; + $field_priorities = []; + $delimiter = $form->get('delimiter')->getData(); + + foreach ($detected_fields as $field) { + $sanitized_field = preg_replace('/[^a-zA-Z0-9_-]/', '_', $field); + $target = $form->get('mapping_' . $sanitized_field)->getData(); + if (!empty($target)) { + $field_mapping[$field] = $target; + + // Get priority from request (default to 10) + $priority = $request->request->get('priority_' . $sanitized_field, 10); + $field_priorities[$field] = (int) $priority; + } + } + + // Validate field mapping + $validation = $BOMImporter->validateFieldMapping($field_mapping, $detected_fields); + + if (!$validation['is_valid']) { + foreach ($validation['errors'] as $error) { + $this->addFlash('error', $error); + } + foreach ($validation['warnings'] as $warning) { + $this->addFlash('warning', $warning); + } + + return $this->render('projects/import_bom_map_fields.html.twig', [ + 'project' => $project, + 'form' => $form->createView(), + 'detected_fields' => $detected_fields, + 'suggested_mapping' => $suggested_mapping, + 'field_name_mapping' => $field_name_mapping, + ]); + } + + // Show warnings but continue + foreach ($validation['warnings'] as $warning) { + $this->addFlash('warning', $warning); + } + + try { + // Re-detect fields with chosen delimiter + $detected_fields = $BOMImporter->detectFields($file_content, $delimiter); + + // Clear existing BOM entries if requested + if ($clear_existing) { + $existing_count = $project->getBomEntries()->count(); + $logger->info('Clearing existing BOM entries', [ + 'existing_count' => $existing_count, + 'project_id' => $project->getID(), + ]); + $project->getBomEntries()->clear(); + $entityManager->flush(); + $logger->info('Existing BOM entries cleared'); + } else { + $existing_count = $project->getBomEntries()->count(); + $logger->info('Keeping existing BOM entries', [ + 'existing_count' => $existing_count, + 'project_id' => $project->getID(), + ]); + } + + // Validate data before importing + $validation_result = $BOMImporter->validateBOMData($file_content, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'field_priorities' => $field_priorities, + 'delimiter' => $delimiter, + ]); + + // Log validation results + $logger->info('BOM import validation completed', [ + 'total_entries' => $validation_result['total_entries'], + 'valid_entries' => $validation_result['valid_entries'], + 'invalid_entries' => $validation_result['invalid_entries'], + 'error_count' => count($validation_result['errors']), + 'warning_count' => count($validation_result['warnings']), + ]); + + // Show validation warnings to user + foreach ($validation_result['warnings'] as $warning) { + $this->addFlash('warning', $warning); + } + + // If there are validation errors, show them and stop + if (!empty($validation_result['errors'])) { + foreach ($validation_result['errors'] as $error) { + $this->addFlash('error', $error); + } + + return $this->render('projects/import_bom_map_fields.html.twig', [ + 'project' => $project, + 'form' => $form->createView(), + 'detected_fields' => $detected_fields, + 'suggested_mapping' => $suggested_mapping, + 'field_name_mapping' => $field_name_mapping, + 'validation_result' => $validation_result, + ]); + } + + // Import with field mapping and priorities (validation already passed) + $entries = $BOMImporter->stringToBOMEntries($file_content, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'field_priorities' => $field_priorities, + 'delimiter' => $delimiter, + ]); + + // Log entry details for debugging + $logger->info('BOM entries created', [ + 'total_entries' => count($entries), + ]); + + foreach ($entries as $index => $entry) { + $logger->debug("BOM entry {$index}", [ + 'name' => $entry->getName(), + 'mountnames' => $entry->getMountnames(), + 'quantity' => $entry->getQuantity(), + 'comment' => $entry->getComment(), + 'part_id' => $entry->getPart()?->getID(), + ]); + } + + // Assign entries to project + $logger->info('Adding BOM entries to project', [ + 'entries_count' => count($entries), + 'project_id' => $project->getID(), + ]); + + foreach ($entries as $index => $entry) { + $logger->debug("Adding BOM entry {$index} to project", [ + 'name' => $entry->getName(), + 'part_id' => $entry->getPart()?->getID(), + 'quantity' => $entry->getQuantity(), + ]); + $project->addBomEntry($entry); + } + + // Validate the project entries (includes collection constraints) + $errors = $validator->validateProperty($project, 'bom_entries'); + + // If no validation errors occurred, save and redirect + if (count($errors) === 0) { + $this->addFlash('success', t('project.bom_import.flash.success', ['%count%' => count($entries)])); + $entityManager->flush(); + + // Clear session data + $request->getSession()->remove('bom_import_data'); + $request->getSession()->remove('bom_import_clear'); + + return $this->redirectToRoute('project_edit', ['id' => $project->getID()]); + } + + // When we get here, there were validation errors + $this->addFlash('error', t('project.bom_import.flash.invalid_entries')); + + } catch (\UnexpectedValueException | SyntaxError $e) { + $this->addFlash('error', t('project.bom_import.flash.invalid_file', ['%message%' => $e->getMessage()])); + } + } + + return $this->render('projects/import_bom_map_fields.html.twig', [ + 'project' => $project, + 'form' => $form, + 'detected_fields' => $detected_fields, + 'suggested_mapping' => $suggested_mapping, + 'field_name_mapping' => $field_name_mapping, + ]); + } + #[Route(path: '/add_parts', name: 'project_add_parts_no_id')] #[Route(path: '/{id}/add_parts', name: 'project_add_parts', requirements: ['id' => '\d+'])] public function addPart(Request $request, EntityManagerInterface $entityManager, ?Project $project): Response { - if($project instanceof Project) { + if ($project instanceof Project) { $this->denyAccessUnlessGranted('edit', $project); } else { $this->denyAccessUnlessGranted('@projects.edit'); @@ -242,7 +514,7 @@ class ProjectController extends AbstractController $data = $form->getData(); $bom_entries = $data['bom_entries']; - foreach ($bom_entries as $bom_entry){ + foreach ($bom_entries as $bom_entry) { $target_project->addBOMEntry($bom_entry); } diff --git a/src/Services/ImportExportSystem/BOMImporter.php b/src/Services/ImportExportSystem/BOMImporter.php index d4876445..862fa463 100644 --- a/src/Services/ImportExportSystem/BOMImporter.php +++ b/src/Services/ImportExportSystem/BOMImporter.php @@ -22,10 +22,13 @@ declare(strict_types=1); */ namespace App\Services\ImportExportSystem; +use App\Entity\Parts\Part; use App\Entity\ProjectSystem\Project; use App\Entity\ProjectSystem\ProjectBOMEntry; +use Doctrine\ORM\EntityManagerInterface; use InvalidArgumentException; use League\Csv\Reader; +use Psr\Log\LoggerInterface; use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\OptionsResolver\OptionsResolver; @@ -44,14 +47,25 @@ class BOMImporter 5 => 'Supplier and ref', ]; - public function __construct() - { + public function __construct( + private readonly EntityManagerInterface $entityManager, + private readonly LoggerInterface $logger, + private readonly BOMValidationService $validationService + ) { } protected function configureOptions(OptionsResolver $resolver): OptionsResolver { $resolver->setRequired('type'); - $resolver->setAllowedValues('type', ['kicad_pcbnew']); + $resolver->setAllowedValues('type', ['kicad_pcbnew', 'kicad_schematic']); + + // For flexible schematic import with field mapping + $resolver->setDefined(['field_mapping', 'field_priorities', 'delimiter']); + $resolver->setDefault('delimiter', ','); + $resolver->setDefault('field_priorities', []); + $resolver->setAllowedTypes('field_mapping', 'array'); + $resolver->setAllowedTypes('field_priorities', 'array'); + $resolver->setAllowedTypes('delimiter', 'string'); return $resolver; } @@ -82,6 +96,23 @@ class BOMImporter return $this->stringToBOMEntries($file->getContent(), $options); } + /** + * Validate BOM data before importing + * @return array Validation result with errors, warnings, and info + */ + public function validateBOMData(string $data, array $options): array + { + $resolver = new OptionsResolver(); + $resolver = $this->configureOptions($resolver); + $options = $resolver->resolve($options); + + return match ($options['type']) { + 'kicad_pcbnew' => $this->validateKiCADPCB($data), + 'kicad_schematic' => $this->validateKiCADSchematicData($data, $options), + default => throw new InvalidArgumentException('Invalid import type!'), + }; + } + /** * Import string data into an array of BOM entries, which are not yet assigned to a project. * @param string $data The data to import @@ -95,12 +126,13 @@ class BOMImporter $options = $resolver->resolve($options); return match ($options['type']) { - 'kicad_pcbnew' => $this->parseKiCADPCB($data, $options), + 'kicad_pcbnew' => $this->parseKiCADPCB($data), + 'kicad_schematic' => $this->parseKiCADSchematic($data, $options), default => throw new InvalidArgumentException('Invalid import type!'), }; } - private function parseKiCADPCB(string $data, array $options = []): array + private function parseKiCADPCB(string $data): array { $csv = Reader::createFromString($data); $csv->setDelimiter(';'); @@ -113,17 +145,17 @@ class BOMImporter $entry = $this->normalizeColumnNames($entry); //Ensure that the entry has all required fields - if (!isset ($entry['Designator'])) { - throw new \UnexpectedValueException('Designator missing at line '.($offset + 1).'!'); + if (!isset($entry['Designator'])) { + throw new \UnexpectedValueException('Designator missing at line ' . ($offset + 1) . '!'); } - if (!isset ($entry['Package'])) { - throw new \UnexpectedValueException('Package missing at line '.($offset + 1).'!'); + if (!isset($entry['Package'])) { + throw new \UnexpectedValueException('Package missing at line ' . ($offset + 1) . '!'); } - if (!isset ($entry['Designation'])) { - throw new \UnexpectedValueException('Designation missing at line '.($offset + 1).'!'); + if (!isset($entry['Designation'])) { + throw new \UnexpectedValueException('Designation missing at line ' . ($offset + 1) . '!'); } - if (!isset ($entry['Quantity'])) { - throw new \UnexpectedValueException('Quantity missing at line '.($offset + 1).'!'); + if (!isset($entry['Quantity'])) { + throw new \UnexpectedValueException('Quantity missing at line ' . ($offset + 1) . '!'); } $bom_entry = new ProjectBOMEntry(); @@ -138,6 +170,63 @@ class BOMImporter return $bom_entries; } + /** + * Validate KiCad PCB data + */ + private function validateKiCADPCB(string $data): array + { + $csv = Reader::createFromString($data); + $csv->setDelimiter(';'); + $csv->setHeaderOffset(0); + + $mapped_entries = []; + + foreach ($csv->getRecords() as $offset => $entry) { + // Translate the german field names to english + $entry = $this->normalizeColumnNames($entry); + $mapped_entries[] = $entry; + } + + return $this->validationService->validateBOMEntries($mapped_entries); + } + + /** + * Validate KiCad schematic data + */ + private function validateKiCADSchematicData(string $data, array $options): array + { + $delimiter = $options['delimiter'] ?? ','; + $field_mapping = $options['field_mapping'] ?? []; + $field_priorities = $options['field_priorities'] ?? []; + + // Handle potential BOM (Byte Order Mark) at the beginning + $data = preg_replace('/^\xEF\xBB\xBF/', '', $data); + + $csv = Reader::createFromString($data); + $csv->setDelimiter($delimiter); + $csv->setHeaderOffset(0); + + // Handle quoted fields properly + $csv->setEscape('\\'); + $csv->setEnclosure('"'); + + $mapped_entries = []; + + foreach ($csv->getRecords() as $offset => $entry) { + // Apply field mapping to translate column names + $mapped_entry = $this->applyFieldMapping($entry, $field_mapping, $field_priorities); + + // Extract footprint package name if it contains library prefix + if (isset($mapped_entry['Package']) && str_contains($mapped_entry['Package'], ':')) { + $mapped_entry['Package'] = explode(':', $mapped_entry['Package'], 2)[1]; + } + + $mapped_entries[] = $mapped_entry; + } + + return $this->validationService->validateBOMEntries($mapped_entries, $options); + } + /** * This function uses the order of the fields in the CSV files to make them locale independent. * @param array $entry @@ -160,4 +249,482 @@ class BOMImporter return $out; } + + /** + * Parse KiCad schematic BOM with flexible field mapping + */ + private function parseKiCADSchematic(string $data, array $options = []): array + { + $delimiter = $options['delimiter'] ?? ','; + $field_mapping = $options['field_mapping'] ?? []; + $field_priorities = $options['field_priorities'] ?? []; + + // Handle potential BOM (Byte Order Mark) at the beginning + $data = preg_replace('/^\xEF\xBB\xBF/', '', $data); + + $csv = Reader::createFromString($data); + $csv->setDelimiter($delimiter); + $csv->setHeaderOffset(0); + + // Handle quoted fields properly + $csv->setEscape('\\'); + $csv->setEnclosure('"'); + + $bom_entries = []; + $entries_by_key = []; // Track entries by name+part combination + $mapped_entries = []; // Collect all mapped entries for validation + + foreach ($csv->getRecords() as $offset => $entry) { + // Apply field mapping to translate column names + $mapped_entry = $this->applyFieldMapping($entry, $field_mapping, $field_priorities); + + // Extract footprint package name if it contains library prefix + if (isset($mapped_entry['Package']) && str_contains($mapped_entry['Package'], ':')) { + $mapped_entry['Package'] = explode(':', $mapped_entry['Package'], 2)[1]; + } + + $mapped_entries[] = $mapped_entry; + } + + // Validate all entries before processing + $validation_result = $this->validationService->validateBOMEntries($mapped_entries, $options); + + // Log validation results + $this->logger->info('BOM import validation completed', [ + 'total_entries' => $validation_result['total_entries'], + 'valid_entries' => $validation_result['valid_entries'], + 'invalid_entries' => $validation_result['invalid_entries'], + 'error_count' => count($validation_result['errors']), + 'warning_count' => count($validation_result['warnings']), + ]); + + // If there are validation errors, throw an exception with detailed messages + if (!empty($validation_result['errors'])) { + $error_message = $this->validationService->getErrorMessage($validation_result); + throw new \UnexpectedValueException("BOM import validation failed:\n" . $error_message); + } + + // Process validated entries + foreach ($mapped_entries as $offset => $mapped_entry) { + + // Set name - prefer MPN, fall back to Value, then default format + $mpn = trim($mapped_entry['MPN'] ?? ''); + $designation = trim($mapped_entry['Designation'] ?? ''); + $value = trim($mapped_entry['Value'] ?? ''); + + // Use the first non-empty value, or 'Unknown Component' if all are empty + $name = ''; + if (!empty($mpn)) { + $name = $mpn; + } elseif (!empty($designation)) { + $name = $designation; + } elseif (!empty($value)) { + $name = $value; + } else { + $name = 'Unknown Component'; + } + + if (isset($mapped_entry['Package']) && !empty(trim($mapped_entry['Package']))) { + $name .= ' (' . trim($mapped_entry['Package']) . ')'; + } + + // Set mountnames and quantity + // The Designator field contains comma-separated mount names for all instances + $designator = trim($mapped_entry['Designator']); + $quantity = (float) $mapped_entry['Quantity']; + + // Get mountnames array (validation already ensured they match quantity) + $mountnames_array = array_map('trim', explode(',', $designator)); + + // Try to link existing Part-DB part if ID is provided + $part = null; + if (isset($mapped_entry['Part-DB ID']) && !empty($mapped_entry['Part-DB ID'])) { + $partDbId = (int) $mapped_entry['Part-DB ID']; + $existingPart = $this->entityManager->getRepository(Part::class)->find($partDbId); + + if ($existingPart) { + $part = $existingPart; + // Update name with actual part name + $name = $existingPart->getName(); + } + } + + // Create unique key for this entry (name + part ID) + $entry_key = $name . '|' . ($part ? $part->getID() : 'null'); + + // Check if we already have an entry with the same name and part + if (isset($entries_by_key[$entry_key])) { + // Merge with existing entry + $existing_entry = $entries_by_key[$entry_key]; + + // Combine mountnames + $existing_mountnames = $existing_entry->getMountnames(); + $combined_mountnames = $existing_mountnames . ',' . $designator; + $existing_entry->setMountnames($combined_mountnames); + + // Add quantities + $existing_quantity = $existing_entry->getQuantity(); + $existing_entry->setQuantity($existing_quantity + $quantity); + + $this->logger->info('Merged duplicate BOM entry', [ + 'name' => $name, + 'part_id' => $part ? $part->getID() : null, + 'original_quantity' => $existing_quantity, + 'added_quantity' => $quantity, + 'new_quantity' => $existing_quantity + $quantity, + 'original_mountnames' => $existing_mountnames, + 'added_mountnames' => $designator, + ]); + + continue; // Skip creating new entry + } + + // Create new BOM entry + $bom_entry = new ProjectBOMEntry(); + $bom_entry->setName($name); + $bom_entry->setMountnames($designator); + $bom_entry->setQuantity($quantity); + + if ($part) { + $bom_entry->setPart($part); + } + + // Set comment with additional info + $comment_parts = []; + if (isset($mapped_entry['Value']) && $mapped_entry['Value'] !== ($mapped_entry['MPN'] ?? '')) { + $comment_parts[] = 'Value: ' . $mapped_entry['Value']; + } + if (isset($mapped_entry['MPN'])) { + $comment_parts[] = 'MPN: ' . $mapped_entry['MPN']; + } + if (isset($mapped_entry['Manufacturer'])) { + $comment_parts[] = 'Manf: ' . $mapped_entry['Manufacturer']; + } + if (isset($mapped_entry['LCSC'])) { + $comment_parts[] = 'LCSC: ' . $mapped_entry['LCSC']; + } + if (isset($mapped_entry['Supplier and ref'])) { + $comment_parts[] = $mapped_entry['Supplier and ref']; + } + + if ($part) { + $comment_parts[] = "Part-DB ID: " . $part->getID(); + } elseif (isset($mapped_entry['Part-DB ID']) && !empty($mapped_entry['Part-DB ID'])) { + $comment_parts[] = "Part-DB ID: " . $mapped_entry['Part-DB ID'] . " (NOT FOUND)"; + } + + $bom_entry->setComment(implode(', ', $comment_parts)); + + $bom_entries[] = $bom_entry; + $entries_by_key[$entry_key] = $bom_entry; + } + + return $bom_entries; + } + + /** + * Get all available field mapping targets with descriptions + */ + public function getAvailableFieldTargets(): array + { + $targets = [ + 'Designator' => [ + 'label' => 'Designator', + 'description' => 'Component reference designators (e.g., R1, C2, U3)', + 'required' => true, + 'multiple' => false, + ], + 'Quantity' => [ + 'label' => 'Quantity', + 'description' => 'Number of components', + 'required' => true, + 'multiple' => false, + ], + 'Designation' => [ + 'label' => 'Designation', + 'description' => 'Component designation/part number', + 'required' => false, + 'multiple' => true, + ], + 'Value' => [ + 'label' => 'Value', + 'description' => 'Component value (e.g., 10k, 100nF)', + 'required' => false, + 'multiple' => true, + ], + 'Package' => [ + 'label' => 'Package', + 'description' => 'Component package/footprint', + 'required' => false, + 'multiple' => true, + ], + 'MPN' => [ + 'label' => 'MPN', + 'description' => 'Manufacturer Part Number', + 'required' => false, + 'multiple' => true, + ], + 'Manufacturer' => [ + 'label' => 'Manufacturer', + 'description' => 'Component manufacturer name', + 'required' => false, + 'multiple' => true, + ], + 'Part-DB ID' => [ + 'label' => 'Part-DB ID', + 'description' => 'Existing Part-DB part ID for linking', + 'required' => false, + 'multiple' => false, + ], + 'Comment' => [ + 'label' => 'Comment', + 'description' => 'Additional component information', + 'required' => false, + 'multiple' => true, + ], + ]; + + // Add dynamic supplier fields based on available suppliers in the database + $suppliers = $this->entityManager->getRepository(\App\Entity\Parts\Supplier::class)->findAll(); + foreach ($suppliers as $supplier) { + $supplierName = $supplier->getName(); + $targets[$supplierName . ' SPN'] = [ + 'label' => $supplierName . ' SPN', + 'description' => "Supplier part number for {$supplierName}", + 'required' => false, + 'multiple' => true, + 'supplier_id' => $supplier->getID(), + ]; + } + + return $targets; + } + + /** + * Get suggested field mappings based on common field names + */ + public function getSuggestedFieldMapping(array $detected_fields): array + { + $suggestions = []; + + $field_patterns = [ + 'Part-DB ID' => ['part-db id', 'partdb_id', 'part_db_id', 'db_id', 'partdb'], + 'Designator' => ['reference', 'ref', 'designator', 'component', 'comp'], + 'Quantity' => ['qty', 'quantity', 'count', 'number', 'amount'], + 'Value' => ['value', 'val', 'component_value'], + 'Designation' => ['designation', 'part_number', 'partnumber', 'part'], + 'Package' => ['footprint', 'package', 'housing', 'fp'], + 'MPN' => ['mpn', 'part_number', 'partnumber', 'manf#', 'mfr_part_number', 'manufacturer_part'], + 'Manufacturer' => ['manufacturer', 'manf', 'mfr', 'brand', 'vendor'], + 'Comment' => ['comment', 'comments', 'note', 'notes', 'description'], + ]; + + // Add supplier-specific patterns + $suppliers = $this->entityManager->getRepository(\App\Entity\Parts\Supplier::class)->findAll(); + foreach ($suppliers as $supplier) { + $supplierName = $supplier->getName(); + $supplierLower = strtolower($supplierName); + + // Create patterns for each supplier + $field_patterns[$supplierName . ' SPN'] = [ + $supplierLower, + $supplierLower . '#', + $supplierLower . '_part', + $supplierLower . '_number', + $supplierLower . 'pn', + $supplierLower . '_spn', + $supplierLower . ' spn', + // Common abbreviations + $supplierLower === 'mouser' ? 'mouser' : null, + $supplierLower === 'digikey' ? 'dk' : null, + $supplierLower === 'farnell' ? 'farnell' : null, + $supplierLower === 'rs' ? 'rs' : null, + $supplierLower === 'lcsc' ? 'lcsc' : null, + ]; + + // Remove null values + $field_patterns[$supplierName . ' SPN'] = array_filter($field_patterns[$supplierName . ' SPN'], fn($value) => $value !== null); + } + + foreach ($detected_fields as $field) { + $field_lower = strtolower(trim($field)); + + foreach ($field_patterns as $target => $patterns) { + foreach ($patterns as $pattern) { + if (str_contains($field_lower, $pattern)) { + $suggestions[$field] = $target; + break 2; // Break both loops + } + } + } + } + + return $suggestions; + } + + /** + * Validate field mapping configuration + */ + public function validateFieldMapping(array $field_mapping, array $detected_fields): array + { + $errors = []; + $warnings = []; + $available_targets = $this->getAvailableFieldTargets(); + + // Check for required fields + $mapped_targets = array_values($field_mapping); + $required_fields = ['Designator', 'Quantity']; + + foreach ($required_fields as $required) { + if (!in_array($required, $mapped_targets, true)) { + $errors[] = "Required field '{$required}' is not mapped from any CSV column."; + } + } + + // Check for invalid target fields + foreach ($field_mapping as $csv_field => $target) { + if (!empty($target) && !isset($available_targets[$target])) { + $errors[] = "Invalid target field '{$target}' for CSV field '{$csv_field}'."; + } + } + + // Check for unmapped fields (warnings) + $unmapped_fields = array_diff($detected_fields, array_keys($field_mapping)); + if (!empty($unmapped_fields)) { + $warnings[] = "The following CSV fields are not mapped: " . implode(', ', $unmapped_fields); + } + + return [ + 'errors' => $errors, + 'warnings' => $warnings, + 'is_valid' => empty($errors), + ]; + } + + /** + * Apply field mapping with support for multiple fields and priority + */ + private function applyFieldMapping(array $entry, array $field_mapping, array $field_priorities = []): array + { + $mapped = []; + $field_groups = []; + + // Group fields by target with priority information + foreach ($field_mapping as $csv_field => $target) { + if (!empty($target)) { + if (!isset($field_groups[$target])) { + $field_groups[$target] = []; + } + $priority = $field_priorities[$csv_field] ?? 10; + $field_groups[$target][] = [ + 'field' => $csv_field, + 'priority' => $priority, + 'value' => $entry[$csv_field] ?? '' + ]; + } + } + + // Process each target field + foreach ($field_groups as $target => $field_data) { + // Sort by priority (lower number = higher priority) + usort($field_data, function ($a, $b) { + return $a['priority'] <=> $b['priority']; + }); + + $values = []; + $non_empty_values = []; + + // Collect all non-empty values for this target + foreach ($field_data as $data) { + $value = trim($data['value']); + if (!empty($value)) { + $non_empty_values[] = $value; + } + $values[] = $value; + } + + // Use the first non-empty value (highest priority) + if (!empty($non_empty_values)) { + $mapped[$target] = $non_empty_values[0]; + + // If multiple non-empty values exist, add alternatives to comment + if (count($non_empty_values) > 1) { + $mapped[$target . '_alternatives'] = array_slice($non_empty_values, 1); + } + } + } + + return $mapped; + } + + /** + * Detect available fields in CSV data for field mapping UI + */ + public function detectFields(string $data, ?string $delimiter = null): array + { + if ($delimiter === null) { + // Detect delimiter by counting occurrences in the first row (header) + $delimiters = [',', ';', "\t"]; + $lines = explode("\n", $data, 2); + $header_line = $lines[0] ?? ''; + $delimiter_counts = []; + foreach ($delimiters as $delim) { + $delimiter_counts[$delim] = substr_count($header_line, $delim); + } + // Choose the delimiter with the highest count, default to comma if all are zero + $max_count = max($delimiter_counts); + $delimiter = array_search($max_count, $delimiter_counts, true); + if ($max_count === 0 || $delimiter === false) { + $delimiter = ','; + } + } + // Handle potential BOM (Byte Order Mark) at the beginning + $data = preg_replace('/^\xEF\xBB\xBF/', '', $data); + + // Get first line only for header detection + $lines = explode("\n", $data); + $header_line = trim($lines[0] ?? ''); + + + // Simple manual parsing for header detection + // This handles quoted CSV fields better than the library for detection + $fields = []; + $current_field = ''; + $in_quotes = false; + $quote_char = '"'; + + for ($i = 0; $i < strlen($header_line); $i++) { + $char = $header_line[$i]; + + if ($char === $quote_char && !$in_quotes) { + $in_quotes = true; + } elseif ($char === $quote_char && $in_quotes) { + // Check for escaped quote (double quote) + if ($i + 1 < strlen($header_line) && $header_line[$i + 1] === $quote_char) { + $current_field .= $quote_char; + $i++; // Skip next quote + } else { + $in_quotes = false; + } + } elseif ($char === $delimiter && !$in_quotes) { + $fields[] = trim($current_field); + $current_field = ''; + } else { + $current_field .= $char; + } + } + + // Add the last field + if ($current_field !== '') { + $fields[] = trim($current_field); + } + + // Clean up headers - remove quotes and trim whitespace + $headers = array_map(function ($header) { + return trim($header, '"\''); + }, $fields); + + + return array_values($headers); + } } diff --git a/src/Services/ImportExportSystem/BOMValidationService.php b/src/Services/ImportExportSystem/BOMValidationService.php new file mode 100644 index 00000000..74f81fe3 --- /dev/null +++ b/src/Services/ImportExportSystem/BOMValidationService.php @@ -0,0 +1,476 @@ +. + */ +namespace App\Services\ImportExportSystem; + +use App\Entity\Parts\Part; +use App\Entity\ProjectSystem\ProjectBOMEntry; +use Doctrine\ORM\EntityManagerInterface; +use Symfony\Contracts\Translation\TranslatorInterface; + +/** + * Service for validating BOM import data with comprehensive validation rules + * and user-friendly error messages. + */ +class BOMValidationService +{ + public function __construct( + private readonly EntityManagerInterface $entityManager, + private readonly TranslatorInterface $translator + ) { + } + + /** + * Validation result structure + */ + public static function createValidationResult(): array + { + return [ + 'errors' => [], + 'warnings' => [], + 'info' => [], + 'is_valid' => true, + 'total_entries' => 0, + 'valid_entries' => 0, + 'invalid_entries' => 0, + ]; + } + + /** + * Validate a single BOM entry with comprehensive checks + */ + public function validateBOMEntry(array $mapped_entry, int $line_number, array $options = []): array + { + $result = [ + 'line_number' => $line_number, + 'errors' => [], + 'warnings' => [], + 'info' => [], + 'is_valid' => true, + ]; + + // Run all validation rules + $this->validateRequiredFields($mapped_entry, $result); + $this->validateDesignatorFormat($mapped_entry, $result); + $this->validateQuantityFormat($mapped_entry, $result); + $this->validateDesignatorQuantityMatch($mapped_entry, $result); + $this->validatePartDBLink($mapped_entry, $result); + $this->validateComponentName($mapped_entry, $result); + $this->validatePackageFormat($mapped_entry, $result); + $this->validateNumericFields($mapped_entry, $result); + + $result['is_valid'] = empty($result['errors']); + + return $result; + } + + /** + * Validate multiple BOM entries and provide summary + */ + public function validateBOMEntries(array $mapped_entries, array $options = []): array + { + $result = self::createValidationResult(); + $result['total_entries'] = count($mapped_entries); + + $line_results = []; + $all_errors = []; + $all_warnings = []; + $all_info = []; + + foreach ($mapped_entries as $index => $entry) { + $line_number = $index + 1; + $line_result = $this->validateBOMEntry($entry, $line_number, $options); + + $line_results[] = $line_result; + + if ($line_result['is_valid']) { + $result['valid_entries']++; + } else { + $result['invalid_entries']++; + } + + // Collect all messages + $all_errors = array_merge($all_errors, $line_result['errors']); + $all_warnings = array_merge($all_warnings, $line_result['warnings']); + $all_info = array_merge($all_info, $line_result['info']); + } + + // Add summary messages + $this->addSummaryMessages($result, $all_errors, $all_warnings, $all_info); + + $result['errors'] = $all_errors; + $result['warnings'] = $all_warnings; + $result['info'] = $all_info; + $result['line_results'] = $line_results; + $result['is_valid'] = empty($all_errors); + + return $result; + } + + /** + * Validate required fields are present + */ + private function validateRequiredFields(array $entry, array &$result): void + { + $required_fields = ['Designator', 'Quantity']; + + foreach ($required_fields as $field) { + if (!isset($entry[$field]) || trim($entry[$field]) === '') { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.required_field_missing', [ + '%line%' => $result['line_number'], + '%field%' => $field + ]); + } + } + } + + /** + * Validate designator format and content + */ + private function validateDesignatorFormat(array $entry, array &$result): void + { + if (!isset($entry['Designator']) || trim($entry['Designator']) === '') { + return; // Already handled by required fields validation + } + + $designator = trim($entry['Designator']); + $mountnames = array_map('trim', explode(',', $designator)); + + // Remove empty entries + $mountnames = array_filter($mountnames, fn($name) => !empty($name)); + + if (empty($mountnames)) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.no_valid_designators', [ + '%line%' => $result['line_number'] + ]); + return; + } + + // Validate each mountname format (allow 1-2 uppercase letters, followed by 1+ digits) + $invalid_mountnames = []; + foreach ($mountnames as $mountname) { + if (!preg_match('/^[A-Z]{1,2}[0-9]+$/', $mountname)) { + $invalid_mountnames[] = $mountname; + } + } + + if (!empty($invalid_mountnames)) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.unusual_designator_format', [ + '%line%' => $result['line_number'], + '%designators%' => implode(', ', $invalid_mountnames) + ]); + } + + // Check for duplicate mountnames within the same line + $duplicates = array_diff_assoc($mountnames, array_unique($mountnames)); + if (!empty($duplicates)) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.duplicate_designators', [ + '%line%' => $result['line_number'], + '%designators%' => implode(', ', array_unique($duplicates)) + ]); + } + } + + /** + * Validate quantity format and value + */ + private function validateQuantityFormat(array $entry, array &$result): void + { + if (!isset($entry['Quantity']) || trim($entry['Quantity']) === '') { + return; // Already handled by required fields validation + } + + $quantity_str = trim($entry['Quantity']); + + // Check if it's a valid number + if (!is_numeric($quantity_str)) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.invalid_quantity', [ + '%line%' => $result['line_number'], + '%quantity%' => $quantity_str + ]); + return; + } + + $quantity = (float) $quantity_str; + + // Check for reasonable quantity values + if ($quantity <= 0) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.quantity_zero_or_negative', [ + '%line%' => $result['line_number'], + '%quantity%' => $quantity_str + ]); + } elseif ($quantity > 10000) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.quantity_unusually_high', [ + '%line%' => $result['line_number'], + '%quantity%' => $quantity_str + ]); + } + + // Check if quantity is a whole number when it should be + if (isset($entry['Designator'])) { + $designator = trim($entry['Designator']); + $mountnames = array_map('trim', explode(',', $designator)); + $mountnames = array_filter($mountnames, fn($name) => !empty($name)); + + if (count($mountnames) > 0 && $quantity != (int) $quantity) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.quantity_not_whole_number', [ + '%line%' => $result['line_number'], + '%quantity%' => $quantity_str, + '%count%' => count($mountnames) + ]); + } + } + } + + /** + * Validate that designator count matches quantity + */ + private function validateDesignatorQuantityMatch(array $entry, array &$result): void + { + if (!isset($entry['Designator']) || !isset($entry['Quantity'])) { + return; // Already handled by required fields validation + } + + $designator = trim($entry['Designator']); + $quantity_str = trim($entry['Quantity']); + + if (!is_numeric($quantity_str)) { + return; // Already handled by quantity validation + } + + $mountnames = array_map('trim', explode(',', $designator)); + $mountnames = array_filter($mountnames, fn($name) => !empty($name)); + $mountnames_count = count($mountnames); + $quantity = (float) $quantity_str; + + if ($mountnames_count !== (int) $quantity) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.quantity_designator_mismatch', [ + '%line%' => $result['line_number'], + '%quantity%' => $quantity_str, + '%count%' => $mountnames_count, + '%designators%' => $designator + ]); + } + } + + /** + * Validate Part-DB ID link + */ + private function validatePartDBLink(array $entry, array &$result): void + { + if (!isset($entry['Part-DB ID']) || trim($entry['Part-DB ID']) === '') { + return; + } + + $part_db_id = trim($entry['Part-DB ID']); + + if (!is_numeric($part_db_id)) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.invalid_partdb_id', [ + '%line%' => $result['line_number'], + '%id%' => $part_db_id + ]); + return; + } + + $part_id = (int) $part_db_id; + + if ($part_id <= 0) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.partdb_id_zero_or_negative', [ + '%line%' => $result['line_number'], + '%id%' => $part_id + ]); + return; + } + + // Check if part exists in database + $existing_part = $this->entityManager->getRepository(Part::class)->find($part_id); + if (!$existing_part) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.partdb_id_not_found', [ + '%line%' => $result['line_number'], + '%id%' => $part_id + ]); + } else { + $result['info'][] = $this->translator->trans('project.bom_import.validation.info.partdb_link_success', [ + '%line%' => $result['line_number'], + '%name%' => $existing_part->getName(), + '%id%' => $part_id + ]); + } + } + + /** + * Validate component name/designation + */ + private function validateComponentName(array $entry, array &$result): void + { + $name_fields = ['MPN', 'Designation', 'Value']; + $has_name = false; + + foreach ($name_fields as $field) { + if (isset($entry[$field]) && trim($entry[$field]) !== '') { + $has_name = true; + break; + } + } + + if (!$has_name) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.no_component_name', [ + '%line%' => $result['line_number'] + ]); + } + } + + /** + * Validate package format + */ + private function validatePackageFormat(array $entry, array &$result): void + { + if (!isset($entry['Package']) || trim($entry['Package']) === '') { + return; + } + + $package = trim($entry['Package']); + + // Check for common package format issues + if (strlen($package) > 100) { + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.package_name_too_long', [ + '%line%' => $result['line_number'], + '%package%' => $package + ]); + } + + // Check for library prefixes (KiCad format) + if (str_contains($package, ':')) { + $result['info'][] = $this->translator->trans('project.bom_import.validation.info.library_prefix_detected', [ + '%line%' => $result['line_number'], + '%package%' => $package + ]); + } + } + + /** + * Validate numeric fields + */ + private function validateNumericFields(array $entry, array &$result): void + { + $numeric_fields = ['Quantity', 'Part-DB ID']; + + foreach ($numeric_fields as $field) { + if (isset($entry[$field]) && trim($entry[$field]) !== '') { + $value = trim($entry[$field]); + if (!is_numeric($value)) { + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.non_numeric_field', [ + '%line%' => $result['line_number'], + '%field%' => $field, + '%value%' => $value + ]); + } + } + } + } + + /** + * Add summary messages to validation result + */ + private function addSummaryMessages(array &$result, array $errors, array $warnings, array $info): void + { + $total_entries = $result['total_entries']; + $valid_entries = $result['valid_entries']; + $invalid_entries = $result['invalid_entries']; + + // Add summary info + if ($total_entries > 0) { + $result['info'][] = $this->translator->trans('project.bom_import.validation.info.import_summary', [ + '%total%' => $total_entries, + '%valid%' => $valid_entries, + '%invalid%' => $invalid_entries + ]); + } + + // Add error summary + if (!empty($errors)) { + $error_count = count($errors); + $result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.summary', [ + '%count%' => $error_count + ]); + } + + // Add warning summary + if (!empty($warnings)) { + $warning_count = count($warnings); + $result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.summary', [ + '%count%' => $warning_count + ]); + } + + // Add success message if all entries are valid + if ($total_entries > 0 && $invalid_entries === 0) { + $result['info'][] = $this->translator->trans('project.bom_import.validation.info.all_valid'); + } + } + + /** + * Get user-friendly error message for a validation result + */ + public function getErrorMessage(array $validation_result): string + { + if ($validation_result['is_valid']) { + return ''; + } + + $messages = []; + + if (!empty($validation_result['errors'])) { + $messages[] = 'Errors:'; + foreach ($validation_result['errors'] as $error) { + $messages[] = '• ' . $error; + } + } + + if (!empty($validation_result['warnings'])) { + $messages[] = 'Warnings:'; + foreach ($validation_result['warnings'] as $warning) { + $messages[] = '• ' . $warning; + } + } + + return implode("\n", $messages); + } + + /** + * Get validation statistics + */ + public function getValidationStats(array $validation_result): array + { + return [ + 'total_entries' => $validation_result['total_entries'] ?? 0, + 'valid_entries' => $validation_result['valid_entries'] ?? 0, + 'invalid_entries' => $validation_result['invalid_entries'] ?? 0, + 'error_count' => count($validation_result['errors'] ?? []), + 'warning_count' => count($validation_result['warnings'] ?? []), + 'info_count' => count($validation_result['info'] ?? []), + 'success_rate' => $validation_result['total_entries'] > 0 + ? round(($validation_result['valid_entries'] / $validation_result['total_entries']) * 100, 1) + : 0, + ]; + } +} \ No newline at end of file diff --git a/templates/projects/_bom_validation_results.html.twig b/templates/projects/_bom_validation_results.html.twig new file mode 100644 index 00000000..68f1b827 --- /dev/null +++ b/templates/projects/_bom_validation_results.html.twig @@ -0,0 +1,186 @@ +{# BOM Validation Results Component #} +{# + Usage: + {% include 'projects/_bom_validation_results.html.twig' with { + validation_result: validation_result, + show_summary: true, + show_details: true + } %} +#} + +{% if validation_result is defined and validation_result is not empty %} + {% set stats = validation_result %} + + {# Validation Summary #} + {% if show_summary is defined and show_summary %} +
+
+
+
+
+ + {% trans %}project.bom_import.validation.summary{% endtrans %} +
+
+
+
+
+
+
{{ stats.total_entries }}
+ {% trans %}project.bom_import.validation.total_entries{% endtrans %} +
+
+
+
+
{{ stats.valid_entries }}
+ {% trans %}project.bom_import.validation.valid_entries{% endtrans %} +
+
+
+
+
{{ stats.invalid_entries }}
+ {% trans %}project.bom_import.validation.invalid_entries{% endtrans %} +
+
+
+
+
+ {% if stats.total_entries > 0 %} + {{ ((stats.valid_entries / stats.total_entries) * 100) | round(1) }}% + {% else %} + 0% + {% endif %} +
+ {% trans %}project.bom_import.validation.success_rate{% endtrans %} +
+
+
+
+
+
+
+ {% endif %} + + {# Validation Messages #} + {% if validation_result.errors is defined and validation_result.errors is not empty %} +
+

{% trans %}project.bom_import.validation.errors.title{% endtrans %}

+

{% trans %}project.bom_import.validation.errors.description{% endtrans %}

+
    + {% for error in validation_result.errors %} +
  • {{ error|raw }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if validation_result.warnings is defined and validation_result.warnings is not empty %} +
+

{% trans %}project.bom_import.validation.warnings.title{% endtrans %}

+

{% trans %}project.bom_import.validation.warnings.description{% endtrans %}

+
    + {% for warning in validation_result.warnings %} +
  • {{ warning|raw }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if validation_result.info is defined and validation_result.info is not empty %} +
+

{% trans %}project.bom_import.validation.info.title{% endtrans %}

+
    + {% for info in validation_result.info %} +
  • {{ info|raw }}
    • + {% endfor %} +
+
+ {% endif %} + + {# Detailed Line-by-Line Results #} + {% if show_details is defined and show_details and validation_result.line_results is defined %} +
+
+
+ + {% trans %}project.bom_import.validation.details.title{% endtrans %} +
+
+
+
+ + + + + + + + + + {% for line_result in validation_result.line_results %} + + + + + + {% endfor %} + +
{% trans %}project.bom_import.validation.details.line{% endtrans %}{% trans %}project.bom_import.validation.details.status{% endtrans %}{% trans %}project.bom_import.validation.details.messages{% endtrans %}
+ {{ line_result.line_number }} + + {% if line_result.is_valid %} + + + {% trans %}project.bom_import.validation.details.valid{% endtrans %} + + {% else %} + + + {% trans %}project.bom_import.validation.details.invalid{% endtrans %} + + {% endif %} + + {% if line_result.errors is not empty %} +
+ {% for error in line_result.errors %} +
{{ error|raw }}
+ {% endfor %} +
+ {% endif %} + {% if line_result.warnings is not empty %} +
+ {% for warning in line_result.warnings %} +
{{ warning|raw }}
+ {% endfor %} +
+ {% endif %} + {% if line_result.info is not empty %} +
+ {% for info in line_result.info %} +
{{ info|raw }}
+ {% endfor %} +
+ {% endif %} +
+
+
+
+ {% endif %} + + {# Action Buttons #} + {% if validation_result.is_valid is defined %} +
+ {% if validation_result.is_valid %} +
+ + {% trans %}project.bom_import.validation.all_valid{% endtrans %} +
+ {% else %} +
+ + {% trans %}project.bom_import.validation.fix_errors{% endtrans %} +
+ {% endif %} +
+ {% endif %} +{% endif %} \ No newline at end of file diff --git a/templates/projects/import_bom_map_fields.html.twig b/templates/projects/import_bom_map_fields.html.twig new file mode 100644 index 00000000..ba10c9c5 --- /dev/null +++ b/templates/projects/import_bom_map_fields.html.twig @@ -0,0 +1,204 @@ +{% extends "main_card.html.twig" %} + +{% block title %}{% trans %}project.bom_import.map_fields{% endtrans %}{% endblock %} + +{% block card_title %} + + {% trans %}project.bom_import.map_fields{% endtrans %}{% if project %}: {{ project.name }}{% endif %} +{% endblock %} + +{% block card_content %} + {% if validation_result is defined %} + {% include 'projects/_bom_validation_results.html.twig' with { + validation_result: validation_result, + show_summary: true, + show_details: false + } %} + {% endif %} + +
+
+
+ + {% trans %}project.bom_import.map_fields.help{% endtrans %} +
+
+ + {% trans %}project.bom_import.field_mapping.priority_note{% endtrans %} +
+
+
+ + {{ form_start(form) }} + +
+
+ {{ form_row(form.delimiter) }} +
+
+ +
+
+
+ + {% trans %}project.bom_import.field_mapping.title{% endtrans %} +
+
+
+
+ + + + + + + + + + + {% for field in detected_fields %} + + + + + + + {% endfor %} + +
{% trans %}project.bom_import.field_mapping.csv_field{% endtrans %}{% trans %}project.bom_import.field_mapping.maps_to{% endtrans %}{% trans %}project.bom_import.field_mapping.suggestion{% endtrans %}{% trans %}project.bom_import.field_mapping.priority{% endtrans %}
+ {{ field }} + + {{ form_widget(form['mapping_' ~ field_name_mapping[field]], { + 'attr': { + 'class': 'form-select field-mapping-select', + 'data-field': field + } + }) }} + + {% if suggested_mapping[field] is defined %} + + + {{ suggested_mapping[field] }} + + {% else %} + + + {% trans %}project.bom_import.field_mapping.no_suggestion{% endtrans %} + + {% endif %} + + +
+
+ +
+
{% trans %}project.bom_import.field_mapping.summary{% endtrans %}:
+
+ + {% trans %}project.bom_import.field_mapping.select_to_see_summary{% endtrans %} +
+
+
+
+ +
+ {{ form_widget(form.submit, { + 'attr': { + 'class': 'btn btn-primary' + } + }) }} + + + {% trans %}common.back{% endtrans %} + +
+ + {{ form_end(form) }} + + +{% endblock %} \ No newline at end of file From 7c1ab6460d05e4d30e0707217c8b3ca67210c7bb Mon Sep 17 00:00:00 2001 From: barisgit Date: Sun, 3 Aug 2025 18:58:31 +0200 Subject: [PATCH 03/10] Add tests to cover new additions --- makefile | 4 + .../ImportExportSystem/BOMImporterTest.php | 494 ++++++++++++++++++ .../BOMValidationServiceTest.php | 349 +++++++++++++ 3 files changed, 847 insertions(+) create mode 100644 tests/Services/ImportExportSystem/BOMValidationServiceTest.php diff --git a/makefile b/makefile index 6b5ac61f..9041ba0f 100644 --- a/makefile +++ b/makefile @@ -73,6 +73,10 @@ test-run: @echo "🧪 Running tests..." php bin/phpunit +test-typecheck: + @echo "🧪 Running type checks..." + COMPOSER_MEMORY_LIMIT=-1 composer phpstan + # Quick test reset (clean + migrate + fixtures, skip DB creation) test-reset: test-cache-clear test-db-migrate test-fixtures @echo "✅ Test environment reset complete!" diff --git a/tests/Services/ImportExportSystem/BOMImporterTest.php b/tests/Services/ImportExportSystem/BOMImporterTest.php index b9aba1d4..52c633d0 100644 --- a/tests/Services/ImportExportSystem/BOMImporterTest.php +++ b/tests/Services/ImportExportSystem/BOMImporterTest.php @@ -22,9 +22,12 @@ declare(strict_types=1); */ namespace App\Tests\Services\ImportExportSystem; +use App\Entity\Parts\Part; +use App\Entity\Parts\Supplier; use App\Entity\ProjectSystem\Project; use App\Entity\ProjectSystem\ProjectBOMEntry; use App\Services\ImportExportSystem\BOMImporter; +use Doctrine\ORM\EntityManagerInterface; use Symfony\Bundle\FrameworkBundle\Test\WebTestCase; use Symfony\Component\HttpFoundation\File\File; @@ -36,11 +39,17 @@ class BOMImporterTest extends WebTestCase */ protected $service; + /** + * @var EntityManagerInterface + */ + protected $entityManager; + protected function setUp(): void { //Get a service instance. self::bootKernel(); $this->service = self::getContainer()->get(BOMImporter::class); + $this->entityManager = self::getContainer()->get(EntityManagerInterface::class); } public function testImportFileIntoProject(): void @@ -119,4 +128,489 @@ class BOMImporterTest extends WebTestCase $this->service->stringToBOMEntries($input, ['type' => 'kicad_pcbnew']); } + + public function testDetectFields(): void + { + $input = <<service->detectFields($input); + + $this->assertIsArray($fields); + $this->assertCount(8, $fields); + $this->assertContains('Reference', $fields); + $this->assertContains('Value', $fields); + $this->assertContains('Footprint', $fields); + $this->assertContains('Quantity', $fields); + $this->assertContains('MPN', $fields); + $this->assertContains('Manufacturer', $fields); + $this->assertContains('LCSC SPN', $fields); + $this->assertContains('Mouser SPN', $fields); + } + + public function testDetectFieldsWithQuotes(): void + { + $input = <<service->detectFields($input); + + $this->assertIsArray($fields); + $this->assertCount(8, $fields); + $this->assertEquals('Reference', $fields[0]); + $this->assertEquals('Value', $fields[1]); + } + + public function testDetectFieldsWithSemicolon(): void + { + $input = <<service->detectFields($input, ';'); + + $this->assertIsArray($fields); + $this->assertCount(8, $fields); + $this->assertEquals('Reference', $fields[0]); + $this->assertEquals('Value', $fields[1]); + } + + public function testGetAvailableFieldTargets(): void + { + $targets = $this->service->getAvailableFieldTargets(); + + $this->assertIsArray($targets); + $this->assertArrayHasKey('Designator', $targets); + $this->assertArrayHasKey('Quantity', $targets); + $this->assertArrayHasKey('Value', $targets); + $this->assertArrayHasKey('Package', $targets); + $this->assertArrayHasKey('MPN', $targets); + $this->assertArrayHasKey('Manufacturer', $targets); + $this->assertArrayHasKey('Part-DB ID', $targets); + $this->assertArrayHasKey('Comment', $targets); + + // Check structure of a target + $this->assertArrayHasKey('label', $targets['Designator']); + $this->assertArrayHasKey('description', $targets['Designator']); + $this->assertArrayHasKey('required', $targets['Designator']); + $this->assertArrayHasKey('multiple', $targets['Designator']); + + $this->assertTrue($targets['Designator']['required']); + $this->assertTrue($targets['Quantity']['required']); + $this->assertFalse($targets['Value']['required']); + } + + public function testGetAvailableFieldTargetsWithSuppliers(): void + { + // Create test suppliers + $supplier1 = new Supplier(); + $supplier1->setName('LCSC'); + $supplier2 = new Supplier(); + $supplier2->setName('Mouser'); + + $this->entityManager->persist($supplier1); + $this->entityManager->persist($supplier2); + $this->entityManager->flush(); + + $targets = $this->service->getAvailableFieldTargets(); + + $this->assertArrayHasKey('LCSC SPN', $targets); + $this->assertArrayHasKey('Mouser SPN', $targets); + + $this->assertEquals('LCSC SPN', $targets['LCSC SPN']['label']); + $this->assertEquals('Mouser SPN', $targets['Mouser SPN']['label']); + $this->assertFalse($targets['LCSC SPN']['required']); + $this->assertTrue($targets['LCSC SPN']['multiple']); + + // Clean up + $this->entityManager->remove($supplier1); + $this->entityManager->remove($supplier2); + $this->entityManager->flush(); + } + + public function testGetSuggestedFieldMapping(): void + { + $detected_fields = [ + 'Reference', + 'Value', + 'Footprint', + 'Quantity', + 'MPN', + 'Manufacturer', + 'LCSC', + 'Mouser', + 'Part-DB ID', + 'Comment' + ]; + + $suggestions = $this->service->getSuggestedFieldMapping($detected_fields); + + $this->assertIsArray($suggestions); + $this->assertEquals('Designator', $suggestions['Reference']); + $this->assertEquals('Value', $suggestions['Value']); + $this->assertEquals('Package', $suggestions['Footprint']); + $this->assertEquals('Quantity', $suggestions['Quantity']); + $this->assertEquals('MPN', $suggestions['MPN']); + $this->assertEquals('Manufacturer', $suggestions['Manufacturer']); + $this->assertEquals('Part-DB ID', $suggestions['Part-DB ID']); + $this->assertEquals('Comment', $suggestions['Comment']); + } + + public function testGetSuggestedFieldMappingWithSuppliers(): void + { + // Create test suppliers + $supplier1 = new Supplier(); + $supplier1->setName('LCSC'); + $supplier2 = new Supplier(); + $supplier2->setName('Mouser'); + + $this->entityManager->persist($supplier1); + $this->entityManager->persist($supplier2); + $this->entityManager->flush(); + + $detected_fields = [ + 'Reference', + 'LCSC', + 'Mouser', + 'lcsc_part', + 'mouser_spn' + ]; + + $suggestions = $this->service->getSuggestedFieldMapping($detected_fields); + + $this->assertIsArray($suggestions); + $this->assertEquals('Designator', $suggestions['Reference']); + // Note: The exact mapping depends on the pattern matching logic + // We just check that supplier fields are mapped to something + $this->assertArrayHasKey('LCSC', $suggestions); + $this->assertArrayHasKey('Mouser', $suggestions); + $this->assertArrayHasKey('lcsc_part', $suggestions); + $this->assertArrayHasKey('mouser_spn', $suggestions); + + // Clean up + $this->entityManager->remove($supplier1); + $this->entityManager->remove($supplier2); + $this->entityManager->flush(); + } + + public function testValidateFieldMappingValid(): void + { + $field_mapping = [ + 'Reference' => 'Designator', + 'Quantity' => 'Quantity', + 'Value' => 'Value' + ]; + + $detected_fields = ['Reference', 'Quantity', 'Value', 'MPN']; + + $result = $this->service->validateFieldMapping($field_mapping, $detected_fields); + + $this->assertIsArray($result); + $this->assertArrayHasKey('errors', $result); + $this->assertArrayHasKey('warnings', $result); + $this->assertArrayHasKey('is_valid', $result); + + $this->assertTrue($result['is_valid']); + $this->assertEmpty($result['errors']); + $this->assertNotEmpty($result['warnings']); // Should warn about unmapped MPN + } + + public function testValidateFieldMappingMissingRequired(): void + { + $field_mapping = [ + 'Value' => 'Value', + 'MPN' => 'MPN' + ]; + + $detected_fields = ['Value', 'MPN']; + + $result = $this->service->validateFieldMapping($field_mapping, $detected_fields); + + $this->assertFalse($result['is_valid']); + $this->assertNotEmpty($result['errors']); + $this->assertContains("Required field 'Designator' is not mapped from any CSV column.", $result['errors']); + $this->assertContains("Required field 'Quantity' is not mapped from any CSV column.", $result['errors']); + } + + public function testValidateFieldMappingInvalidTarget(): void + { + $field_mapping = [ + 'Reference' => 'Designator', + 'Quantity' => 'Quantity', + 'Value' => 'InvalidTarget' + ]; + + $detected_fields = ['Reference', 'Quantity', 'Value']; + + $result = $this->service->validateFieldMapping($field_mapping, $detected_fields); + + $this->assertFalse($result['is_valid']); + $this->assertNotEmpty($result['errors']); + $this->assertContains("Invalid target field 'InvalidTarget' for CSV field 'Value'.", $result['errors']); + } + + public function testStringToBOMEntriesKiCADSchematic(): void + { + $input = << 'Designator', + 'Value' => 'Value', + 'Footprint' => 'Package', + 'Quantity' => 'Quantity', + 'MPN' => 'MPN', + 'Manufacturer' => 'Manufacturer', + 'LCSC SPN' => 'LCSC SPN', + 'Mouser SPN' => 'Mouser SPN' + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(2, $bom_entries); + + // Check first entry + $this->assertEquals('R1,R2', $bom_entries[0]->getMountnames()); + $this->assertEquals(2.0, $bom_entries[0]->getQuantity()); + $this->assertEquals('CRCW080510K0FKEA (R_0805_2012Metric)', $bom_entries[0]->getName()); + $this->assertStringContainsString('Value: 10k', $bom_entries[0]->getComment()); + $this->assertStringContainsString('MPN: CRCW080510K0FKEA', $bom_entries[0]->getComment()); + $this->assertStringContainsString('Manf: Vishay', $bom_entries[0]->getComment()); + + // Check second entry + $this->assertEquals('C1', $bom_entries[1]->getMountnames()); + $this->assertEquals(1.0, $bom_entries[1]->getQuantity()); + } + + public function testStringToBOMEntriesKiCADSchematicWithPriority(): void + { + $input = << 'Designator', + 'Value' => 'Value', + 'MPN1' => 'MPN', + 'MPN2' => 'MPN', + 'Quantity' => 'Quantity' + ]; + + $field_priorities = [ + 'MPN1' => 1, + 'MPN2' => 2 + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'field_priorities' => $field_priorities, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(2, $bom_entries); + + // First entry should use MPN1 (higher priority) + $this->assertEquals('CRCW080510K0FKEA', $bom_entries[0]->getName()); + + // Second entry should use MPN2 (MPN1 is empty) + $this->assertEquals('CL21A104KOCLRNC', $bom_entries[1]->getName()); + } + + public function testStringToBOMEntriesKiCADSchematicWithPartDBID(): void + { + // Create a test part with required fields + $part = new Part(); + $part->setName('Test Part'); + $part->setCategory($this->getDefaultCategory($this->entityManager)); + $this->entityManager->persist($part); + $this->entityManager->flush(); + + $input = <<getID()}","2" + CSV; + + $field_mapping = [ + 'Reference' => 'Designator', + 'Value' => 'Value', + 'Part-DB ID' => 'Part-DB ID', + 'Quantity' => 'Quantity' + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(1, $bom_entries); + + $this->assertEquals('Test Part', $bom_entries[0]->getName()); + $this->assertSame($part, $bom_entries[0]->getPart()); + $this->assertStringContainsString("Part-DB ID: {$part->getID()}", $bom_entries[0]->getComment()); + + // Clean up + $this->entityManager->remove($part); + $this->entityManager->flush(); + } + + public function testStringToBOMEntriesKiCADSchematicWithInvalidPartDBID(): void + { + $input = << 'Designator', + 'Value' => 'Value', + 'Part-DB ID' => 'Part-DB ID', + 'Quantity' => 'Quantity' + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(1, $bom_entries); + + $this->assertEquals('10k', $bom_entries[0]->getName()); // Should use Value as name + $this->assertNull($bom_entries[0]->getPart()); // Should not link to part + $this->assertStringContainsString("Part-DB ID: 99999 (NOT FOUND)", $bom_entries[0]->getComment()); + } + + public function testStringToBOMEntriesKiCADSchematicMergeDuplicates(): void + { + $input = << 'Designator', + 'Value' => 'Value', + 'MPN' => 'MPN', + 'Quantity' => 'Quantity' + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(1, $bom_entries); // Should merge into one entry + + $this->assertEquals('R1,R2', $bom_entries[0]->getMountnames()); + $this->assertEquals(2.0, $bom_entries[0]->getQuantity()); + $this->assertEquals('CRCW080510K0FKEA', $bom_entries[0]->getName()); + } + + public function testStringToBOMEntriesKiCADSchematicMissingRequired(): void + { + $input = << 'Value', + 'MPN' => 'MPN' + ]; + + $this->expectException(\UnexpectedValueException::class); + $this->expectExceptionMessage('Required field "Designator" is missing or empty'); + + $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + } + + public function testStringToBOMEntriesKiCADSchematicQuantityMismatch(): void + { + $input = << 'Designator', + 'Value' => 'Value', + 'Quantity' => 'Quantity' + ]; + + $this->expectException(\UnexpectedValueException::class); + $this->expectExceptionMessage('Mismatch between quantity and component references'); + + $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + } + + public function testStringToBOMEntriesKiCADSchematicWithBOM(): void + { + // Test with BOM (Byte Order Mark) + $input = "\xEF\xBB\xBF" . << 'Designator', + 'Value' => 'Value', + 'Quantity' => 'Quantity' + ]; + + $bom_entries = $this->service->stringToBOMEntries($input, [ + 'type' => 'kicad_schematic', + 'field_mapping' => $field_mapping, + 'delimiter' => ',' + ]); + + $this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries); + $this->assertCount(1, $bom_entries); + $this->assertEquals('R1,R2', $bom_entries[0]->getMountnames()); + } + + private function getDefaultCategory(EntityManagerInterface $entityManager) + { + // Get the first available category or create a default one + $categoryRepo = $entityManager->getRepository(\App\Entity\Parts\Category::class); + $categories = $categoryRepo->findAll(); + + if (empty($categories)) { + // Create a default category if none exists + $category = new \App\Entity\Parts\Category(); + $category->setName('Default Category'); + $entityManager->persist($category); + $entityManager->flush(); + return $category; + } + + return $categories[0]; + } } diff --git a/tests/Services/ImportExportSystem/BOMValidationServiceTest.php b/tests/Services/ImportExportSystem/BOMValidationServiceTest.php new file mode 100644 index 00000000..055db8b4 --- /dev/null +++ b/tests/Services/ImportExportSystem/BOMValidationServiceTest.php @@ -0,0 +1,349 @@ +. + */ +namespace App\Tests\Services\ImportExportSystem; + +use App\Entity\Parts\Part; +use App\Services\ImportExportSystem\BOMValidationService; +use Doctrine\ORM\EntityManagerInterface; +use PHPUnit\Framework\TestCase; +use Symfony\Bundle\FrameworkBundle\Test\WebTestCase; +use Symfony\Contracts\Translation\TranslatorInterface; + +/** + * @see \App\Services\ImportExportSystem\BOMValidationService + */ +class BOMValidationServiceTest extends WebTestCase +{ + private BOMValidationService $validationService; + private EntityManagerInterface $entityManager; + private TranslatorInterface $translator; + + protected function setUp(): void + { + self::bootKernel(); + $this->entityManager = self::getContainer()->get(EntityManagerInterface::class); + $this->translator = self::getContainer()->get(TranslatorInterface::class); + $this->validationService = new BOMValidationService($this->entityManager, $this->translator); + } + + public function testValidateBOMEntryWithValidData(): void + { + $entry = [ + 'Designator' => 'R1,C2,R3', + 'Quantity' => '3', + 'MPN' => 'RES-10K', + 'Package' => '0603', + 'Value' => '10k', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); + $this->assertEmpty($result['errors']); + $this->assertEquals(1, $result['line_number']); + } + + public function testValidateBOMEntryWithMissingRequiredFields(): void + { + $entry = [ + 'MPN' => 'RES-10K', + 'Package' => '0603', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertCount(2, $result['errors']); + $this->assertStringContainsString('Designator', (string) $result['errors'][0]); + $this->assertStringContainsString('Quantity', (string) $result['errors'][1]); + } + + public function testValidateBOMEntryWithQuantityMismatch(): void + { + $entry = [ + 'Designator' => 'R1,C2,R3,C4', + 'Quantity' => '3', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('Mismatch between quantity and component references', (string) $result['errors'][0]); + } + + public function testValidateBOMEntryWithInvalidQuantity(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => 'abc', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertGreaterThanOrEqual(1, count($result['errors'])); + $this->assertStringContainsString('not a valid number', implode(' ', array_map('strval', $result['errors']))); + } + + public function testValidateBOMEntryWithZeroQuantity(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '0', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertGreaterThanOrEqual(1, count($result['errors'])); + $this->assertStringContainsString('must be greater than 0', implode(' ', array_map('strval', $result['errors']))); + } + + public function testValidateBOMEntryWithDuplicateDesignators(): void + { + $entry = [ + 'Designator' => 'R1,R1,C2', + 'Quantity' => '3', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('Duplicate component references', (string) $result['errors'][0]); + } + + public function testValidateBOMEntryWithInvalidDesignatorFormat(): void + { + $entry = [ + 'Designator' => 'R1,invalid,C2', + 'Quantity' => '3', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); // Warnings don't make it invalid + $this->assertCount(1, $result['warnings']); + $this->assertStringContainsString('unusual format', (string) $result['warnings'][0]); + } + + public function testValidateBOMEntryWithEmptyDesignator(): void + { + $entry = [ + 'Designator' => '', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertGreaterThanOrEqual(1, count($result['errors'])); + $this->assertStringContainsString('Required field "Designator" is missing or empty', implode(' ', array_map('strval', $result['errors']))); + } + + public function testValidateBOMEntryWithInvalidPartDBID(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + 'Part-DB ID' => 'abc', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertFalse($result['is_valid']); + $this->assertGreaterThanOrEqual(1, count($result['errors'])); + $this->assertStringContainsString('not a valid number', implode(' ', array_map('strval', $result['errors']))); + } + + public function testValidateBOMEntryWithNonExistentPartDBID(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + 'Part-DB ID' => '999999', // Use very high ID that doesn't exist + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); // Warnings don't make it invalid + $this->assertCount(1, $result['warnings']); + $this->assertStringContainsString('not found in database', (string) $result['warnings'][0]); + } + + public function testValidateBOMEntryWithNoComponentName(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'Package' => '0603', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); // Warnings don't make it invalid + $this->assertCount(1, $result['warnings']); + $this->assertStringContainsString('No component name/designation', (string) $result['warnings'][0]); + } + + public function testValidateBOMEntryWithLongPackageName(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + 'Package' => str_repeat('A', 150), // Very long package name + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); // Warnings don't make it invalid + $this->assertCount(1, $result['warnings']); + $this->assertStringContainsString('unusually long', (string) $result['warnings'][0]); + } + + public function testValidateBOMEntryWithLibraryPrefix(): void + { + $entry = [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + 'Package' => 'Resistor_SMD:R_0603_1608Metric', + ]; + + $result = $this->validationService->validateBOMEntry($entry, 1); + + $this->assertTrue($result['is_valid']); + $this->assertCount(1, $result['info']); + $this->assertStringContainsString('library prefix', $result['info'][0]); + } + + public function testValidateBOMEntriesWithMultipleEntries(): void + { + $entries = [ + [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + ], + [ + 'Designator' => 'C1,C2', + 'Quantity' => '2', + 'MPN' => 'CAP-100nF', + ], + ]; + + $result = $this->validationService->validateBOMEntries($entries); + + $this->assertTrue($result['is_valid']); + $this->assertEquals(2, $result['total_entries']); + $this->assertEquals(2, $result['valid_entries']); + $this->assertEquals(0, $result['invalid_entries']); + $this->assertCount(2, $result['line_results']); + } + + public function testValidateBOMEntriesWithMixedResults(): void + { + $entries = [ + [ + 'Designator' => 'R1', + 'Quantity' => '1', + 'MPN' => 'RES-10K', + ], + [ + 'Designator' => 'C1,C2', + 'Quantity' => '1', // Mismatch + 'MPN' => 'CAP-100nF', + ], + ]; + + $result = $this->validationService->validateBOMEntries($entries); + + $this->assertFalse($result['is_valid']); + $this->assertEquals(2, $result['total_entries']); + $this->assertEquals(1, $result['valid_entries']); + $this->assertEquals(1, $result['invalid_entries']); + $this->assertCount(1, $result['errors']); + } + + public function testGetValidationStats(): void + { + $validation_result = [ + 'total_entries' => 10, + 'valid_entries' => 8, + 'invalid_entries' => 2, + 'errors' => ['Error 1', 'Error 2'], + 'warnings' => ['Warning 1'], + 'info' => ['Info 1', 'Info 2'], + ]; + + $stats = $this->validationService->getValidationStats($validation_result); + + $this->assertEquals(10, $stats['total_entries']); + $this->assertEquals(8, $stats['valid_entries']); + $this->assertEquals(2, $stats['invalid_entries']); + $this->assertEquals(2, $stats['error_count']); + $this->assertEquals(1, $stats['warning_count']); + $this->assertEquals(2, $stats['info_count']); + $this->assertEquals(80.0, $stats['success_rate']); + } + + public function testGetErrorMessage(): void + { + $validation_result = [ + 'is_valid' => false, + 'errors' => ['Error 1', 'Error 2'], + 'warnings' => ['Warning 1'], + ]; + + $message = $this->validationService->getErrorMessage($validation_result); + + $this->assertStringContainsString('Errors:', $message); + $this->assertStringContainsString('• Error 1', $message); + $this->assertStringContainsString('• Error 2', $message); + $this->assertStringContainsString('Warnings:', $message); + $this->assertStringContainsString('• Warning 1', $message); + } + + public function testGetErrorMessageWithValidResult(): void + { + $validation_result = [ + 'is_valid' => true, + 'errors' => [], + 'warnings' => [], + ]; + + $message = $this->validationService->getErrorMessage($validation_result); + + $this->assertEquals('', $message); + } +} \ No newline at end of file From 72e3766be534ca3621da5fc19d62133410ec7d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:10:12 +0200 Subject: [PATCH 04/10] Added missing translations that got removed during rebase --- translations/messages.en.xlf | 366 +++++++++++++++++++++++++++++++++++ 1 file changed, 366 insertions(+) diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 8d1e55c8..bbbbb075 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -13057,5 +13057,371 @@ Please note, that you can not impersonate a disabled user. If you try you will g Redacted for security reasons + + + project.bom_import.map_fields + Map Fields + + + + + project.bom_import.map_fields.help + Configure how CSV columns map to BOM fields + + + + + project.bom_import.delimiter + Delimiter + + + + + project.bom_import.delimiter.comma + Comma (,) + + + + + project.bom_import.delimiter.semicolon + Semicolon (;) + + + + + project.bom_import.delimiter.tab + Tab + + + + + project.bom_import.field_mapping.title + Field Mapping + + + + + project.bom_import.field_mapping.csv_field + CSV Field + + + + + project.bom_import.field_mapping.maps_to + Maps To + + + + + project.bom_import.field_mapping.suggestion + Suggestion + + + + + project.bom_import.field_mapping.priority + Priority + + + + + project.bom_import.field_mapping.priority_help + Priority (lower number = higher priority) + + + + + project.bom_import.field_mapping.priority_short + P + + + + + project.bom_import.field_mapping.priority_note + Priority Tip: Lower numbers = higher priority. Default priority is 10. Use priorities 1-9 for most important fields, 10+ for normal priority. + + + + + project.bom_import.field_mapping.summary + Field Mapping Summary + + + + + project.bom_import.field_mapping.select_to_see_summary + Select field mappings to see summary + + + + + project.bom_import.field_mapping.no_suggestion + No suggestion + + + + + project.bom_import.preview + Preview + + + + + project.bom_import.flash.session_expired + Import session has expired. Please upload your file again. + + + + + project.bom_import.field_mapping.ignore + Ignore + + + + + project.bom_import.type.kicad_schematic + KiCAD Schematic BOM (CSV file) + + + + + common.back + Back + + + + + project.bom_import.validation.errors.required_field_missing + Line %line%: Required field "%field%" is missing or empty. Please ensure this field is mapped and contains data. + + + + + project.bom_import.validation.errors.no_valid_designators + Line %line%: Designator field contains no valid component references. Expected format: "R1,C2,U3" or "R1, C2, U3". + + + + + project.bom_import.validation.warnings.unusual_designator_format + Line %line%: Some component references may have unusual format: %designators%. Expected format: "R1", "C2", "U3", etc. + + + + + project.bom_import.validation.errors.duplicate_designators + Line %line%: Duplicate component references found: %designators%. Each component should be referenced only once per line. + + + + + project.bom_import.validation.errors.invalid_quantity + Line %line%: Quantity "%quantity%" is not a valid number. Please enter a numeric value (e.g., 1, 2.5, 10). + + + + + project.bom_import.validation.errors.quantity_zero_or_negative + Line %line%: Quantity must be greater than 0, got %quantity%. + + + + + project.bom_import.validation.warnings.quantity_unusually_high + Line %line%: Quantity %quantity% seems unusually high. Please verify this is correct. + + + + + project.bom_import.validation.warnings.quantity_not_whole_number + Line %line%: Quantity %quantity% is not a whole number, but you have %count% component references. This may indicate a mismatch. + + + + + project.bom_import.validation.errors.quantity_designator_mismatch + Line %line%: Mismatch between quantity and component references. Quantity: %quantity%, References: %count% (%designators%). These should match. Either adjust the quantity or check your component references. + + + + + project.bom_import.validation.errors.invalid_partdb_id + Line %line%: Part-DB ID "%id%" is not a valid number. Please enter a numeric ID. + + + + + project.bom_import.validation.errors.partdb_id_zero_or_negative + Line %line%: Part-DB ID must be greater than 0, got %id%. + + + + + project.bom_import.validation.warnings.partdb_id_not_found + Line %line%: Part-DB ID %id% not found in database. The component will be imported without linking to an existing part. + + + + + project.bom_import.validation.info.partdb_link_success + Line %line%: Successfully linked to Part-DB part "%name%" (ID: %id%). + + + + + project.bom_import.validation.warnings.no_component_name + Line %line%: No component name/designation provided (MPN, Designation, or Value). Component will be named "Unknown Component". + + + + + project.bom_import.validation.warnings.package_name_too_long + Line %line%: Package name "%package%" is unusually long. Please verify this is correct. + + + + + project.bom_import.validation.info.library_prefix_detected + Line %line%: Package "%package%" contains library prefix. This will be automatically removed during import. + + + + + project.bom_import.validation.errors.non_numeric_field + Line %line%: Field "%field%" contains non-numeric value "%value%". Please enter a valid number. + + + + + project.bom_import.validation.info.import_summary + Import summary: %total% total entries, %valid% valid, %invalid% with issues. + + + + + project.bom_import.validation.errors.summary + Found %count% validation error(s) that must be fixed before import can proceed. + + + + + project.bom_import.validation.warnings.summary + Found %count% warning(s). Please review these issues before proceeding. + + + + + project.bom_import.validation.info.all_valid + All entries passed validation successfully! + + + + + project.bom_import.validation.summary + Validation Summary + + + + + project.bom_import.validation.total_entries + Total Entries + + + + + project.bom_import.validation.valid_entries + Valid Entries + + + + + project.bom_import.validation.invalid_entries + Invalid Entries + + + + + project.bom_import.validation.success_rate + Success Rate + + + + + project.bom_import.validation.errors.title + Validation Errors + + + + + project.bom_import.validation.errors.description + The following errors must be fixed before the import can proceed: + + + + + project.bom_import.validation.warnings.title + Validation Warnings + + + + + project.bom_import.validation.warnings.description + The following warnings should be reviewed before proceeding: + + + + + project.bom_import.validation.info.title + Information + + + + + project.bom_import.validation.details.title + Detailed Validation Results + + + + + project.bom_import.validation.details.line + Line + + + + + project.bom_import.validation.details.status + Status + + + + + project.bom_import.validation.details.messages + Messages + + + + + project.bom_import.validation.details.valid + Valid + + + + + project.bom_import.validation.details.invalid + Invalid + + + + + project.bom_import.validation.all_valid + All entries are valid and ready for import! + + + + + project.bom_import.validation.fix_errors + Please fix the validation errors before proceeding with the import. + + From 1d33d95c57831eb7ba5790918d5f9ef0c9e9a077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:10:47 +0200 Subject: [PATCH 05/10] Show validation error messages in mapping step --- src/Controller/ProjectController.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Controller/ProjectController.php b/src/Controller/ProjectController.php index 444ff5b3..ec9147c1 100644 --- a/src/Controller/ProjectController.php +++ b/src/Controller/ProjectController.php @@ -378,7 +378,7 @@ class ProjectController extends AbstractController } // If there are validation errors, show them and stop - if (!empty($validation_result['errors'])) { + if (!empty($validation_result['errors'])) { foreach ($validation_result['errors'] as $error) { $this->addFlash('error', $error); } @@ -449,6 +449,16 @@ class ProjectController extends AbstractController // When we get here, there were validation errors $this->addFlash('error', t('project.bom_import.flash.invalid_entries')); + //Print validation errors to log for debugging + foreach ($errors as $error) { + $logger->error('BOM entry validation error', [ + 'message' => $error->getMessage(), + 'invalid_value' => $error->getInvalidValue(), + ]); + //And show as flash message + $this->addFlash('error', $error->getMessage(),); + } + } catch (\UnexpectedValueException | SyntaxError $e) { $this->addFlash('error', t('project.bom_import.flash.invalid_file', ['%message%' => $e->getMessage()])); } From 76f3c379b508036f4bb38ce307ee5ac4f9ac77b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:20:07 +0200 Subject: [PATCH 06/10] Added generic CSV type option, to highlight the universal nature of the importer --- src/Controller/ProjectController.php | 3 +- translations/messages.en.xlf | 762 ++++++++++++++------------- 2 files changed, 386 insertions(+), 379 deletions(-) diff --git a/src/Controller/ProjectController.php b/src/Controller/ProjectController.php index ec9147c1..2a6d19ee 100644 --- a/src/Controller/ProjectController.php +++ b/src/Controller/ProjectController.php @@ -149,6 +149,7 @@ class ProjectController extends AbstractController 'choices' => [ 'project.bom_import.type.kicad_pcbnew' => 'kicad_pcbnew', 'project.bom_import.type.kicad_schematic' => 'kicad_schematic', + 'project.bom_import.type.generic_csv' => 'generic_csv', ] ]); $builder->add('clear_existing_bom', CheckboxType::class, [ @@ -176,7 +177,7 @@ class ProjectController extends AbstractController try { // For schematic imports, redirect to field mapping step - if ($import_type === 'kicad_schematic') { + if (in_array($import_type, ['kicad_schematic', 'generic_csv'], true)) { // Store file content and options in session for field mapping step $file_content = $form->get('file')->getData()->getContent(); $clear_existing = $form->get('clear_existing_bom')->getData(); diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index bbbbb075..888384da 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -12333,7 +12333,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.ips.element14.apiKey.help - You can register for an API key on <a href="https://partner.element14.com/">https://partner.element14.com/</a>. + https://partner.element14.com/.]]> @@ -12345,7 +12345,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.ips.element14.storeId.help - The store domain to retrieve the data from. This decides the language and currency of results. See <a href="https://partner.element14.com/docs/Product_Search_API_REST__Description">here</a> for a list of valid domains. + here for a list of valid domains.]]> @@ -12363,7 +12363,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.ips.tme.token.help - You can get an API token and secret on <a href="https://developers.tme.eu/en/">https://developers.tme.eu/en/</a>. + https://developers.tme.eu/en/.]]> @@ -12411,7 +12411,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.ips.mouser.apiKey.help - You can register for an API key on <a href="https://eu.mouser.com/api-hub/">https://eu.mouser.com/api-hub/</a>. + https://eu.mouser.com/api-hub/.]]> @@ -12489,7 +12489,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.system.attachments - Attachments & Files + @@ -12513,7 +12513,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.system.attachments.allowDownloads.help - With this option users can download external files into Part-DB by providing an URL. <b>Attention: This can be a security issue, as it might allow users to access intranet ressources via Part-DB!</b> + Attention: This can be a security issue, as it might allow users to access intranet ressources via Part-DB!]]> @@ -12687,8 +12687,8 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.system.localization.base_currency_description - The currency that is used to store price information and exchange rates in. This currency is assumed, when no currency is set for a price information. -<b>Please note that the currencies are not converted, when changing this value. So changing the default currency after you already added price information, will result in wrong prices!</b> + Please note that the currencies are not converted, when changing this value. So changing the default currency after you already added price information, will result in wrong prices!]]> @@ -12718,7 +12718,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.misc.kicad_eda.category_depth.help - This value determines the depth of the category tree, that is visible inside KiCad. 0 means that only the top level categories are visible. Set to a value > 0 to show more levels. Set to -1, to show all parts of Part-DB inside a sigle cnategory in KiCad. + 0 to show more levels. Set to -1, to show all parts of Part-DB inside a sigle cnategory in KiCad.]]> @@ -12736,7 +12736,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.behavior.sidebar.items.help - The menus which appear at the sidebar by default. Order of items can be changed via drag & drop. + @@ -12784,7 +12784,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.behavior.table.parts_default_columns.help - The columns to show by default in part tables. Order of items can be changed via drag & drop. + @@ -12838,7 +12838,7 @@ Please note, that you can not impersonate a disabled user. If you try you will g settings.ips.oemsecrets.sortMode.M - Completeness & Manufacturer name + @@ -13057,371 +13057,377 @@ Please note, that you can not impersonate a disabled user. If you try you will g Redacted for security reasons - - - project.bom_import.map_fields - Map Fields - - - - - project.bom_import.map_fields.help - Configure how CSV columns map to BOM fields - - - - - project.bom_import.delimiter - Delimiter - - - - - project.bom_import.delimiter.comma - Comma (,) - - - - - project.bom_import.delimiter.semicolon - Semicolon (;) - - - - - project.bom_import.delimiter.tab - Tab - - - - - project.bom_import.field_mapping.title - Field Mapping - - - - - project.bom_import.field_mapping.csv_field - CSV Field - - - - - project.bom_import.field_mapping.maps_to - Maps To - - - - - project.bom_import.field_mapping.suggestion - Suggestion - - - - - project.bom_import.field_mapping.priority - Priority - - - - - project.bom_import.field_mapping.priority_help - Priority (lower number = higher priority) - - - - - project.bom_import.field_mapping.priority_short - P - - - - - project.bom_import.field_mapping.priority_note - Priority Tip: Lower numbers = higher priority. Default priority is 10. Use priorities 1-9 for most important fields, 10+ for normal priority. - - - - - project.bom_import.field_mapping.summary - Field Mapping Summary - - - - - project.bom_import.field_mapping.select_to_see_summary - Select field mappings to see summary - - - - - project.bom_import.field_mapping.no_suggestion - No suggestion - - - - - project.bom_import.preview - Preview - - - - - project.bom_import.flash.session_expired - Import session has expired. Please upload your file again. - - - - - project.bom_import.field_mapping.ignore - Ignore - - - - - project.bom_import.type.kicad_schematic - KiCAD Schematic BOM (CSV file) - - - - - common.back - Back - - - - - project.bom_import.validation.errors.required_field_missing - Line %line%: Required field "%field%" is missing or empty. Please ensure this field is mapped and contains data. - - - - - project.bom_import.validation.errors.no_valid_designators - Line %line%: Designator field contains no valid component references. Expected format: "R1,C2,U3" or "R1, C2, U3". - - - - - project.bom_import.validation.warnings.unusual_designator_format - Line %line%: Some component references may have unusual format: %designators%. Expected format: "R1", "C2", "U3", etc. - - - - - project.bom_import.validation.errors.duplicate_designators - Line %line%: Duplicate component references found: %designators%. Each component should be referenced only once per line. - - - - - project.bom_import.validation.errors.invalid_quantity - Line %line%: Quantity "%quantity%" is not a valid number. Please enter a numeric value (e.g., 1, 2.5, 10). - - - - - project.bom_import.validation.errors.quantity_zero_or_negative - Line %line%: Quantity must be greater than 0, got %quantity%. - - - - - project.bom_import.validation.warnings.quantity_unusually_high - Line %line%: Quantity %quantity% seems unusually high. Please verify this is correct. - - - - - project.bom_import.validation.warnings.quantity_not_whole_number - Line %line%: Quantity %quantity% is not a whole number, but you have %count% component references. This may indicate a mismatch. - - - - - project.bom_import.validation.errors.quantity_designator_mismatch - Line %line%: Mismatch between quantity and component references. Quantity: %quantity%, References: %count% (%designators%). These should match. Either adjust the quantity or check your component references. - - - - - project.bom_import.validation.errors.invalid_partdb_id - Line %line%: Part-DB ID "%id%" is not a valid number. Please enter a numeric ID. - - - - - project.bom_import.validation.errors.partdb_id_zero_or_negative - Line %line%: Part-DB ID must be greater than 0, got %id%. - - - - - project.bom_import.validation.warnings.partdb_id_not_found - Line %line%: Part-DB ID %id% not found in database. The component will be imported without linking to an existing part. - - - - - project.bom_import.validation.info.partdb_link_success - Line %line%: Successfully linked to Part-DB part "%name%" (ID: %id%). - - - - - project.bom_import.validation.warnings.no_component_name - Line %line%: No component name/designation provided (MPN, Designation, or Value). Component will be named "Unknown Component". - - - - - project.bom_import.validation.warnings.package_name_too_long - Line %line%: Package name "%package%" is unusually long. Please verify this is correct. - - - - - project.bom_import.validation.info.library_prefix_detected - Line %line%: Package "%package%" contains library prefix. This will be automatically removed during import. - - - - - project.bom_import.validation.errors.non_numeric_field - Line %line%: Field "%field%" contains non-numeric value "%value%". Please enter a valid number. - - - - - project.bom_import.validation.info.import_summary - Import summary: %total% total entries, %valid% valid, %invalid% with issues. - - - - - project.bom_import.validation.errors.summary - Found %count% validation error(s) that must be fixed before import can proceed. - - - - - project.bom_import.validation.warnings.summary - Found %count% warning(s). Please review these issues before proceeding. - - - - - project.bom_import.validation.info.all_valid - All entries passed validation successfully! - - - - - project.bom_import.validation.summary - Validation Summary - - - - - project.bom_import.validation.total_entries - Total Entries - - - - - project.bom_import.validation.valid_entries - Valid Entries - - - - - project.bom_import.validation.invalid_entries - Invalid Entries - - - - - project.bom_import.validation.success_rate - Success Rate - - - - - project.bom_import.validation.errors.title - Validation Errors - - - - - project.bom_import.validation.errors.description - The following errors must be fixed before the import can proceed: - - - - - project.bom_import.validation.warnings.title - Validation Warnings - - - - - project.bom_import.validation.warnings.description - The following warnings should be reviewed before proceeding: - - - - - project.bom_import.validation.info.title - Information - - - - - project.bom_import.validation.details.title - Detailed Validation Results - - - - - project.bom_import.validation.details.line - Line - - - - - project.bom_import.validation.details.status - Status - - - - - project.bom_import.validation.details.messages - Messages - - - - - project.bom_import.validation.details.valid - Valid - - - - - project.bom_import.validation.details.invalid - Invalid - - - - - project.bom_import.validation.all_valid - All entries are valid and ready for import! - - - - - project.bom_import.validation.fix_errors - Please fix the validation errors before proceeding with the import. - - + + + project.bom_import.map_fields + Map Fields + + + + + project.bom_import.map_fields.help + Configure how CSV columns map to BOM fields + + + + + project.bom_import.delimiter + Delimiter + + + + + project.bom_import.delimiter.comma + Comma (,) + + + + + project.bom_import.delimiter.semicolon + Semicolon (;) + + + + + project.bom_import.delimiter.tab + Tab + + + + + project.bom_import.field_mapping.title + Field Mapping + + + + + project.bom_import.field_mapping.csv_field + CSV Field + + + + + project.bom_import.field_mapping.maps_to + Maps To + + + + + project.bom_import.field_mapping.suggestion + Suggestion + + + + + project.bom_import.field_mapping.priority + Priority + + + + + project.bom_import.field_mapping.priority_help + Priority (lower number = higher priority) + + + + + project.bom_import.field_mapping.priority_short + P + + + + + project.bom_import.field_mapping.priority_note + Priority Tip: Lower numbers = higher priority. Default priority is 10. Use priorities 1-9 for most important fields, 10+ for normal priority. + + + + + project.bom_import.field_mapping.summary + Field Mapping Summary + + + + + project.bom_import.field_mapping.select_to_see_summary + Select field mappings to see summary + + + + + project.bom_import.field_mapping.no_suggestion + No suggestion + + + + + project.bom_import.preview + Preview + + + + + project.bom_import.flash.session_expired + Import session has expired. Please upload your file again. + + + + + project.bom_import.field_mapping.ignore + Ignore + + + + + project.bom_import.type.kicad_schematic + KiCAD Schematic BOM (CSV file) + + + + + common.back + Back + + + + + project.bom_import.validation.errors.required_field_missing + Line %line%: Required field "%field%" is missing or empty. Please ensure this field is mapped and contains data. + + + + + project.bom_import.validation.errors.no_valid_designators + Line %line%: Designator field contains no valid component references. Expected format: "R1,C2,U3" or "R1, C2, U3". + + + + + project.bom_import.validation.warnings.unusual_designator_format + Line %line%: Some component references may have unusual format: %designators%. Expected format: "R1", "C2", "U3", etc. + + + + + project.bom_import.validation.errors.duplicate_designators + Line %line%: Duplicate component references found: %designators%. Each component should be referenced only once per line. + + + + + project.bom_import.validation.errors.invalid_quantity + Line %line%: Quantity "%quantity%" is not a valid number. Please enter a numeric value (e.g., 1, 2.5, 10). + + + + + project.bom_import.validation.errors.quantity_zero_or_negative + Line %line%: Quantity must be greater than 0, got %quantity%. + + + + + project.bom_import.validation.warnings.quantity_unusually_high + Line %line%: Quantity %quantity% seems unusually high. Please verify this is correct. + + + + + project.bom_import.validation.warnings.quantity_not_whole_number + Line %line%: Quantity %quantity% is not a whole number, but you have %count% component references. This may indicate a mismatch. + + + + + project.bom_import.validation.errors.quantity_designator_mismatch + Line %line%: Mismatch between quantity and component references. Quantity: %quantity%, References: %count% (%designators%). These should match. Either adjust the quantity or check your component references. + + + + + project.bom_import.validation.errors.invalid_partdb_id + Line %line%: Part-DB ID "%id%" is not a valid number. Please enter a numeric ID. + + + + + project.bom_import.validation.errors.partdb_id_zero_or_negative + Line %line%: Part-DB ID must be greater than 0, got %id%. + + + + + project.bom_import.validation.warnings.partdb_id_not_found + Line %line%: Part-DB ID %id% not found in database. The component will be imported without linking to an existing part. + + + + + project.bom_import.validation.info.partdb_link_success + Line %line%: Successfully linked to Part-DB part "%name%" (ID: %id%). + + + + + project.bom_import.validation.warnings.no_component_name + Line %line%: No component name/designation provided (MPN, Designation, or Value). Component will be named "Unknown Component". + + + + + project.bom_import.validation.warnings.package_name_too_long + Line %line%: Package name "%package%" is unusually long. Please verify this is correct. + + + + + project.bom_import.validation.info.library_prefix_detected + Line %line%: Package "%package%" contains library prefix. This will be automatically removed during import. + + + + + project.bom_import.validation.errors.non_numeric_field + Line %line%: Field "%field%" contains non-numeric value "%value%". Please enter a valid number. + + + + + project.bom_import.validation.info.import_summary + Import summary: %total% total entries, %valid% valid, %invalid% with issues. + + + + + project.bom_import.validation.errors.summary + Found %count% validation error(s) that must be fixed before import can proceed. + + + + + project.bom_import.validation.warnings.summary + Found %count% warning(s). Please review these issues before proceeding. + + + + + project.bom_import.validation.info.all_valid + All entries passed validation successfully! + + + + + project.bom_import.validation.summary + Validation Summary + + + + + project.bom_import.validation.total_entries + Total Entries + + + + + project.bom_import.validation.valid_entries + Valid Entries + + + + + project.bom_import.validation.invalid_entries + Invalid Entries + + + + + project.bom_import.validation.success_rate + Success Rate + + + + + project.bom_import.validation.errors.title + Validation Errors + + + + + project.bom_import.validation.errors.description + The following errors must be fixed before the import can proceed: + + + + + project.bom_import.validation.warnings.title + Validation Warnings + + + + + project.bom_import.validation.warnings.description + The following warnings should be reviewed before proceeding: + + + + + project.bom_import.validation.info.title + Information + + + + + project.bom_import.validation.details.title + Detailed Validation Results + + + + + project.bom_import.validation.details.line + Line + + + + + project.bom_import.validation.details.status + Status + + + + + project.bom_import.validation.details.messages + Messages + + + + + project.bom_import.validation.details.valid + Valid + + + + + project.bom_import.validation.details.invalid + Invalid + + + + + project.bom_import.validation.all_valid + All entries are valid and ready for import! + + + + + project.bom_import.validation.fix_errors + Please fix the validation errors before proceeding with the import. + + + + + project.bom_import.type.generic_csv + Generic CSV + + From 90f83273da8a02af42b5979566b22d8e7f606198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:24:32 +0200 Subject: [PATCH 07/10] Added nonce to scripts to ensure that they are working with enabled CSP --- .../projects/import_bom_map_fields.html.twig | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/templates/projects/import_bom_map_fields.html.twig b/templates/projects/import_bom_map_fields.html.twig index ba10c9c5..4e45eb08 100644 --- a/templates/projects/import_bom_map_fields.html.twig +++ b/templates/projects/import_bom_map_fields.html.twig @@ -15,7 +15,7 @@ show_details: false } %} {% endif %} - +
@@ -30,7 +30,7 @@
{{ form_start(form) }} - +
{{ form_row(form.delimiter) }} @@ -83,10 +83,10 @@ {% endif %} - @@ -96,7 +96,7 @@
- +
{% trans %}project.bom_import.field_mapping.summary{% endtrans %}:
@@ -121,12 +121,12 @@ {{ form_end(form) }} - -{% endblock %} \ No newline at end of file +{% endblock %} From 2b28aa8ba9f15a8b6206173b45e0cc2d2b03060f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:29:19 +0200 Subject: [PATCH 08/10] Enable CSP also in debug mode, as otherwise it complains about missing nonce function --- config/packages/nelmio_security.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config/packages/nelmio_security.yaml b/config/packages/nelmio_security.yaml index 1cb74da7..c283cd8e 100644 --- a/config/packages/nelmio_security.yaml +++ b/config/packages/nelmio_security.yaml @@ -69,9 +69,3 @@ nelmio_security: - 'data:' block-all-mixed-content: true # defaults to false, blocks HTTP content over HTTPS transport # upgrade-insecure-requests: true # defaults to false, upgrades HTTP requests to HTTPS transport - -when@dev: - # disables the Content-Security-Policy header - nelmio_security: - csp: - enabled: false \ No newline at end of file From fb92db8c051f08785e97f05f54fd762b12f9857e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:32:08 +0200 Subject: [PATCH 09/10] Use body element as dropdownParent for tomselect elements This improves UX --- .../controllers/elements/attachment_autocomplete_controller.js | 1 + assets/controllers/elements/part_select_controller.js | 3 ++- assets/controllers/elements/select_controller.js | 3 ++- assets/controllers/elements/select_multiple_controller.js | 3 ++- .../elements/static_file_autocomplete_controller.js | 1 + .../elements/structural_entity_select_controller.js | 1 + assets/controllers/elements/tagsinput_controller.js | 3 ++- 7 files changed, 11 insertions(+), 4 deletions(-) diff --git a/assets/controllers/elements/attachment_autocomplete_controller.js b/assets/controllers/elements/attachment_autocomplete_controller.js index f8bc301e..0175b284 100644 --- a/assets/controllers/elements/attachment_autocomplete_controller.js +++ b/assets/controllers/elements/attachment_autocomplete_controller.js @@ -42,6 +42,7 @@ export default class extends Controller { selectOnTab: true, //This a an ugly solution to disable the delimiter parsing of the TomSelect plugin delimiter: 'VERY_L0NG_D€LIMITER_WHICH_WILL_NEVER_BE_ENCOUNTERED_IN_A_STRING', + dropdownParent: 'body', render: { item: (data, escape) => { return '' + escape(data.label) + ''; diff --git a/assets/controllers/elements/part_select_controller.js b/assets/controllers/elements/part_select_controller.js index 5abd5ba3..0658f4b4 100644 --- a/assets/controllers/elements/part_select_controller.js +++ b/assets/controllers/elements/part_select_controller.js @@ -16,6 +16,7 @@ export default class extends Controller { searchField: ["name", "description", "category", "footprint"], valueField: "id", labelField: "name", + dropdownParent: 'body', preload: "focus", render: { item: (data, escape) => { @@ -71,4 +72,4 @@ export default class extends Controller { //Destroy the TomSelect instance this._tomSelect.destroy(); } -} \ No newline at end of file +} diff --git a/assets/controllers/elements/select_controller.js b/assets/controllers/elements/select_controller.js index cdafe4d0..f933731a 100644 --- a/assets/controllers/elements/select_controller.js +++ b/assets/controllers/elements/select_controller.js @@ -44,6 +44,7 @@ export default class extends Controller { allowEmptyOption: true, selectOnTab: true, maxOptions: null, + dropdownParent: 'body', render: { item: this.renderItem.bind(this), @@ -108,4 +109,4 @@ export default class extends Controller { //Destroy the TomSelect instance this._tomSelect.destroy(); } -} \ No newline at end of file +} diff --git a/assets/controllers/elements/select_multiple_controller.js b/assets/controllers/elements/select_multiple_controller.js index df37871d..daa6b0a1 100644 --- a/assets/controllers/elements/select_multiple_controller.js +++ b/assets/controllers/elements/select_multiple_controller.js @@ -29,6 +29,7 @@ export default class extends Controller { this._tomSelect = new TomSelect(this.element, { maxItems: 1000, allowEmptyOption: true, + dropdownParent: 'body', plugins: ['remove_button'], }); } @@ -39,4 +40,4 @@ export default class extends Controller { this._tomSelect.destroy(); } -} \ No newline at end of file +} diff --git a/assets/controllers/elements/static_file_autocomplete_controller.js b/assets/controllers/elements/static_file_autocomplete_controller.js index 31ca0314..0421a26d 100644 --- a/assets/controllers/elements/static_file_autocomplete_controller.js +++ b/assets/controllers/elements/static_file_autocomplete_controller.js @@ -50,6 +50,7 @@ export default class extends Controller { valueField: 'text', searchField: 'text', orderField: 'text', + dropdownParent: 'body', //This a an ugly solution to disable the delimiter parsing of the TomSelect plugin delimiter: 'VERY_L0NG_D€LIMITER_WHICH_WILL_NEVER_BE_ENCOUNTERED_IN_A_STRING', diff --git a/assets/controllers/elements/structural_entity_select_controller.js b/assets/controllers/elements/structural_entity_select_controller.js index a1114a97..5c6f9490 100644 --- a/assets/controllers/elements/structural_entity_select_controller.js +++ b/assets/controllers/elements/structural_entity_select_controller.js @@ -54,6 +54,7 @@ export default class extends Controller { maxItems: 1, delimiter: "$$VERY_LONG_DELIMITER_THAT_SHOULD_NEVER_APPEAR$$", splitOn: null, + dropdownParent: 'body', searchField: [ {field: "text", weight : 2}, diff --git a/assets/controllers/elements/tagsinput_controller.js b/assets/controllers/elements/tagsinput_controller.js index 1f10c457..53bf7608 100644 --- a/assets/controllers/elements/tagsinput_controller.js +++ b/assets/controllers/elements/tagsinput_controller.js @@ -43,6 +43,7 @@ export default class extends Controller { selectOnTab: true, createOnBlur: true, create: true, + dropdownParent: 'body', }; if(this.element.dataset.autocomplete) { @@ -73,4 +74,4 @@ export default class extends Controller { //Destroy the TomSelect instance this._tomSelect.destroy(); } -} \ No newline at end of file +} From 5a5691a8c4142d8c4d849d424cc37212503a4cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Sat, 6 Sep 2025 23:34:47 +0200 Subject: [PATCH 10/10] Added documentation about the new BOM file types --- docs/usage/bom_import.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/usage/bom_import.md b/docs/usage/bom_import.md index 94a06d55..b4bcb2be 100644 --- a/docs/usage/bom_import.md +++ b/docs/usage/bom_import.md @@ -34,3 +34,12 @@ select the BOM file you want to import and some options for the import process: has a different format and does not work with this type. You can generate this BOM file by going to "File" -> "Fabrication Outputs" -> "Bill of Materials" in Pcbnew and save the file to your desired location. +* **KiCAD Schematic BOM (CSV file)**: A CSV file of the Bill of Material (BOM) generated + by [KiCAD Eeschema](https://www.kicad.org/). + You can generate this BOM file by going to "Tools" -> "Generate Bill of Materials" in Eeschema and save the file to your + desired location. In the next step you can customize the mapping of the fields in Part-DB, if you have any special fields + in your BOM to locate your fields correctly. +* **Generic CSV file**: A generic CSV file. You can use this option if you use some different ECAD software or wanna create + your own CSV file. You will need to specify at least the designators, quantity and value fields in the CSV. In the next + step you can customize the mapping of the fields in Part-DB, if you have any special fields in your BOM to locate your + parts correctly.