mirror of
https://github.com/Part-DB/Part-DB-server.git
synced 2025-12-09 12:39:30 +00:00
Implement functionality to import schematic csv (or any other csv for that matter), with ability to map input columns to output columns with input validation and error handling
This commit is contained in:
barisgit
Jan Böhmer
parent
4277f42285
commit
d0f2422e0d
6 changed files with 1733 additions and 28 deletions
|
|
@ -22,10 +22,13 @@ declare(strict_types=1);
|
|||
*/
|
||||
namespace App\Services\ImportExportSystem;
|
||||
|
||||
use App\Entity\Parts\Part;
|
||||
use App\Entity\ProjectSystem\Project;
|
||||
use App\Entity\ProjectSystem\ProjectBOMEntry;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use InvalidArgumentException;
|
||||
use League\Csv\Reader;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\HttpFoundation\File\File;
|
||||
use Symfony\Component\OptionsResolver\OptionsResolver;
|
||||
|
||||
|
|
@ -44,14 +47,25 @@ class BOMImporter
|
|||
5 => 'Supplier and ref',
|
||||
];
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
public function __construct(
|
||||
private readonly EntityManagerInterface $entityManager,
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly BOMValidationService $validationService
|
||||
) {
|
||||
}
|
||||
|
||||
protected function configureOptions(OptionsResolver $resolver): OptionsResolver
|
||||
{
|
||||
$resolver->setRequired('type');
|
||||
$resolver->setAllowedValues('type', ['kicad_pcbnew']);
|
||||
$resolver->setAllowedValues('type', ['kicad_pcbnew', 'kicad_schematic']);
|
||||
|
||||
// For flexible schematic import with field mapping
|
||||
$resolver->setDefined(['field_mapping', 'field_priorities', 'delimiter']);
|
||||
$resolver->setDefault('delimiter', ',');
|
||||
$resolver->setDefault('field_priorities', []);
|
||||
$resolver->setAllowedTypes('field_mapping', 'array');
|
||||
$resolver->setAllowedTypes('field_priorities', 'array');
|
||||
$resolver->setAllowedTypes('delimiter', 'string');
|
||||
|
||||
return $resolver;
|
||||
}
|
||||
|
|
@ -82,6 +96,23 @@ class BOMImporter
|
|||
return $this->stringToBOMEntries($file->getContent(), $options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate BOM data before importing
|
||||
* @return array Validation result with errors, warnings, and info
|
||||
*/
|
||||
public function validateBOMData(string $data, array $options): array
|
||||
{
|
||||
$resolver = new OptionsResolver();
|
||||
$resolver = $this->configureOptions($resolver);
|
||||
$options = $resolver->resolve($options);
|
||||
|
||||
return match ($options['type']) {
|
||||
'kicad_pcbnew' => $this->validateKiCADPCB($data),
|
||||
'kicad_schematic' => $this->validateKiCADSchematicData($data, $options),
|
||||
default => throw new InvalidArgumentException('Invalid import type!'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import string data into an array of BOM entries, which are not yet assigned to a project.
|
||||
* @param string $data The data to import
|
||||
|
|
@ -95,12 +126,13 @@ class BOMImporter
|
|||
$options = $resolver->resolve($options);
|
||||
|
||||
return match ($options['type']) {
|
||||
'kicad_pcbnew' => $this->parseKiCADPCB($data, $options),
|
||||
'kicad_pcbnew' => $this->parseKiCADPCB($data),
|
||||
'kicad_schematic' => $this->parseKiCADSchematic($data, $options),
|
||||
default => throw new InvalidArgumentException('Invalid import type!'),
|
||||
};
|
||||
}
|
||||
|
||||
private function parseKiCADPCB(string $data, array $options = []): array
|
||||
private function parseKiCADPCB(string $data): array
|
||||
{
|
||||
$csv = Reader::createFromString($data);
|
||||
$csv->setDelimiter(';');
|
||||
|
|
@ -113,17 +145,17 @@ class BOMImporter
|
|||
$entry = $this->normalizeColumnNames($entry);
|
||||
|
||||
//Ensure that the entry has all required fields
|
||||
if (!isset ($entry['Designator'])) {
|
||||
throw new \UnexpectedValueException('Designator missing at line '.($offset + 1).'!');
|
||||
if (!isset($entry['Designator'])) {
|
||||
throw new \UnexpectedValueException('Designator missing at line ' . ($offset + 1) . '!');
|
||||
}
|
||||
if (!isset ($entry['Package'])) {
|
||||
throw new \UnexpectedValueException('Package missing at line '.($offset + 1).'!');
|
||||
if (!isset($entry['Package'])) {
|
||||
throw new \UnexpectedValueException('Package missing at line ' . ($offset + 1) . '!');
|
||||
}
|
||||
if (!isset ($entry['Designation'])) {
|
||||
throw new \UnexpectedValueException('Designation missing at line '.($offset + 1).'!');
|
||||
if (!isset($entry['Designation'])) {
|
||||
throw new \UnexpectedValueException('Designation missing at line ' . ($offset + 1) . '!');
|
||||
}
|
||||
if (!isset ($entry['Quantity'])) {
|
||||
throw new \UnexpectedValueException('Quantity missing at line '.($offset + 1).'!');
|
||||
if (!isset($entry['Quantity'])) {
|
||||
throw new \UnexpectedValueException('Quantity missing at line ' . ($offset + 1) . '!');
|
||||
}
|
||||
|
||||
$bom_entry = new ProjectBOMEntry();
|
||||
|
|
@ -138,6 +170,63 @@ class BOMImporter
|
|||
return $bom_entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate KiCad PCB data
|
||||
*/
|
||||
private function validateKiCADPCB(string $data): array
|
||||
{
|
||||
$csv = Reader::createFromString($data);
|
||||
$csv->setDelimiter(';');
|
||||
$csv->setHeaderOffset(0);
|
||||
|
||||
$mapped_entries = [];
|
||||
|
||||
foreach ($csv->getRecords() as $offset => $entry) {
|
||||
// Translate the german field names to english
|
||||
$entry = $this->normalizeColumnNames($entry);
|
||||
$mapped_entries[] = $entry;
|
||||
}
|
||||
|
||||
return $this->validationService->validateBOMEntries($mapped_entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate KiCad schematic data
|
||||
*/
|
||||
private function validateKiCADSchematicData(string $data, array $options): array
|
||||
{
|
||||
$delimiter = $options['delimiter'] ?? ',';
|
||||
$field_mapping = $options['field_mapping'] ?? [];
|
||||
$field_priorities = $options['field_priorities'] ?? [];
|
||||
|
||||
// Handle potential BOM (Byte Order Mark) at the beginning
|
||||
$data = preg_replace('/^\xEF\xBB\xBF/', '', $data);
|
||||
|
||||
$csv = Reader::createFromString($data);
|
||||
$csv->setDelimiter($delimiter);
|
||||
$csv->setHeaderOffset(0);
|
||||
|
||||
// Handle quoted fields properly
|
||||
$csv->setEscape('\\');
|
||||
$csv->setEnclosure('"');
|
||||
|
||||
$mapped_entries = [];
|
||||
|
||||
foreach ($csv->getRecords() as $offset => $entry) {
|
||||
// Apply field mapping to translate column names
|
||||
$mapped_entry = $this->applyFieldMapping($entry, $field_mapping, $field_priorities);
|
||||
|
||||
// Extract footprint package name if it contains library prefix
|
||||
if (isset($mapped_entry['Package']) && str_contains($mapped_entry['Package'], ':')) {
|
||||
$mapped_entry['Package'] = explode(':', $mapped_entry['Package'], 2)[1];
|
||||
}
|
||||
|
||||
$mapped_entries[] = $mapped_entry;
|
||||
}
|
||||
|
||||
return $this->validationService->validateBOMEntries($mapped_entries, $options);
|
||||
}
|
||||
|
||||
/**
|
||||
* This function uses the order of the fields in the CSV files to make them locale independent.
|
||||
* @param array $entry
|
||||
|
|
@ -160,4 +249,482 @@ class BOMImporter
|
|||
|
||||
return $out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse KiCad schematic BOM with flexible field mapping
|
||||
*/
|
||||
private function parseKiCADSchematic(string $data, array $options = []): array
|
||||
{
|
||||
$delimiter = $options['delimiter'] ?? ',';
|
||||
$field_mapping = $options['field_mapping'] ?? [];
|
||||
$field_priorities = $options['field_priorities'] ?? [];
|
||||
|
||||
// Handle potential BOM (Byte Order Mark) at the beginning
|
||||
$data = preg_replace('/^\xEF\xBB\xBF/', '', $data);
|
||||
|
||||
$csv = Reader::createFromString($data);
|
||||
$csv->setDelimiter($delimiter);
|
||||
$csv->setHeaderOffset(0);
|
||||
|
||||
// Handle quoted fields properly
|
||||
$csv->setEscape('\\');
|
||||
$csv->setEnclosure('"');
|
||||
|
||||
$bom_entries = [];
|
||||
$entries_by_key = []; // Track entries by name+part combination
|
||||
$mapped_entries = []; // Collect all mapped entries for validation
|
||||
|
||||
foreach ($csv->getRecords() as $offset => $entry) {
|
||||
// Apply field mapping to translate column names
|
||||
$mapped_entry = $this->applyFieldMapping($entry, $field_mapping, $field_priorities);
|
||||
|
||||
// Extract footprint package name if it contains library prefix
|
||||
if (isset($mapped_entry['Package']) && str_contains($mapped_entry['Package'], ':')) {
|
||||
$mapped_entry['Package'] = explode(':', $mapped_entry['Package'], 2)[1];
|
||||
}
|
||||
|
||||
$mapped_entries[] = $mapped_entry;
|
||||
}
|
||||
|
||||
// Validate all entries before processing
|
||||
$validation_result = $this->validationService->validateBOMEntries($mapped_entries, $options);
|
||||
|
||||
// Log validation results
|
||||
$this->logger->info('BOM import validation completed', [
|
||||
'total_entries' => $validation_result['total_entries'],
|
||||
'valid_entries' => $validation_result['valid_entries'],
|
||||
'invalid_entries' => $validation_result['invalid_entries'],
|
||||
'error_count' => count($validation_result['errors']),
|
||||
'warning_count' => count($validation_result['warnings']),
|
||||
]);
|
||||
|
||||
// If there are validation errors, throw an exception with detailed messages
|
||||
if (!empty($validation_result['errors'])) {
|
||||
$error_message = $this->validationService->getErrorMessage($validation_result);
|
||||
throw new \UnexpectedValueException("BOM import validation failed:\n" . $error_message);
|
||||
}
|
||||
|
||||
// Process validated entries
|
||||
foreach ($mapped_entries as $offset => $mapped_entry) {
|
||||
|
||||
// Set name - prefer MPN, fall back to Value, then default format
|
||||
$mpn = trim($mapped_entry['MPN'] ?? '');
|
||||
$designation = trim($mapped_entry['Designation'] ?? '');
|
||||
$value = trim($mapped_entry['Value'] ?? '');
|
||||
|
||||
// Use the first non-empty value, or 'Unknown Component' if all are empty
|
||||
$name = '';
|
||||
if (!empty($mpn)) {
|
||||
$name = $mpn;
|
||||
} elseif (!empty($designation)) {
|
||||
$name = $designation;
|
||||
} elseif (!empty($value)) {
|
||||
$name = $value;
|
||||
} else {
|
||||
$name = 'Unknown Component';
|
||||
}
|
||||
|
||||
if (isset($mapped_entry['Package']) && !empty(trim($mapped_entry['Package']))) {
|
||||
$name .= ' (' . trim($mapped_entry['Package']) . ')';
|
||||
}
|
||||
|
||||
// Set mountnames and quantity
|
||||
// The Designator field contains comma-separated mount names for all instances
|
||||
$designator = trim($mapped_entry['Designator']);
|
||||
$quantity = (float) $mapped_entry['Quantity'];
|
||||
|
||||
// Get mountnames array (validation already ensured they match quantity)
|
||||
$mountnames_array = array_map('trim', explode(',', $designator));
|
||||
|
||||
// Try to link existing Part-DB part if ID is provided
|
||||
$part = null;
|
||||
if (isset($mapped_entry['Part-DB ID']) && !empty($mapped_entry['Part-DB ID'])) {
|
||||
$partDbId = (int) $mapped_entry['Part-DB ID'];
|
||||
$existingPart = $this->entityManager->getRepository(Part::class)->find($partDbId);
|
||||
|
||||
if ($existingPart) {
|
||||
$part = $existingPart;
|
||||
// Update name with actual part name
|
||||
$name = $existingPart->getName();
|
||||
}
|
||||
}
|
||||
|
||||
// Create unique key for this entry (name + part ID)
|
||||
$entry_key = $name . '|' . ($part ? $part->getID() : 'null');
|
||||
|
||||
// Check if we already have an entry with the same name and part
|
||||
if (isset($entries_by_key[$entry_key])) {
|
||||
// Merge with existing entry
|
||||
$existing_entry = $entries_by_key[$entry_key];
|
||||
|
||||
// Combine mountnames
|
||||
$existing_mountnames = $existing_entry->getMountnames();
|
||||
$combined_mountnames = $existing_mountnames . ',' . $designator;
|
||||
$existing_entry->setMountnames($combined_mountnames);
|
||||
|
||||
// Add quantities
|
||||
$existing_quantity = $existing_entry->getQuantity();
|
||||
$existing_entry->setQuantity($existing_quantity + $quantity);
|
||||
|
||||
$this->logger->info('Merged duplicate BOM entry', [
|
||||
'name' => $name,
|
||||
'part_id' => $part ? $part->getID() : null,
|
||||
'original_quantity' => $existing_quantity,
|
||||
'added_quantity' => $quantity,
|
||||
'new_quantity' => $existing_quantity + $quantity,
|
||||
'original_mountnames' => $existing_mountnames,
|
||||
'added_mountnames' => $designator,
|
||||
]);
|
||||
|
||||
continue; // Skip creating new entry
|
||||
}
|
||||
|
||||
// Create new BOM entry
|
||||
$bom_entry = new ProjectBOMEntry();
|
||||
$bom_entry->setName($name);
|
||||
$bom_entry->setMountnames($designator);
|
||||
$bom_entry->setQuantity($quantity);
|
||||
|
||||
if ($part) {
|
||||
$bom_entry->setPart($part);
|
||||
}
|
||||
|
||||
// Set comment with additional info
|
||||
$comment_parts = [];
|
||||
if (isset($mapped_entry['Value']) && $mapped_entry['Value'] !== ($mapped_entry['MPN'] ?? '')) {
|
||||
$comment_parts[] = 'Value: ' . $mapped_entry['Value'];
|
||||
}
|
||||
if (isset($mapped_entry['MPN'])) {
|
||||
$comment_parts[] = 'MPN: ' . $mapped_entry['MPN'];
|
||||
}
|
||||
if (isset($mapped_entry['Manufacturer'])) {
|
||||
$comment_parts[] = 'Manf: ' . $mapped_entry['Manufacturer'];
|
||||
}
|
||||
if (isset($mapped_entry['LCSC'])) {
|
||||
$comment_parts[] = 'LCSC: ' . $mapped_entry['LCSC'];
|
||||
}
|
||||
if (isset($mapped_entry['Supplier and ref'])) {
|
||||
$comment_parts[] = $mapped_entry['Supplier and ref'];
|
||||
}
|
||||
|
||||
if ($part) {
|
||||
$comment_parts[] = "Part-DB ID: " . $part->getID();
|
||||
} elseif (isset($mapped_entry['Part-DB ID']) && !empty($mapped_entry['Part-DB ID'])) {
|
||||
$comment_parts[] = "Part-DB ID: " . $mapped_entry['Part-DB ID'] . " (NOT FOUND)";
|
||||
}
|
||||
|
||||
$bom_entry->setComment(implode(', ', $comment_parts));
|
||||
|
||||
$bom_entries[] = $bom_entry;
|
||||
$entries_by_key[$entry_key] = $bom_entry;
|
||||
}
|
||||
|
||||
return $bom_entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available field mapping targets with descriptions
|
||||
*/
|
||||
public function getAvailableFieldTargets(): array
|
||||
{
|
||||
$targets = [
|
||||
'Designator' => [
|
||||
'label' => 'Designator',
|
||||
'description' => 'Component reference designators (e.g., R1, C2, U3)',
|
||||
'required' => true,
|
||||
'multiple' => false,
|
||||
],
|
||||
'Quantity' => [
|
||||
'label' => 'Quantity',
|
||||
'description' => 'Number of components',
|
||||
'required' => true,
|
||||
'multiple' => false,
|
||||
],
|
||||
'Designation' => [
|
||||
'label' => 'Designation',
|
||||
'description' => 'Component designation/part number',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
'Value' => [
|
||||
'label' => 'Value',
|
||||
'description' => 'Component value (e.g., 10k, 100nF)',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
'Package' => [
|
||||
'label' => 'Package',
|
||||
'description' => 'Component package/footprint',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
'MPN' => [
|
||||
'label' => 'MPN',
|
||||
'description' => 'Manufacturer Part Number',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
'Manufacturer' => [
|
||||
'label' => 'Manufacturer',
|
||||
'description' => 'Component manufacturer name',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
'Part-DB ID' => [
|
||||
'label' => 'Part-DB ID',
|
||||
'description' => 'Existing Part-DB part ID for linking',
|
||||
'required' => false,
|
||||
'multiple' => false,
|
||||
],
|
||||
'Comment' => [
|
||||
'label' => 'Comment',
|
||||
'description' => 'Additional component information',
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
],
|
||||
];
|
||||
|
||||
// Add dynamic supplier fields based on available suppliers in the database
|
||||
$suppliers = $this->entityManager->getRepository(\App\Entity\Parts\Supplier::class)->findAll();
|
||||
foreach ($suppliers as $supplier) {
|
||||
$supplierName = $supplier->getName();
|
||||
$targets[$supplierName . ' SPN'] = [
|
||||
'label' => $supplierName . ' SPN',
|
||||
'description' => "Supplier part number for {$supplierName}",
|
||||
'required' => false,
|
||||
'multiple' => true,
|
||||
'supplier_id' => $supplier->getID(),
|
||||
];
|
||||
}
|
||||
|
||||
return $targets;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get suggested field mappings based on common field names
|
||||
*/
|
||||
public function getSuggestedFieldMapping(array $detected_fields): array
|
||||
{
|
||||
$suggestions = [];
|
||||
|
||||
$field_patterns = [
|
||||
'Part-DB ID' => ['part-db id', 'partdb_id', 'part_db_id', 'db_id', 'partdb'],
|
||||
'Designator' => ['reference', 'ref', 'designator', 'component', 'comp'],
|
||||
'Quantity' => ['qty', 'quantity', 'count', 'number', 'amount'],
|
||||
'Value' => ['value', 'val', 'component_value'],
|
||||
'Designation' => ['designation', 'part_number', 'partnumber', 'part'],
|
||||
'Package' => ['footprint', 'package', 'housing', 'fp'],
|
||||
'MPN' => ['mpn', 'part_number', 'partnumber', 'manf#', 'mfr_part_number', 'manufacturer_part'],
|
||||
'Manufacturer' => ['manufacturer', 'manf', 'mfr', 'brand', 'vendor'],
|
||||
'Comment' => ['comment', 'comments', 'note', 'notes', 'description'],
|
||||
];
|
||||
|
||||
// Add supplier-specific patterns
|
||||
$suppliers = $this->entityManager->getRepository(\App\Entity\Parts\Supplier::class)->findAll();
|
||||
foreach ($suppliers as $supplier) {
|
||||
$supplierName = $supplier->getName();
|
||||
$supplierLower = strtolower($supplierName);
|
||||
|
||||
// Create patterns for each supplier
|
||||
$field_patterns[$supplierName . ' SPN'] = [
|
||||
$supplierLower,
|
||||
$supplierLower . '#',
|
||||
$supplierLower . '_part',
|
||||
$supplierLower . '_number',
|
||||
$supplierLower . 'pn',
|
||||
$supplierLower . '_spn',
|
||||
$supplierLower . ' spn',
|
||||
// Common abbreviations
|
||||
$supplierLower === 'mouser' ? 'mouser' : null,
|
||||
$supplierLower === 'digikey' ? 'dk' : null,
|
||||
$supplierLower === 'farnell' ? 'farnell' : null,
|
||||
$supplierLower === 'rs' ? 'rs' : null,
|
||||
$supplierLower === 'lcsc' ? 'lcsc' : null,
|
||||
];
|
||||
|
||||
// Remove null values
|
||||
$field_patterns[$supplierName . ' SPN'] = array_filter($field_patterns[$supplierName . ' SPN'], fn($value) => $value !== null);
|
||||
}
|
||||
|
||||
foreach ($detected_fields as $field) {
|
||||
$field_lower = strtolower(trim($field));
|
||||
|
||||
foreach ($field_patterns as $target => $patterns) {
|
||||
foreach ($patterns as $pattern) {
|
||||
if (str_contains($field_lower, $pattern)) {
|
||||
$suggestions[$field] = $target;
|
||||
break 2; // Break both loops
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $suggestions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate field mapping configuration
|
||||
*/
|
||||
public function validateFieldMapping(array $field_mapping, array $detected_fields): array
|
||||
{
|
||||
$errors = [];
|
||||
$warnings = [];
|
||||
$available_targets = $this->getAvailableFieldTargets();
|
||||
|
||||
// Check for required fields
|
||||
$mapped_targets = array_values($field_mapping);
|
||||
$required_fields = ['Designator', 'Quantity'];
|
||||
|
||||
foreach ($required_fields as $required) {
|
||||
if (!in_array($required, $mapped_targets, true)) {
|
||||
$errors[] = "Required field '{$required}' is not mapped from any CSV column.";
|
||||
}
|
||||
}
|
||||
|
||||
// Check for invalid target fields
|
||||
foreach ($field_mapping as $csv_field => $target) {
|
||||
if (!empty($target) && !isset($available_targets[$target])) {
|
||||
$errors[] = "Invalid target field '{$target}' for CSV field '{$csv_field}'.";
|
||||
}
|
||||
}
|
||||
|
||||
// Check for unmapped fields (warnings)
|
||||
$unmapped_fields = array_diff($detected_fields, array_keys($field_mapping));
|
||||
if (!empty($unmapped_fields)) {
|
||||
$warnings[] = "The following CSV fields are not mapped: " . implode(', ', $unmapped_fields);
|
||||
}
|
||||
|
||||
return [
|
||||
'errors' => $errors,
|
||||
'warnings' => $warnings,
|
||||
'is_valid' => empty($errors),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply field mapping with support for multiple fields and priority
|
||||
*/
|
||||
private function applyFieldMapping(array $entry, array $field_mapping, array $field_priorities = []): array
|
||||
{
|
||||
$mapped = [];
|
||||
$field_groups = [];
|
||||
|
||||
// Group fields by target with priority information
|
||||
foreach ($field_mapping as $csv_field => $target) {
|
||||
if (!empty($target)) {
|
||||
if (!isset($field_groups[$target])) {
|
||||
$field_groups[$target] = [];
|
||||
}
|
||||
$priority = $field_priorities[$csv_field] ?? 10;
|
||||
$field_groups[$target][] = [
|
||||
'field' => $csv_field,
|
||||
'priority' => $priority,
|
||||
'value' => $entry[$csv_field] ?? ''
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Process each target field
|
||||
foreach ($field_groups as $target => $field_data) {
|
||||
// Sort by priority (lower number = higher priority)
|
||||
usort($field_data, function ($a, $b) {
|
||||
return $a['priority'] <=> $b['priority'];
|
||||
});
|
||||
|
||||
$values = [];
|
||||
$non_empty_values = [];
|
||||
|
||||
// Collect all non-empty values for this target
|
||||
foreach ($field_data as $data) {
|
||||
$value = trim($data['value']);
|
||||
if (!empty($value)) {
|
||||
$non_empty_values[] = $value;
|
||||
}
|
||||
$values[] = $value;
|
||||
}
|
||||
|
||||
// Use the first non-empty value (highest priority)
|
||||
if (!empty($non_empty_values)) {
|
||||
$mapped[$target] = $non_empty_values[0];
|
||||
|
||||
// If multiple non-empty values exist, add alternatives to comment
|
||||
if (count($non_empty_values) > 1) {
|
||||
$mapped[$target . '_alternatives'] = array_slice($non_empty_values, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $mapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available fields in CSV data for field mapping UI
|
||||
*/
|
||||
public function detectFields(string $data, ?string $delimiter = null): array
|
||||
{
|
||||
if ($delimiter === null) {
|
||||
// Detect delimiter by counting occurrences in the first row (header)
|
||||
$delimiters = [',', ';', "\t"];
|
||||
$lines = explode("\n", $data, 2);
|
||||
$header_line = $lines[0] ?? '';
|
||||
$delimiter_counts = [];
|
||||
foreach ($delimiters as $delim) {
|
||||
$delimiter_counts[$delim] = substr_count($header_line, $delim);
|
||||
}
|
||||
// Choose the delimiter with the highest count, default to comma if all are zero
|
||||
$max_count = max($delimiter_counts);
|
||||
$delimiter = array_search($max_count, $delimiter_counts, true);
|
||||
if ($max_count === 0 || $delimiter === false) {
|
||||
$delimiter = ',';
|
||||
}
|
||||
}
|
||||
// Handle potential BOM (Byte Order Mark) at the beginning
|
||||
$data = preg_replace('/^\xEF\xBB\xBF/', '', $data);
|
||||
|
||||
// Get first line only for header detection
|
||||
$lines = explode("\n", $data);
|
||||
$header_line = trim($lines[0] ?? '');
|
||||
|
||||
|
||||
// Simple manual parsing for header detection
|
||||
// This handles quoted CSV fields better than the library for detection
|
||||
$fields = [];
|
||||
$current_field = '';
|
||||
$in_quotes = false;
|
||||
$quote_char = '"';
|
||||
|
||||
for ($i = 0; $i < strlen($header_line); $i++) {
|
||||
$char = $header_line[$i];
|
||||
|
||||
if ($char === $quote_char && !$in_quotes) {
|
||||
$in_quotes = true;
|
||||
} elseif ($char === $quote_char && $in_quotes) {
|
||||
// Check for escaped quote (double quote)
|
||||
if ($i + 1 < strlen($header_line) && $header_line[$i + 1] === $quote_char) {
|
||||
$current_field .= $quote_char;
|
||||
$i++; // Skip next quote
|
||||
} else {
|
||||
$in_quotes = false;
|
||||
}
|
||||
} elseif ($char === $delimiter && !$in_quotes) {
|
||||
$fields[] = trim($current_field);
|
||||
$current_field = '';
|
||||
} else {
|
||||
$current_field .= $char;
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last field
|
||||
if ($current_field !== '') {
|
||||
$fields[] = trim($current_field);
|
||||
}
|
||||
|
||||
// Clean up headers - remove quotes and trim whitespace
|
||||
$headers = array_map(function ($header) {
|
||||
return trim($header, '"\'');
|
||||
}, $fields);
|
||||
|
||||
|
||||
return array_values($headers);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
476
src/Services/ImportExportSystem/BOMValidationService.php
Normal file
476
src/Services/ImportExportSystem/BOMValidationService.php
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
|
||||
*
|
||||
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
namespace App\Services\ImportExportSystem;
|
||||
|
||||
use App\Entity\Parts\Part;
|
||||
use App\Entity\ProjectSystem\ProjectBOMEntry;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use Symfony\Contracts\Translation\TranslatorInterface;
|
||||
|
||||
/**
|
||||
* Service for validating BOM import data with comprehensive validation rules
|
||||
* and user-friendly error messages.
|
||||
*/
|
||||
class BOMValidationService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly EntityManagerInterface $entityManager,
|
||||
private readonly TranslatorInterface $translator
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation result structure
|
||||
*/
|
||||
public static function createValidationResult(): array
|
||||
{
|
||||
return [
|
||||
'errors' => [],
|
||||
'warnings' => [],
|
||||
'info' => [],
|
||||
'is_valid' => true,
|
||||
'total_entries' => 0,
|
||||
'valid_entries' => 0,
|
||||
'invalid_entries' => 0,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a single BOM entry with comprehensive checks
|
||||
*/
|
||||
public function validateBOMEntry(array $mapped_entry, int $line_number, array $options = []): array
|
||||
{
|
||||
$result = [
|
||||
'line_number' => $line_number,
|
||||
'errors' => [],
|
||||
'warnings' => [],
|
||||
'info' => [],
|
||||
'is_valid' => true,
|
||||
];
|
||||
|
||||
// Run all validation rules
|
||||
$this->validateRequiredFields($mapped_entry, $result);
|
||||
$this->validateDesignatorFormat($mapped_entry, $result);
|
||||
$this->validateQuantityFormat($mapped_entry, $result);
|
||||
$this->validateDesignatorQuantityMatch($mapped_entry, $result);
|
||||
$this->validatePartDBLink($mapped_entry, $result);
|
||||
$this->validateComponentName($mapped_entry, $result);
|
||||
$this->validatePackageFormat($mapped_entry, $result);
|
||||
$this->validateNumericFields($mapped_entry, $result);
|
||||
|
||||
$result['is_valid'] = empty($result['errors']);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate multiple BOM entries and provide summary
|
||||
*/
|
||||
public function validateBOMEntries(array $mapped_entries, array $options = []): array
|
||||
{
|
||||
$result = self::createValidationResult();
|
||||
$result['total_entries'] = count($mapped_entries);
|
||||
|
||||
$line_results = [];
|
||||
$all_errors = [];
|
||||
$all_warnings = [];
|
||||
$all_info = [];
|
||||
|
||||
foreach ($mapped_entries as $index => $entry) {
|
||||
$line_number = $index + 1;
|
||||
$line_result = $this->validateBOMEntry($entry, $line_number, $options);
|
||||
|
||||
$line_results[] = $line_result;
|
||||
|
||||
if ($line_result['is_valid']) {
|
||||
$result['valid_entries']++;
|
||||
} else {
|
||||
$result['invalid_entries']++;
|
||||
}
|
||||
|
||||
// Collect all messages
|
||||
$all_errors = array_merge($all_errors, $line_result['errors']);
|
||||
$all_warnings = array_merge($all_warnings, $line_result['warnings']);
|
||||
$all_info = array_merge($all_info, $line_result['info']);
|
||||
}
|
||||
|
||||
// Add summary messages
|
||||
$this->addSummaryMessages($result, $all_errors, $all_warnings, $all_info);
|
||||
|
||||
$result['errors'] = $all_errors;
|
||||
$result['warnings'] = $all_warnings;
|
||||
$result['info'] = $all_info;
|
||||
$result['line_results'] = $line_results;
|
||||
$result['is_valid'] = empty($all_errors);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate required fields are present
|
||||
*/
|
||||
private function validateRequiredFields(array $entry, array &$result): void
|
||||
{
|
||||
$required_fields = ['Designator', 'Quantity'];
|
||||
|
||||
foreach ($required_fields as $field) {
|
||||
if (!isset($entry[$field]) || trim($entry[$field]) === '') {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.required_field_missing', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%field%' => $field
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate designator format and content
|
||||
*/
|
||||
private function validateDesignatorFormat(array $entry, array &$result): void
|
||||
{
|
||||
if (!isset($entry['Designator']) || trim($entry['Designator']) === '') {
|
||||
return; // Already handled by required fields validation
|
||||
}
|
||||
|
||||
$designator = trim($entry['Designator']);
|
||||
$mountnames = array_map('trim', explode(',', $designator));
|
||||
|
||||
// Remove empty entries
|
||||
$mountnames = array_filter($mountnames, fn($name) => !empty($name));
|
||||
|
||||
if (empty($mountnames)) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.no_valid_designators', [
|
||||
'%line%' => $result['line_number']
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate each mountname format (allow 1-2 uppercase letters, followed by 1+ digits)
|
||||
$invalid_mountnames = [];
|
||||
foreach ($mountnames as $mountname) {
|
||||
if (!preg_match('/^[A-Z]{1,2}[0-9]+$/', $mountname)) {
|
||||
$invalid_mountnames[] = $mountname;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($invalid_mountnames)) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.unusual_designator_format', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%designators%' => implode(', ', $invalid_mountnames)
|
||||
]);
|
||||
}
|
||||
|
||||
// Check for duplicate mountnames within the same line
|
||||
$duplicates = array_diff_assoc($mountnames, array_unique($mountnames));
|
||||
if (!empty($duplicates)) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.duplicate_designators', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%designators%' => implode(', ', array_unique($duplicates))
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate quantity format and value
|
||||
*/
|
||||
private function validateQuantityFormat(array $entry, array &$result): void
|
||||
{
|
||||
if (!isset($entry['Quantity']) || trim($entry['Quantity']) === '') {
|
||||
return; // Already handled by required fields validation
|
||||
}
|
||||
|
||||
$quantity_str = trim($entry['Quantity']);
|
||||
|
||||
// Check if it's a valid number
|
||||
if (!is_numeric($quantity_str)) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.invalid_quantity', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%quantity%' => $quantity_str
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
$quantity = (float) $quantity_str;
|
||||
|
||||
// Check for reasonable quantity values
|
||||
if ($quantity <= 0) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.quantity_zero_or_negative', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%quantity%' => $quantity_str
|
||||
]);
|
||||
} elseif ($quantity > 10000) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.quantity_unusually_high', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%quantity%' => $quantity_str
|
||||
]);
|
||||
}
|
||||
|
||||
// Check if quantity is a whole number when it should be
|
||||
if (isset($entry['Designator'])) {
|
||||
$designator = trim($entry['Designator']);
|
||||
$mountnames = array_map('trim', explode(',', $designator));
|
||||
$mountnames = array_filter($mountnames, fn($name) => !empty($name));
|
||||
|
||||
if (count($mountnames) > 0 && $quantity != (int) $quantity) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.quantity_not_whole_number', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%quantity%' => $quantity_str,
|
||||
'%count%' => count($mountnames)
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that designator count matches quantity
|
||||
*/
|
||||
private function validateDesignatorQuantityMatch(array $entry, array &$result): void
|
||||
{
|
||||
if (!isset($entry['Designator']) || !isset($entry['Quantity'])) {
|
||||
return; // Already handled by required fields validation
|
||||
}
|
||||
|
||||
$designator = trim($entry['Designator']);
|
||||
$quantity_str = trim($entry['Quantity']);
|
||||
|
||||
if (!is_numeric($quantity_str)) {
|
||||
return; // Already handled by quantity validation
|
||||
}
|
||||
|
||||
$mountnames = array_map('trim', explode(',', $designator));
|
||||
$mountnames = array_filter($mountnames, fn($name) => !empty($name));
|
||||
$mountnames_count = count($mountnames);
|
||||
$quantity = (float) $quantity_str;
|
||||
|
||||
if ($mountnames_count !== (int) $quantity) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.quantity_designator_mismatch', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%quantity%' => $quantity_str,
|
||||
'%count%' => $mountnames_count,
|
||||
'%designators%' => $designator
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate Part-DB ID link
|
||||
*/
|
||||
private function validatePartDBLink(array $entry, array &$result): void
|
||||
{
|
||||
if (!isset($entry['Part-DB ID']) || trim($entry['Part-DB ID']) === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$part_db_id = trim($entry['Part-DB ID']);
|
||||
|
||||
if (!is_numeric($part_db_id)) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.invalid_partdb_id', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%id%' => $part_db_id
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
$part_id = (int) $part_db_id;
|
||||
|
||||
if ($part_id <= 0) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.partdb_id_zero_or_negative', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%id%' => $part_id
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if part exists in database
|
||||
$existing_part = $this->entityManager->getRepository(Part::class)->find($part_id);
|
||||
if (!$existing_part) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.partdb_id_not_found', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%id%' => $part_id
|
||||
]);
|
||||
} else {
|
||||
$result['info'][] = $this->translator->trans('project.bom_import.validation.info.partdb_link_success', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%name%' => $existing_part->getName(),
|
||||
'%id%' => $part_id
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate component name/designation
|
||||
*/
|
||||
private function validateComponentName(array $entry, array &$result): void
|
||||
{
|
||||
$name_fields = ['MPN', 'Designation', 'Value'];
|
||||
$has_name = false;
|
||||
|
||||
foreach ($name_fields as $field) {
|
||||
if (isset($entry[$field]) && trim($entry[$field]) !== '') {
|
||||
$has_name = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$has_name) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.no_component_name', [
|
||||
'%line%' => $result['line_number']
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate package format
|
||||
*/
|
||||
private function validatePackageFormat(array $entry, array &$result): void
|
||||
{
|
||||
if (!isset($entry['Package']) || trim($entry['Package']) === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$package = trim($entry['Package']);
|
||||
|
||||
// Check for common package format issues
|
||||
if (strlen($package) > 100) {
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.package_name_too_long', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%package%' => $package
|
||||
]);
|
||||
}
|
||||
|
||||
// Check for library prefixes (KiCad format)
|
||||
if (str_contains($package, ':')) {
|
||||
$result['info'][] = $this->translator->trans('project.bom_import.validation.info.library_prefix_detected', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%package%' => $package
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate numeric fields
|
||||
*/
|
||||
private function validateNumericFields(array $entry, array &$result): void
|
||||
{
|
||||
$numeric_fields = ['Quantity', 'Part-DB ID'];
|
||||
|
||||
foreach ($numeric_fields as $field) {
|
||||
if (isset($entry[$field]) && trim($entry[$field]) !== '') {
|
||||
$value = trim($entry[$field]);
|
||||
if (!is_numeric($value)) {
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.non_numeric_field', [
|
||||
'%line%' => $result['line_number'],
|
||||
'%field%' => $field,
|
||||
'%value%' => $value
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add summary messages to validation result
|
||||
*/
|
||||
private function addSummaryMessages(array &$result, array $errors, array $warnings, array $info): void
|
||||
{
|
||||
$total_entries = $result['total_entries'];
|
||||
$valid_entries = $result['valid_entries'];
|
||||
$invalid_entries = $result['invalid_entries'];
|
||||
|
||||
// Add summary info
|
||||
if ($total_entries > 0) {
|
||||
$result['info'][] = $this->translator->trans('project.bom_import.validation.info.import_summary', [
|
||||
'%total%' => $total_entries,
|
||||
'%valid%' => $valid_entries,
|
||||
'%invalid%' => $invalid_entries
|
||||
]);
|
||||
}
|
||||
|
||||
// Add error summary
|
||||
if (!empty($errors)) {
|
||||
$error_count = count($errors);
|
||||
$result['errors'][] = $this->translator->trans('project.bom_import.validation.errors.summary', [
|
||||
'%count%' => $error_count
|
||||
]);
|
||||
}
|
||||
|
||||
// Add warning summary
|
||||
if (!empty($warnings)) {
|
||||
$warning_count = count($warnings);
|
||||
$result['warnings'][] = $this->translator->trans('project.bom_import.validation.warnings.summary', [
|
||||
'%count%' => $warning_count
|
||||
]);
|
||||
}
|
||||
|
||||
// Add success message if all entries are valid
|
||||
if ($total_entries > 0 && $invalid_entries === 0) {
|
||||
$result['info'][] = $this->translator->trans('project.bom_import.validation.info.all_valid');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user-friendly error message for a validation result
|
||||
*/
|
||||
public function getErrorMessage(array $validation_result): string
|
||||
{
|
||||
if ($validation_result['is_valid']) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$messages = [];
|
||||
|
||||
if (!empty($validation_result['errors'])) {
|
||||
$messages[] = 'Errors:';
|
||||
foreach ($validation_result['errors'] as $error) {
|
||||
$messages[] = '• ' . $error;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($validation_result['warnings'])) {
|
||||
$messages[] = 'Warnings:';
|
||||
foreach ($validation_result['warnings'] as $warning) {
|
||||
$messages[] = '• ' . $warning;
|
||||
}
|
||||
}
|
||||
|
||||
return implode("\n", $messages);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get validation statistics
|
||||
*/
|
||||
public function getValidationStats(array $validation_result): array
|
||||
{
|
||||
return [
|
||||
'total_entries' => $validation_result['total_entries'] ?? 0,
|
||||
'valid_entries' => $validation_result['valid_entries'] ?? 0,
|
||||
'invalid_entries' => $validation_result['invalid_entries'] ?? 0,
|
||||
'error_count' => count($validation_result['errors'] ?? []),
|
||||
'warning_count' => count($validation_result['warnings'] ?? []),
|
||||
'info_count' => count($validation_result['info'] ?? []),
|
||||
'success_rate' => $validation_result['total_entries'] > 0
|
||||
? round(($validation_result['valid_entries'] / $validation_result['total_entries']) * 100, 1)
|
||||
: 0,
|
||||
];
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue