groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-09 12:39:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch 'feature/batch-info-provider-import'

Browse source
This commit is contained in:
Jan Böhmer 2025-09-21 23:14:09 +02:00
commit ed1e51f694
80 changed files with 9789 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/Services/ElementTypeNameGenerator.php
View file

@ -22,13 +22,13 @@ declare(strict_types=1);
namespace App\Services;
use App\Entity\Attachments\AttachmentContainingDBElement;
use App\Entity\Attachments\Attachment;
use App\Entity\Attachments\AttachmentContainingDBElement;
use App\Entity\Attachments\AttachmentType;
use App\Entity\Base\AbstractDBElement;
use App\Entity\Contracts\NamedElementInterface;
use App\Entity\Parts\PartAssociation;
use App\Entity\ProjectSystem\Project;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJob;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use App\Entity\LabelSystem\LabelProfile;
use App\Entity\Parameters\AbstractParameter;
use App\Entity\Parts\Category;
@ -36,12 +36,14 @@ use App\Entity\Parts\Footprint;
use App\Entity\Parts\Manufacturer;
use App\Entity\Parts\MeasurementUnit;
use App\Entity\Parts\Part;
use App\Entity\Parts\PartAssociation;
use App\Entity\Parts\PartLot;
use App\Entity\Parts\StorageLocation;
use App\Entity\Parts\Supplier;
use App\Entity\PriceInformations\Currency;
use App\Entity\PriceInformations\Orderdetail;
use App\Entity\PriceInformations\Pricedetail;
use App\Entity\ProjectSystem\Project;
use App\Entity\ProjectSystem\ProjectBOMEntry;
use App\Entity\UserSystem\Group;
use App\Entity\UserSystem\User;
@ -79,6 +81,8 @@ class ElementTypeNameGenerator
AbstractParameter::class => $this->translator->trans('parameter.label'),
LabelProfile::class => $this->translator->trans('label_profile.label'),
PartAssociation::class => $this->translator->trans('part_association.label'),
BulkInfoProviderImportJob::class => $this->translator->trans('bulk_info_provider_import_job.label'),
BulkInfoProviderImportJobPart::class => $this->translator->trans('bulk_info_provider_import_job_part.label'),
];
}
@ -130,10 +134,10 @@ class ElementTypeNameGenerator
{
$type = $this->getLocalizedTypeLabel($entity);
if ($use_html) {
return '<i>'.$type.':</i> '.htmlspecialchars($entity->getName());
return '<i>' . $type . ':</i> ' . htmlspecialchars($entity->getName());
}
return $type.': '.$entity->getName();
return $type . ': ' . $entity->getName();
}

64
src/Services/EntityMergers/Mergers/PartMerger.php
View file

@ -100,7 +100,8 @@ class PartMerger implements EntityMergerInterface
return $target;
}
private function comparePartAssociations(PartAssociation $t, PartAssociation $o): bool {
private function comparePartAssociations(PartAssociation $t, PartAssociation $o): bool
{
//We compare the translation keys, as it contains info about the type and other type info
return $t->getOther() === $o->getOther()
&& $t->getTypeTranslationKey() === $o->getTypeTranslationKey();
@ -141,40 +142,39 @@ class PartMerger implements EntityMergerInterface
$owner->addAssociatedPartsAsOwner($clone);
}
// Merge orderdetails, considering same supplier+part number as duplicates
$this->mergeCollections($target, $other, 'orderdetails', function (Orderdetail $t, Orderdetail $o) {
//First check that the orderdetails infos are equal
$tmp = $t->getSupplier() === $o->getSupplier()
&& $t->getSupplierPartNr() === $o->getSupplierPartNr()
&& $t->getSupplierProductUrl(false) === $o->getSupplierProductUrl(false);
if (!$tmp) {
return false;
}
//Check if the pricedetails are equal
$t_pricedetails = $t->getPricedetails();
$o_pricedetails = $o->getPricedetails();
//Ensure that both pricedetails have the same length
if (count($t_pricedetails) !== count($o_pricedetails)) {
return false;
}
//Check if all pricedetails are equal
for ($n=0, $nMax = count($t_pricedetails); $n< $nMax; $n++) {
$t_price = $t_pricedetails->get($n);
$o_price = $o_pricedetails->get($n);
if (!$t_price->getPrice()->isEqualTo($o_price->getPrice())
|| $t_price->getCurrency() !== $o_price->getCurrency()
|| $t_price->getPriceRelatedQuantity() !== $o_price->getPriceRelatedQuantity()
|| $t_price->getMinDiscountQuantity() !== $o_price->getMinDiscountQuantity()
) {
return false;
// If supplier and part number match, merge the orderdetails
if ($t->getSupplier() === $o->getSupplier() && $t->getSupplierPartNr() === $o->getSupplierPartNr()) {
// Update URL if target doesn't have one
if (empty($t->getSupplierProductUrl(false)) && !empty($o->getSupplierProductUrl(false))) {
$t->setSupplierProductUrl($o->getSupplierProductUrl(false));
}
// Merge price details: add new ones, update empty ones, keep existing non-empty ones
foreach ($o->getPricedetails() as $otherPrice) {
$found = false;
foreach ($t->getPricedetails() as $targetPrice) {
if ($targetPrice->getMinDiscountQuantity() === $otherPrice->getMinDiscountQuantity()
&& $targetPrice->getCurrency() === $otherPrice->getCurrency()) {
// Only update price if the existing one is zero/empty (most logical)
if ($targetPrice->getPrice()->isZero()) {
$targetPrice->setPrice($otherPrice->getPrice());
$targetPrice->setPriceRelatedQuantity($otherPrice->getPriceRelatedQuantity());
}
$found = true;
break;
}
}
// Add completely new price tiers
if (!$found) {
$clonedPrice = clone $otherPrice;
$clonedPrice->setOrderdetail($t);
$t->addPricedetail($clonedPrice);
}
}
return true; // Consider them equal so the other one gets skipped
}
//If all pricedetails are equal, the orderdetails are equal
return true;
return false; // Different supplier/part number, add as new
});
//The pricedetails are not correctly assigned to the new orderdetails, so fix that
foreach ($target->getOrderdetails() as $orderdetail) {

110
src/Services/ImportExportSystem/EntityExporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use Symfony\Component\Serializer\SerializerInterface;
use function Symfony\Component\String\u;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use PhpOffice\PhpSpreadsheet\Writer\Xlsx;
use PhpOffice\PhpSpreadsheet\Writer\Xls;
/**
* Use this class to export an entity to multiple file formats.
@ -52,7 +55,7 @@ class EntityExporter
protected function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefault('format', 'csv');
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setDefault('csv_delimiter', ';');
$resolver->setAllowedTypes('csv_delimiter', 'string');
@ -88,28 +91,35 @@ class EntityExporter
$options = $resolver->resolve($options);
//Handle Excel formats by converting from CSV
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
return $this->exportToExcel($entities, $options);
}
//If include children is set, then we need to add the include_children group
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
return $this->serializer->serialize($entities, $options['format'],
return $this->serializer->serialize(
$entities,
$options['format'],
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'xml_root_node_name' => 'PartDBExport',
'partdb_export' => true,
//Skip the item normalizer, so that we dont get IRIs in the output
//Skip the item normalizer, so that we dont get IRIs in the output
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
//Handle circular references
//Handle circular references
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
}
private function handleCircularReference(object $object, string $format, array $context): string
private function handleCircularReference(object $object): string
{
if ($object instanceof AbstractStructuralDBElement) {
return $object->getFullPath("->");
@ -119,7 +129,75 @@ class EntityExporter
return $object->__toString();
}
throw new CircularReferenceException('Circular reference detected for object of type '.get_class($object));
throw new CircularReferenceException('Circular reference detected for object of type ' . get_class($object));
}
/**
* Exports entities to Excel format (xlsx or xls).
*
* @param AbstractNamedDBElement[] $entities The entities to export
* @param array $options The export options
*
* @return string The Excel file content as binary string
*/
protected function exportToExcel(array $entities, array $options): string
{
//First get CSV data using existing serializer
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
$csvData = $this->serializer->serialize(
$entities,
'csv',
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'partdb_export' => true,
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
//Convert CSV to Excel
$spreadsheet = new Spreadsheet();
$worksheet = $spreadsheet->getActiveSheet();
$rows = explode("\n", $csvData);
$rowIndex = 1;
foreach ($rows as $row) {
if (trim($row) === '') {
continue;
}
$columns = str_getcsv($row, $options['csv_delimiter'], '"', '\\');
$colIndex = 1;
foreach ($columns as $column) {
$cellCoordinate = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex) . $rowIndex;
$worksheet->setCellValue($cellCoordinate, $column);
$colIndex++;
}
$rowIndex++;
}
//Save to memory stream
$writer = $options['format'] === 'xlsx' ? new Xlsx($spreadsheet) : new Xls($spreadsheet);
$memFile = fopen("php://temp", 'r+b');
$writer->save($memFile);
rewind($memFile);
$content = stream_get_contents($memFile);
fclose($memFile);
if ($content === false) {
throw new \RuntimeException('Failed to read Excel content from memory stream.');
}
return $content;
}
/**
@ -156,19 +234,15 @@ class EntityExporter
//Determine the content type for the response
//Plain text should work for all types
$content_type = 'text/plain';
//Try to use better content types based on the format
$format = $options['format'];
switch ($format) {
case 'xml':
$content_type = 'application/xml';
break;
case 'json':
$content_type = 'application/json';
break;
}
$content_type = match ($format) {
'xml' => 'application/xml',
'json' => 'application/json',
'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'xls' => 'application/vnd.ms-excel',
default => 'text/plain',
};
$response->headers->set('Content-Type', $content_type);
//If view option is not specified, then download the file.
@ -186,7 +260,7 @@ class EntityExporter
$level = $options['level'];
$filename = 'export_'.$entity_name.'_'.$level.'.'.$format;
$filename = "export_{$entity_name}_{$level}.{$format}";
//Sanitize the filename
$filename = FilenameSanatizer::sanitizeFilename($filename);

139
src/Services/ImportExportSystem/EntityImporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\File\File;
use Symfony\Component\OptionsResolver\OptionsResolver;
use Symfony\Component\Serializer\SerializerInterface;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use Psr\Log\LoggerInterface;
/**
* @see \App\Tests\Services\ImportExportSystem\EntityImporterTest
@ -50,7 +53,7 @@ class EntityImporter
*/
private const ENCODINGS = ["ASCII", "UTF-8", "ISO-8859-1", "ISO-8859-15", "Windows-1252", "UTF-16", "UTF-32"];
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator)
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator, protected LoggerInterface $logger)
{
}
@ -102,7 +105,7 @@ class EntityImporter
foreach ($names as $name) {
//Count indentation level (whitespace characters at the beginning of the line)
$identSize = strlen($name)-strlen(ltrim($name));
$identSize = strlen($name) - strlen(ltrim($name));
//If the line is intended more than the last line, we have a new parent element
if ($identSize > end($indentations)) {
@ -195,16 +198,20 @@ class EntityImporter
}
//The [] behind class_name denotes that we expect an array.
$entities = $this->serializer->deserialize($data, $options['class'].'[]', $options['format'],
$entities = $this->serializer->deserialize(
$data,
$options['class'] . '[]',
$options['format'],
[
'groups' => $groups,
'csv_delimiter' => $options['csv_delimiter'],
'create_unknown_datastructures' => $options['create_unknown_datastructures'],
'path_delimiter' => $options['path_delimiter'],
'partdb_import' => true,
//Disable API Platform normalizer, as we don't want to use it here
//Disable API Platform normalizer, as we don't want to use it here
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
]);
]
);
//Ensure we have an array of entity elements.
if (!is_array($entities)) {
@ -279,7 +286,7 @@ class EntityImporter
'path_delimiter' => '->', //The delimiter used to separate the path elements in the name of a structural element
]);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setAllowedTypes('csv_delimiter', 'string');
$resolver->setAllowedTypes('preserve_children', 'bool');
$resolver->setAllowedTypes('class', 'string');
@ -335,6 +342,33 @@ class EntityImporter
*/
public function importFile(File $file, array $options = [], array &$errors = []): array
{
$resolver = new OptionsResolver();
$this->configureOptions($resolver);
$options = $resolver->resolve($options);
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
$this->logger->info('Converting Excel file to CSV', [
'filename' => $file->getFilename(),
'format' => $options['format'],
'delimiter' => $options['csv_delimiter']
]);
$csvData = $this->convertExcelToCsv($file, $options['csv_delimiter']);
$options['format'] = 'csv';
$this->logger->debug('Excel to CSV conversion completed', [
'csv_length' => strlen($csvData),
'csv_lines' => substr_count($csvData, "\n") + 1
]);
// Log the converted CSV for debugging (first 1000 characters)
$this->logger->debug('Converted CSV preview', [
'csv_preview' => substr($csvData, 0, 1000) . (strlen($csvData) > 1000 ? '...' : '')
]);
return $this->importString($csvData, $options, $errors);
}
return $this->importString($file->getContent(), $options, $errors);
}
@ -354,10 +388,103 @@ class EntityImporter
'xml' => 'xml',
'csv', 'tsv' => 'csv',
'yaml', 'yml' => 'yaml',
'xlsx' => 'xlsx',
'xls' => 'xls',
default => null,
};
}
/**
* Converts Excel file to CSV format using PhpSpreadsheet.
*
* @param File $file The Excel file to convert
* @param string $delimiter The CSV delimiter to use
*
* @return string The CSV data as string
*/
protected function convertExcelToCsv(File $file, string $delimiter = ';'): string
{
try {
$this->logger->debug('Loading Excel file', ['path' => $file->getPathname()]);
$spreadsheet = IOFactory::load($file->getPathname());
$worksheet = $spreadsheet->getActiveSheet();
$csvData = [];
$highestRow = $worksheet->getHighestRow();
$highestColumn = $worksheet->getHighestColumn();
$this->logger->debug('Excel file dimensions', [
'rows' => $highestRow,
'columns_detected' => $highestColumn,
'worksheet_title' => $worksheet->getTitle()
]);
$highestColumnIndex = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($highestColumn);
for ($row = 1; $row <= $highestRow; $row++) {
$rowData = [];
// Read all columns using numeric index
for ($colIndex = 1; $colIndex <= $highestColumnIndex; $colIndex++) {
$col = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex);
try {
$cellValue = $worksheet->getCell("{$col}{$row}")->getCalculatedValue();
$rowData[] = $cellValue ?? '';
} catch (\Exception $e) {
$this->logger->warning('Error reading cell value', [
'cell' => "{$col}{$row}",
'error' => $e->getMessage()
]);
$rowData[] = '';
}
}
$csvRow = implode($delimiter, array_map(function ($value) use ($delimiter) {
$value = (string) $value;
if (strpos($value, $delimiter) !== false || strpos($value, '"') !== false || strpos($value, "\n") !== false) {
return '"' . str_replace('"', '""', $value) . '"';
}
return $value;
}, $rowData));
$csvData[] = $csvRow;
// Log first few rows for debugging
if ($row <= 3) {
$this->logger->debug("Row {$row} converted", [
'original_data' => $rowData,
'csv_row' => $csvRow,
'first_cell_raw' => $worksheet->getCell("A{$row}")->getValue(),
'first_cell_calculated' => $worksheet->getCell("A{$row}")->getCalculatedValue()
]);
}
}
$result = implode("\n", $csvData);
$this->logger->info('Excel to CSV conversion successful', [
'total_rows' => count($csvData),
'total_characters' => strlen($result)
]);
$this->logger->debug('Full CSV data', [
'csv_data' => $result
]);
return $result;
} catch (\Exception $e) {
$this->logger->error('Failed to convert Excel to CSV', [
'file' => $file->getFilename(),
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
throw $e;
}
}
/**
* This functions corrects the parent setting based on the children value of the parent.
*

380
src/Services/InfoProviderSystem/BulkInfoProviderService.php Normal file
View file

@ -0,0 +1,380 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchFieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultsDTO;
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpClient\Exception\ClientException;
final class BulkInfoProviderService
{
/** @var array<string, Supplier|null> Cache for normalized supplier names */
private array $supplierCache = [];
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly ExistingPartFinder $existingPartFinder,
private readonly ProviderRegistry $providerRegistry,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
) {}
/**
* Perform bulk search across multiple parts and providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy
* @param bool $prefetchDetails Whether to prefetch detailed information for results
* @return BulkSearchResponseDTO Structured response containing all search results
* @throws \InvalidArgumentException If no valid parts provided
* @throws \RuntimeException If no search results found for any parts
*/
public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO
{
if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import');
}
$partResults = [];
$hasAnyResults = false;
// Group providers by batch capability
$batchProviders = [];
$regularProviders = [];
foreach ($fieldMappings as $mapping) {
foreach ($mapping->providers as $providerKey) {
if (!is_string($providerKey)) {
$this->logger->error('Invalid provider key type', [
'providerKey' => $providerKey,
'type' => gettype($providerKey)
]);
continue;
}
$provider = $this->providerRegistry->getProviderByKey($providerKey);
if ($provider instanceof BatchInfoProviderInterface) {
$batchProviders[$providerKey] = $provider;
} else {
$regularProviders[$providerKey] = $provider;
}
}
}
// Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders);
// Process regular providers
$regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults);
// Combine and format results for each part
foreach ($parts as $part) {
$searchResults = [];
// Get results from batch and regular processing
$allResults = array_merge(
$batchResults[$part->getId()] ?? [],
$regularResults[$part->getId()] ?? []
);
if (!empty($allResults)) {
$hasAnyResults = true;
$searchResults = $this->formatSearchResults($allResults);
}
$partResults[] = new BulkSearchPartResultsDTO(
part: $part,
searchResults: $searchResults,
errors: []
);
}
if (!$hasAnyResults) {
throw new \RuntimeException('No search results found for any of the selected parts');
}
$response = new BulkSearchResponseDTO($partResults);
// Prefetch details if requested
if ($prefetchDetails) {
$this->prefetchDetailsForResults($response);
}
return $response;
}
/**
* Process parts using batch-capable info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, BatchInfoProviderInterface> $batchProviders Batch providers indexed by key
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{
$batchResults = [];
foreach ($batchProviders as $providerKey => $provider) {
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
if (empty($keywords)) {
continue;
}
try {
$providerResults = $provider->searchByKeywordsBatch($keywords);
// Map results back to parts
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) {
$batchResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
}
}
}
} catch (\Exception $e) {
$this->logger->error('Batch search failed for provider ' . $providerKey, [
'error' => $e->getMessage(),
'provider' => $providerKey
]);
}
}
return $batchResults;
}
/**
* Process parts using regular (non-batch) info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, InfoProviderInterface> $regularProviders Regular providers indexed by key
* @param array<int, BulkSearchPartResultDTO[]> $excludeResults Results to exclude (from batch processing)
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{
$regularResults = [];
foreach ($parts as $part) {
$regularResults[$part->getId()] = [];
// Skip if we already have batch results for this part
if (!empty($excludeResults[$part->getId()] ?? [])) {
continue;
}
foreach ($fieldMappings as $mapping) {
$providers = array_intersect($mapping->providers, array_keys($regularProviders));
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if (!$keyword) {
continue;
}
try {
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
} catch (ClientException $e) {
$this->logger->error('Regular search failed', [
'part_id' => $part->getId(),
'field' => $mapping->field,
'error' => $e->getMessage()
]);
}
}
}
return $regularResults;
}
/**
* Collect unique keywords for a specific provider from all parts and field mappings.
*
* @param Part[] $parts Array of parts to collect keywords from
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param string $providerKey The provider key to collect keywords for
* @return string[] Array of unique keywords
*/
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{
$keywords = [];
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword;
}
}
}
return $keywords;
}
private function getKeywordFromField(Part $part, string $field): ?string
{
return match ($field) {
'mpn' => $part->getManufacturerProductNumber(),
'name' => $part->getName(),
default => $this->getSupplierPartNumber($part, $field)
};
}
private function getSupplierPartNumber(Part $part, string $field): ?string
{
if (!str_ends_with($field, '_spn')) {
return null;
}
$supplierKey = substr($field, 0, -4);
$supplier = $this->getSupplierByNormalizedName($supplierKey);
if (!$supplier) {
return null;
}
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null;
}
/**
* Get supplier by normalized name with caching to prevent N+1 queries.
*
* @param string $normalizedKey The normalized supplier key to search for
* @return Supplier|null The matching supplier or null if not found
*/
private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier
{
// Check cache first
if (isset($this->supplierCache[$normalizedKey])) {
return $this->supplierCache[$normalizedKey];
}
// Use efficient database query with PHP normalization
// Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization
if (empty($this->supplierCache)) {
$this->loadSuppliersIntoCache();
}
$supplier = $this->supplierCache[$normalizedKey] ?? null;
// Cache the result (including null results to prevent repeated queries)
$this->supplierCache[$normalizedKey] = $supplier;
return $supplier;
}
/**
* Load all suppliers into cache with normalized names to avoid N+1 queries.
*/
private function loadSuppliersIntoCache(): void
{
/** @var Supplier[] $suppliers */
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
$this->supplierCache[$normalizedName] = $supplier;
}
}
/**
* Format and deduplicate search results.
*
* @param BulkSearchPartResultDTO[] $bulkResults Array of bulk search results
* @return BulkSearchPartResultDTO[] Array of formatted search results with metadata
*/
private function formatSearchResults(array $bulkResults): array
{
// Sort by priority and remove duplicates
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
$uniqueResults = [];
$seenKeys = [];
foreach ($bulkResults as $result) {
$key = "{$result->searchResult->provider_key}|{$result->searchResult->provider_id}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueResults[] = $result;
}
}
return $uniqueResults;
}
/**
* Prefetch detailed information for search results.
*
* @param BulkSearchResponseDTO $searchResults Search results (supports both new DTO and legacy array format)
*/
public function prefetchDetailsForResults(BulkSearchResponseDTO $searchResults): void
{
$prefetchCount = 0;
// Handle both new DTO format and legacy array format for backwards compatibility
foreach ($searchResults->partResults as $partResult) {
foreach ($partResult->searchResults as $result) {
$dto = $result->searchResult;
try {
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
}
}
}
$this->logger->info("Prefetched details for {$prefetchCount} search results");
}
}

91
src/Services/InfoProviderSystem/DTOs/BulkSearchFieldMappingDTO.php Normal file
View file

@ -0,0 +1,91 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
/**
* Represents a mapping between a part field and the info providers that should search in that field.
*/
readonly class BulkSearchFieldMappingDTO
{
/**
* @param string $field The field to search in (e.g., 'mpn', 'name', or supplier-specific fields like 'digikey_spn')
* @param string[] $providers Array of provider keys to search with (e.g., ['digikey', 'farnell'])
* @param int $priority Priority for this field mapping (1-10, lower numbers = higher priority)
*/
public function __construct(
public string $field,
public array $providers,
public int $priority = 1
) {
if ($priority < 1 || $priority > 10) {
throw new \InvalidArgumentException('Priority must be between 1 and 10');
}
}
/**
* Create a FieldMappingDTO from legacy array format.
* @param array{field: string, providers: string[], priority?: int} $data
*/
public static function fromSerializableArray(array $data): self
{
return new self(
field: $data['field'],
providers: $data['providers'] ?? [],
priority: $data['priority'] ?? 1
);
}
/**
* Convert this DTO to the legacy array format for backwards compatibility.
* @return array{field: string, providers: string[], priority: int}
*/
public function toSerializableArray(): array
{
return [
'field' => $this->field,
'providers' => $this->providers,
'priority' => $this->priority,
];
}
/**
* Check if this field mapping is for a supplier part number field.
*/
public function isSupplierPartNumberField(): bool
{
return str_ends_with($this->field, '_spn');
}
/**
* Get the supplier key from a supplier part number field.
* Returns null if this is not a supplier part number field.
*/
public function getSupplierKey(): ?string
{
if (!$this->isSupplierPartNumberField()) {
return null;
}
return substr($this->field, 0, -4);
}
}

44
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultDTO.php Normal file
View file

@ -0,0 +1,44 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents a single search result from bulk search with additional context information, like how the part was found.
*/
readonly class BulkSearchPartResultDTO
{
public function __construct(
/** The base search result DTO containing provider data */
public SearchResultDTO $searchResult,
/** The field that was used to find this result */
public ?string $sourceField = null,
/** The actual keyword that was searched for */
public ?string $sourceKeyword = null,
/** Local part that matches this search result, if any */
public ?Part $localPart = null,
/** Priority for this search result */
public int $priority = 1
) {}
}

83
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultsDTO.php Normal file
View file

@ -0,0 +1,83 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents the search results for a single part from bulk info provider search.
* It contains multiple search results, that match the part.
*/
readonly class BulkSearchPartResultsDTO
{
/**
* @param Part $part The part that was searched for
* @param BulkSearchPartResultDTO[] $searchResults Array of search results found for this part
* @param string[] $errors Array of error messages encountered during search
*/
public function __construct(
public Part $part,
public array $searchResults = [],
public array $errors = []
) {}
/**
* Check if this part has any search results.
*/
public function hasResults(): bool
{
return !empty($this->searchResults);
}
/**
* Check if this part has any errors.
*/
public function hasErrors(): bool
{
return !empty($this->errors);
}
/**
* Get the number of search results for this part.
*/
public function getResultCount(): int
{
return count($this->searchResults);
}
public function getErrorCount(): int
{
return count($this->errors);
}
/**
* Get search results sorted by priority (ascending).
* @return BulkSearchPartResultDTO[]
*/
public function getResultsSortedByPriority(): array
{
$results = $this->searchResults;
usort($results, static fn(BulkSearchPartResultDTO $a, BulkSearchPartResultDTO $b) => $a->priority <=> $b->priority);
return $results;
}
}

231
src/Services/InfoProviderSystem/DTOs/BulkSearchResponseDTO.php Normal file
View file

@ -0,0 +1,231 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
use Doctrine\ORM\EntityManagerInterface;
use Traversable;
/**
* Represents the complete response from a bulk info provider search operation.
* It contains a list of PartSearchResultDTOs, one for each part searched.
*/
readonly class BulkSearchResponseDTO implements \ArrayAccess, \IteratorAggregate
{
/**
* @param BulkSearchPartResultsDTO[] $partResults Array of search results for each part
*/
public function __construct(
public array $partResults
) {}
/**
* Replaces the search results for a specific part, and returns a new instance.
* The part to replaced, is identified by the part property of the new_results parameter.
* The original instance remains unchanged.
* @param BulkSearchPartResultsDTO $new_results
* @return BulkSearchResponseDTO
*/
public function replaceResultsForPart(BulkSearchPartResultsDTO $new_results): self
{
$array = $this->partResults;
$replaced = false;
foreach ($array as $index => $partResult) {
if ($partResult->part === $new_results->part) {
$array[$index] = $new_results;
$replaced = true;
break;
}
}
if (!$replaced) {
throw new \InvalidArgumentException("Part not found in existing results.");
}
return new self($array);
}
/**
* Check if any parts have search results.
*/
public function hasAnyResults(): bool
{
foreach ($this->partResults as $partResult) {
if ($partResult->hasResults()) {
return true;
}
}
return false;
}
/**
* Get the total number of search results across all parts.
*/
public function getTotalResultCount(): int
{
$count = 0;
foreach ($this->partResults as $partResult) {
$count += $partResult->getResultCount();
}
return $count;
}
/**
* Get all parts that have search results.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithResults(): array
{
return array_filter($this->partResults, fn($result) => $result->hasResults());
}
/**
* Get all parts that have errors.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithErrors(): array
{
return array_filter($this->partResults, fn($result) => $result->hasErrors());
}
/**
* Get the number of parts processed.
*/
public function getPartCount(): int
{
return count($this->partResults);
}
/**
* Get the number of parts with successful results.
*/
public function getSuccessfulPartCount(): int
{
return count($this->getPartsWithResults());
}
/**
* Merge multiple BulkSearchResponseDTO instances into one.
* @param BulkSearchResponseDTO ...$responses
* @return BulkSearchResponseDTO
*/
public static function merge(BulkSearchResponseDTO ...$responses): BulkSearchResponseDTO
{
$mergedResults = [];
foreach ($responses as $response) {
foreach ($response->partResults as $partResult) {
$mergedResults[] = $partResult;
}
}
return new BulkSearchResponseDTO($mergedResults);
}
/**
* Convert this DTO to a serializable representation suitable for storage in the database
* @return array
*/
public function toSerializableRepresentation(): array
{
$serialized = [];
foreach ($this->partResults as $partResult) {
$partData = [
'part_id' => $partResult->part->getId(),
'search_results' => [],
'errors' => $partResult->errors ?? []
];
foreach ($partResult->searchResults as $result) {
$partData['search_results'][] = [
'dto' => $result->searchResult->toNormalizedSearchResultArray(),
'source_field' => $result->sourceField ?? null,
'source_keyword' => $result->sourceKeyword ?? null,
'localPart' => $result->localPart?->getId(),
'priority' => $result->priority
];
}
$serialized[] = $partData;
}
return $serialized;
}
/**
* Creates a BulkSearchResponseDTO from a serializable representation.
* @param array $data
* @param EntityManagerInterface $entityManager
* @return BulkSearchResponseDTO
* @throws \Doctrine\ORM\Exception\ORMException
*/
public static function fromSerializableRepresentation(array $data, EntityManagerInterface $entityManager): BulkSearchResponseDTO
{
$partResults = [];
foreach ($data as $partData) {
$partResults[] = new BulkSearchPartResultsDTO(
part: $entityManager->getReference(Part::class, $partData['part_id']),
searchResults: array_map(fn($result) => new BulkSearchPartResultDTO(
searchResult: SearchResultDTO::fromNormalizedSearchResultArray($result['dto']),
sourceField: $result['source_field'] ?? null,
sourceKeyword: $result['source_keyword'] ?? null,
localPart: isset($result['localPart']) ? $entityManager->getReference(Part::class, $result['localPart']) : null,
priority: $result['priority'] ?? null
), $partData['search_results'] ?? []),
errors: $partData['errors'] ?? []
);
}
return new BulkSearchResponseDTO($partResults);
}
public function offsetExists(mixed $offset): bool
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return isset($this->partResults[$offset]);
}
public function offsetGet(mixed $offset): ?BulkSearchPartResultsDTO
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return $this->partResults[$offset] ?? null;
}
public function offsetSet(mixed $offset, mixed $value): void
{
throw new \LogicException("BulkSearchResponseDTO is immutable.");
}
public function offsetUnset(mixed $offset): void
{
throw new \LogicException('BulkSearchResponseDTO is immutable.');
}
public function getIterator(): Traversable
{
return new \ArrayIterator($this->partResults);
}
}

8
src/Services/InfoProviderSystem/DTOs/FileDTO.php
View file

@ -28,12 +28,12 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a datasheet, a 3D model, a picture or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\FileDTOTest
*/
class FileDTO
readonly class FileDTO
{
/**
* @var string The URL where to get this file
*/
public readonly string $url;
public string $url;
/**
* @param string $url The URL where to get this file
@ -41,7 +41,7 @@ class FileDTO
*/
public function __construct(
string $url,
public readonly ?string $name = null,
public ?string $name = null,
) {
//Find all occurrences of non URL safe characters and replace them with their URL encoded version.
//We only want to replace characters which can not have a valid meaning in a URL (what would break the URL).
@ -50,4 +50,4 @@ class FileDTO
}
}
}

18
src/Services/InfoProviderSystem/DTOs/ParameterDTO.php
View file

@ -28,17 +28,17 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a voltage, a current, a temperature or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\ParameterDTOTest
*/
class ParameterDTO
readonly class ParameterDTO
{
public function __construct(
public readonly string $name,
public readonly ?string $value_text = null,
public readonly ?float $value_typ = null,
public readonly ?float $value_min = null,
public readonly ?float $value_max = null,
public readonly ?string $unit = null,
public readonly ?string $symbol = null,
public readonly ?string $group = null,
public string $name,
public ?string $value_text = null,
public ?float $value_typ = null,
public ?float $value_min = null,
public ?float $value_max = null,
public ?string $unit = null,
public ?string $symbol = null,
public ?string $group = null,
) {
}

2
src/Services/InfoProviderSystem/DTOs/PartDetailDTO.php
View file

@ -70,4 +70,4 @@ class PartDetailDTO extends SearchResultDTO
footprint: $footprint,
);
}
}
}

14
src/Services/InfoProviderSystem/DTOs/PriceDTO.php
View file

@ -28,21 +28,21 @@ use Brick\Math\BigDecimal;
/**
* This DTO represents a price for a single unit in a certain discount range
*/
class PriceDTO
readonly class PriceDTO
{
private readonly BigDecimal $price_as_big_decimal;
private BigDecimal $price_as_big_decimal;
public function __construct(
/** @var float The minimum amount that needs to get ordered for this price to be valid */
public readonly float $minimum_discount_amount,
public float $minimum_discount_amount,
/** @var string The price as string (with .) */
public readonly string $price,
public string $price,
/** @var string The currency of the used ISO code of this price detail */
public readonly ?string $currency_iso_code,
public ?string $currency_iso_code,
/** @var bool If the price includes tax */
public readonly ?bool $includes_tax = true,
public ?bool $includes_tax = true,
/** @var float the price related quantity */
public readonly ?float $price_related_quantity = 1.0,
public ?float $price_related_quantity = 1.0,
)
{
$this->price_as_big_decimal = BigDecimal::of($this->price);

12
src/Services/InfoProviderSystem/DTOs/PurchaseInfoDTO.php
View file

@ -27,15 +27,15 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This DTO represents a purchase information for a part (supplier name, order number and prices).
* @see \App\Tests\Services\InfoProviderSystem\DTOs\PurchaseInfoDTOTest
*/
class PurchaseInfoDTO
readonly class PurchaseInfoDTO
{
public function __construct(
public readonly string $distributor_name,
public readonly string $order_number,
public string $distributor_name,
public string $order_number,
/** @var PriceDTO[] */
public readonly array $prices,
public array $prices,
/** @var string|null An url to the product page of the vendor */
public readonly ?string $product_url = null,
public ?string $product_url = null,
)
{
//Ensure that the prices are PriceDTO instances
@ -45,4 +45,4 @@ class PurchaseInfoDTO
}
}
}
}
}

49
src/Services/InfoProviderSystem/DTOs/SearchResultDTO.php
View file

@ -59,8 +59,8 @@ class SearchResultDTO
public readonly ?string $provider_url = null,
/** @var string|null A footprint representation of the providers page */
public readonly ?string $footprint = null,
) {
)
{
if ($preview_image_url !== null) {
//Utilize the escaping mechanism of FileDTO to ensure that the preview image URL is correctly encoded
//See issue #521: https://github.com/Part-DB/Part-DB-server/issues/521
@ -71,4 +71,47 @@ class SearchResultDTO
$this->preview_image_url = null;
}
}
}
/**
* This method creates a normalized array representation of the DTO.
* @return array
*/
public function toNormalizedSearchResultArray(): array
{
return [
'provider_key' => $this->provider_key,
'provider_id' => $this->provider_id,
'name' => $this->name,
'description' => $this->description,
'category' => $this->category,
'manufacturer' => $this->manufacturer,
'mpn' => $this->mpn,
'preview_image_url' => $this->preview_image_url,
'manufacturing_status' => $this->manufacturing_status?->value,
'provider_url' => $this->provider_url,
'footprint' => $this->footprint,
];
}
/**
* Creates a SearchResultDTO from a normalized array representation.
* @param array $data
* @return self
*/
public static function fromNormalizedSearchResultArray(array $data): self
{
return new self(
provider_key: $data['provider_key'],
provider_id: $data['provider_id'],
name: $data['name'],
description: $data['description'],
category: $data['category'] ?? null,
manufacturer: $data['manufacturer'] ?? null,
mpn: $data['mpn'] ?? null,
preview_image_url: $data['preview_image_url'] ?? null,
manufacturing_status: isset($data['manufacturing_status']) ? ManufacturingStatus::tryFrom($data['manufacturing_status']) : null,
provider_url: $data['provider_url'] ?? null,
footprint: $data['footprint'] ?? null,
);
}
}

40
src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
/**
* This interface marks a provider as a info provider which can provide information directly in batch operations
*/
interface BatchInfoProviderInterface extends InfoProviderInterface
{
/**
* Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
* This allows for a more efficient search compared to running multiple single searches.
* @param string[] $keywords
* @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
*/
public function searchByKeywordsBatch(array $keywords): array;
}

145
src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file

@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings;
use Symfony\Component\HttpFoundation\Cookie;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class LCSCProvider implements InfoProviderInterface
class LCSCProvider implements BatchInfoProviderInterface
{
private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';
@ -69,9 +69,10 @@ class LCSCProvider implements InfoProviderInterface
/**
* @param string $id
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO
*/
private function queryDetail(string $id): PartDetailDTO
private function queryDetail(string $id, bool $lightweight = false): PartDetailDTO
{
$response = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
@ -89,7 +90,7 @@ class LCSCProvider implements InfoProviderInterface
throw new \RuntimeException('Could not find product code: ' . $id);
}
return $this->getPartDetail($product);
return $this->getPartDetail($product, $lightweight);
}
/**
@ -99,30 +100,42 @@ class LCSCProvider implements InfoProviderInterface
private function getRealDatasheetUrl(?string $url): string
{
if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.lcsc\.com|www\.lcsc\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) {
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
}
return $url;
}
/**
* @param string $term
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO[]
*/
private function queryByTerm(string $term): array
private function queryByTerm(string $term, bool $lightweight = false): array
{
// Optimize: If term looks like an LCSC part number (starts with C followed by digits),
// use direct detail query instead of slower search
if (preg_match('/^C\d+$/i', trim($term))) {
try {
return [$this->queryDetail(trim($term), $lightweight)];
} catch (\Exception $e) {
// If direct lookup fails, fall back to search
// This handles cases where the C-code might not exist
}
}
$response = $this->lcscClient->request('POST', self::ENDPOINT_URL . "/search/v2/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
@ -145,11 +158,11 @@ class LCSCProvider implements InfoProviderInterface
// detailed product listing. It does so utilizing a product tip field.
// If product tip exists and there are no products in the product list try a detail query
if (count($products) === 0 && $tipProductCode !== null) {
$result[] = $this->queryDetail($tipProductCode);
$result[] = $this->queryDetail($tipProductCode, $lightweight);
}
foreach ($products as $product) {
$result[] = $this->getPartDetail($product);
$result[] = $this->getPartDetail($product, $lightweight);
}
return $result;
@ -178,7 +191,7 @@ class LCSCProvider implements InfoProviderInterface
* @param array $product
* @return PartDetailDTO
*/
private function getPartDetail(array $product): PartDetailDTO
private function getPartDetail(array $product, bool $lightweight = false): PartDetailDTO
{
// Get product images in advance
$product_images = $this->getProductImages($product['productImages'] ?? null);
@ -214,10 +227,10 @@ class LCSCProvider implements InfoProviderInterface
manufacturing_status: null,
provider_url: $this->getProductShortURL($product['productCode']),
footprint: $this->sanitizeField($footprint),
datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images,
parameters: $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
datasheets: $lightweight ? [] : $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images, // Always include images - users need to see them
parameters: $lightweight ? [] : $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $lightweight ? [] : $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
mass: $product['weight'] ?? null,
);
}
@ -286,7 +299,7 @@ class LCSCProvider implements InfoProviderInterface
*/
private function getProductShortURL(string $product_code): string
{
return 'https://www.lcsc.com/product-detail/' . $product_code .'.html';
return 'https://www.lcsc.com/product-detail/' . $product_code . '.html';
}
/**
@ -327,7 +340,7 @@ class LCSCProvider implements InfoProviderInterface
//Skip this attribute if it's empty
if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) {
continue;
continue;
}
$result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null);
@ -338,12 +351,86 @@ class LCSCProvider implements InfoProviderInterface
public function searchByKeyword(string $keyword): array
{
return $this->queryByTerm($keyword);
return $this->queryByTerm($keyword, true); // Use lightweight mode for search
}
/**
* Batch search multiple keywords asynchronously (like JavaScript Promise.all)
* @param array $keywords Array of keywords to search
* @return array Results indexed by keyword
*/
public function searchByKeywordsBatch(array $keywords): array
{
if (empty($keywords)) {
return [];
}
$responses = [];
$results = [];
// Start all requests immediately (like JavaScript promises without await)
foreach ($keywords as $keyword) {
if (preg_match('/^C\d+$/i', trim($keyword))) {
// Direct detail API call for C-codes
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'productCode' => trim($keyword),
],
]);
} else {
// Search API call for other terms
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/search/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'keyword' => $keyword,
],
]);
}
}
// Now collect all results (like .then() in JavaScript)
foreach ($responses as $keyword => $response) {
try {
$arr = $response->toArray(); // This waits for the response
$results[$keyword] = $this->processSearchResponse($arr, $keyword);
} catch (\Exception $e) {
$results[$keyword] = []; // Empty results on error
}
}
return $results;
}
private function processSearchResponse(array $arr, string $keyword): array
{
$result = [];
// Check if this looks like a detail response (direct C-code lookup)
if (isset($arr['result']['productCode'])) {
$product = $arr['result'];
$result[] = $this->getPartDetail($product, true); // lightweight mode
} else {
// This is a search response
$products = $arr['result']['productSearchResultVO']['productList'] ?? [];
$tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null;
// If no products but has tip, we'd need another API call - skip for batch mode
foreach ($products as $product) {
$result[] = $this->getPartDetail($product, true); // lightweight mode
}
}
return $result;
}
public function getDetails(string $id): PartDetailDTO
{
$tmp = $this->queryByTerm($id);
$tmp = $this->queryByTerm($id, false);
if (count($tmp) === 0) {
throw new \RuntimeException('No part found with ID ' . $id);
}

19
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
return $this->responseToDTOArray($response);
}
@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface
]
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
$tmp = $this->responseToDTOArray($response);
//Ensure that we have exactly one result

14
src/Services/Parts/PartsTableActionHandler.php
View file

@ -30,13 +30,11 @@ use App\Entity\Parts\Manufacturer;
use App\Entity\Parts\MeasurementUnit;
use App\Entity\Parts\Part;
use App\Entity\Parts\PartLot;
use App\Repository\PartRepository;
use Doctrine\ORM\EntityManagerInterface;
use InvalidArgumentException;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Symfony\Component\Security\Core\Exception\AccessDeniedException;
use Symfony\Contracts\Translation\TranslatableInterface;
use function Symfony\Component\Translation\t;
@ -100,7 +98,7 @@ implode(',', array_map(static fn (PartLot $lot) => $lot->getID(), $part->getPart
//When action starts with "export_" we have to redirect to the export controller
$matches = [];
if (preg_match('/^export_(json|yaml|xml|csv)$/', $action, $matches)) {
if (preg_match('/^export_(json|yaml|xml|csv|xlsx)$/', $action, $matches)) {
$ids = implode(',', array_map(static fn (Part $part) => $part->getID(), $selected_parts));
$level = match ($target_id) {
2 => 'extended',
@ -119,6 +117,16 @@ implode(',', array_map(static fn (PartLot $lot) => $lot->getID(), $part->getPart
);
}
if ($action === 'bulk_info_provider_import') {
$ids = implode(',', array_map(static fn (Part $part) => $part->getID(), $selected_parts));
return new RedirectResponse(
$this->urlGenerator->generate('bulk_info_provider_step1', [
'ids' => $ids,
'_redirect' => $redirect_url
])
);
}
//Iterate over the parts and apply the action to it:
foreach ($selected_parts as $part) {

5
src/Services/Trees/ToolsTreeBuilder.php
View file

@ -138,6 +138,11 @@ class ToolsTreeBuilder
$this->translator->trans('info_providers.search.title'),
$this->urlGenerator->generate('info_providers_search')
))->setIcon('fa-treeview fa-fw fa-solid fa-cloud-arrow-down');
$nodes[] = (new TreeViewNode(
$this->translator->trans('info_providers.bulk_import.manage_jobs'),
$this->urlGenerator->generate('bulk_info_provider_manage')
))->setIcon('fa-treeview fa-fw fa-solid fa-tasks');
}
return $nodes;