groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-15 23:49:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'feature/batch-info-provider-import'

Browse source
This commit is contained in:
Jan Böhmer 2025-09-21 23:14:09 +02:00
commit ed1e51f694
80 changed files with 9789 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

380
src/Services/InfoProviderSystem/BulkInfoProviderService.php Normal file
View file

@ -0,0 +1,380 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchFieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultsDTO;
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpClient\Exception\ClientException;
final class BulkInfoProviderService
{
/** @var array<string, Supplier|null> Cache for normalized supplier names */
private array $supplierCache = [];
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly ExistingPartFinder $existingPartFinder,
private readonly ProviderRegistry $providerRegistry,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
) {}
/**
* Perform bulk search across multiple parts and providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy
* @param bool $prefetchDetails Whether to prefetch detailed information for results
* @return BulkSearchResponseDTO Structured response containing all search results
* @throws \InvalidArgumentException If no valid parts provided
* @throws \RuntimeException If no search results found for any parts
*/
public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO
{
if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import');
}
$partResults = [];
$hasAnyResults = false;
// Group providers by batch capability
$batchProviders = [];
$regularProviders = [];
foreach ($fieldMappings as $mapping) {
foreach ($mapping->providers as $providerKey) {
if (!is_string($providerKey)) {
$this->logger->error('Invalid provider key type', [
'providerKey' => $providerKey,
'type' => gettype($providerKey)
]);
continue;
}
$provider = $this->providerRegistry->getProviderByKey($providerKey);
if ($provider instanceof BatchInfoProviderInterface) {
$batchProviders[$providerKey] = $provider;
} else {
$regularProviders[$providerKey] = $provider;
}
}
}
// Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders);
// Process regular providers
$regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults);
// Combine and format results for each part
foreach ($parts as $part) {
$searchResults = [];
// Get results from batch and regular processing
$allResults = array_merge(
$batchResults[$part->getId()] ?? [],
$regularResults[$part->getId()] ?? []
);
if (!empty($allResults)) {
$hasAnyResults = true;
$searchResults = $this->formatSearchResults($allResults);
}
$partResults[] = new BulkSearchPartResultsDTO(
part: $part,
searchResults: $searchResults,
errors: []
);
}
if (!$hasAnyResults) {
throw new \RuntimeException('No search results found for any of the selected parts');
}
$response = new BulkSearchResponseDTO($partResults);
// Prefetch details if requested
if ($prefetchDetails) {
$this->prefetchDetailsForResults($response);
}
return $response;
}
/**
* Process parts using batch-capable info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, BatchInfoProviderInterface> $batchProviders Batch providers indexed by key
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{
$batchResults = [];
foreach ($batchProviders as $providerKey => $provider) {
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
if (empty($keywords)) {
continue;
}
try {
$providerResults = $provider->searchByKeywordsBatch($keywords);
// Map results back to parts
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) {
$batchResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
}
}
}
} catch (\Exception $e) {
$this->logger->error('Batch search failed for provider ' . $providerKey, [
'error' => $e->getMessage(),
'provider' => $providerKey
]);
}
}
return $batchResults;
}
/**
* Process parts using regular (non-batch) info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, InfoProviderInterface> $regularProviders Regular providers indexed by key
* @param array<int, BulkSearchPartResultDTO[]> $excludeResults Results to exclude (from batch processing)
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{
$regularResults = [];
foreach ($parts as $part) {
$regularResults[$part->getId()] = [];
// Skip if we already have batch results for this part
if (!empty($excludeResults[$part->getId()] ?? [])) {
continue;
}
foreach ($fieldMappings as $mapping) {
$providers = array_intersect($mapping->providers, array_keys($regularProviders));
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if (!$keyword) {
continue;
}
try {
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
} catch (ClientException $e) {
$this->logger->error('Regular search failed', [
'part_id' => $part->getId(),
'field' => $mapping->field,
'error' => $e->getMessage()
]);
}
}
}
return $regularResults;
}
/**
* Collect unique keywords for a specific provider from all parts and field mappings.
*
* @param Part[] $parts Array of parts to collect keywords from
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param string $providerKey The provider key to collect keywords for
* @return string[] Array of unique keywords
*/
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{
$keywords = [];
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword;
}
}
}
return $keywords;
}
private function getKeywordFromField(Part $part, string $field): ?string
{
return match ($field) {
'mpn' => $part->getManufacturerProductNumber(),
'name' => $part->getName(),
default => $this->getSupplierPartNumber($part, $field)
};
}
private function getSupplierPartNumber(Part $part, string $field): ?string
{
if (!str_ends_with($field, '_spn')) {
return null;
}
$supplierKey = substr($field, 0, -4);
$supplier = $this->getSupplierByNormalizedName($supplierKey);
if (!$supplier) {
return null;
}
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null;
}
/**
* Get supplier by normalized name with caching to prevent N+1 queries.
*
* @param string $normalizedKey The normalized supplier key to search for
* @return Supplier|null The matching supplier or null if not found
*/
private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier
{
// Check cache first
if (isset($this->supplierCache[$normalizedKey])) {
return $this->supplierCache[$normalizedKey];
}
// Use efficient database query with PHP normalization
// Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization
if (empty($this->supplierCache)) {
$this->loadSuppliersIntoCache();
}
$supplier = $this->supplierCache[$normalizedKey] ?? null;
// Cache the result (including null results to prevent repeated queries)
$this->supplierCache[$normalizedKey] = $supplier;
return $supplier;
}
/**
* Load all suppliers into cache with normalized names to avoid N+1 queries.
*/
private function loadSuppliersIntoCache(): void
{
/** @var Supplier[] $suppliers */
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
$this->supplierCache[$normalizedName] = $supplier;
}
}
/**
* Format and deduplicate search results.
*
* @param BulkSearchPartResultDTO[] $bulkResults Array of bulk search results
* @return BulkSearchPartResultDTO[] Array of formatted search results with metadata
*/
private function formatSearchResults(array $bulkResults): array
{
// Sort by priority and remove duplicates
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
$uniqueResults = [];
$seenKeys = [];
foreach ($bulkResults as $result) {
$key = "{$result->searchResult->provider_key}|{$result->searchResult->provider_id}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueResults[] = $result;
}
}
return $uniqueResults;
}
/**
* Prefetch detailed information for search results.
*
* @param BulkSearchResponseDTO $searchResults Search results (supports both new DTO and legacy array format)
*/
public function prefetchDetailsForResults(BulkSearchResponseDTO $searchResults): void
{
$prefetchCount = 0;
// Handle both new DTO format and legacy array format for backwards compatibility
foreach ($searchResults->partResults as $partResult) {
foreach ($partResult->searchResults as $result) {
$dto = $result->searchResult;
try {
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
}
}
}
$this->logger->info("Prefetched details for {$prefetchCount} search results");
}
}

91
src/Services/InfoProviderSystem/DTOs/BulkSearchFieldMappingDTO.php Normal file
View file

@ -0,0 +1,91 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
/**
* Represents a mapping between a part field and the info providers that should search in that field.
*/
readonly class BulkSearchFieldMappingDTO
{
/**
* @param string $field The field to search in (e.g., 'mpn', 'name', or supplier-specific fields like 'digikey_spn')
* @param string[] $providers Array of provider keys to search with (e.g., ['digikey', 'farnell'])
* @param int $priority Priority for this field mapping (1-10, lower numbers = higher priority)
*/
public function __construct(
public string $field,
public array $providers,
public int $priority = 1
) {
if ($priority < 1 || $priority > 10) {
throw new \InvalidArgumentException('Priority must be between 1 and 10');
}
}
/**
* Create a FieldMappingDTO from legacy array format.
* @param array{field: string, providers: string[], priority?: int} $data
*/
public static function fromSerializableArray(array $data): self
{
return new self(
field: $data['field'],
providers: $data['providers'] ?? [],
priority: $data['priority'] ?? 1
);
}
/**
* Convert this DTO to the legacy array format for backwards compatibility.
* @return array{field: string, providers: string[], priority: int}
*/
public function toSerializableArray(): array
{
return [
'field' => $this->field,
'providers' => $this->providers,
'priority' => $this->priority,
];
}
/**
* Check if this field mapping is for a supplier part number field.
*/
public function isSupplierPartNumberField(): bool
{
return str_ends_with($this->field, '_spn');
}
/**
* Get the supplier key from a supplier part number field.
* Returns null if this is not a supplier part number field.
*/
public function getSupplierKey(): ?string
{
if (!$this->isSupplierPartNumberField()) {
return null;
}
return substr($this->field, 0, -4);
}
}

44
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultDTO.php Normal file
View file

@ -0,0 +1,44 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents a single search result from bulk search with additional context information, like how the part was found.
*/
readonly class BulkSearchPartResultDTO
{
public function __construct(
/** The base search result DTO containing provider data */
public SearchResultDTO $searchResult,
/** The field that was used to find this result */
public ?string $sourceField = null,
/** The actual keyword that was searched for */
public ?string $sourceKeyword = null,
/** Local part that matches this search result, if any */
public ?Part $localPart = null,
/** Priority for this search result */
public int $priority = 1
) {}
}

83
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultsDTO.php Normal file
View file

@ -0,0 +1,83 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents the search results for a single part from bulk info provider search.
* It contains multiple search results, that match the part.
*/
readonly class BulkSearchPartResultsDTO
{
/**
* @param Part $part The part that was searched for
* @param BulkSearchPartResultDTO[] $searchResults Array of search results found for this part
* @param string[] $errors Array of error messages encountered during search
*/
public function __construct(
public Part $part,
public array $searchResults = [],
public array $errors = []
) {}
/**
* Check if this part has any search results.
*/
public function hasResults(): bool
{
return !empty($this->searchResults);
}
/**
* Check if this part has any errors.
*/
public function hasErrors(): bool
{
return !empty($this->errors);
}
/**
* Get the number of search results for this part.
*/
public function getResultCount(): int
{
return count($this->searchResults);
}
public function getErrorCount(): int
{
return count($this->errors);
}
/**
* Get search results sorted by priority (ascending).
* @return BulkSearchPartResultDTO[]
*/
public function getResultsSortedByPriority(): array
{
$results = $this->searchResults;
usort($results, static fn(BulkSearchPartResultDTO $a, BulkSearchPartResultDTO $b) => $a->priority <=> $b->priority);
return $results;
}
}

231
src/Services/InfoProviderSystem/DTOs/BulkSearchResponseDTO.php Normal file
View file

@ -0,0 +1,231 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
use Doctrine\ORM\EntityManagerInterface;
use Traversable;
/**
* Represents the complete response from a bulk info provider search operation.
* It contains a list of PartSearchResultDTOs, one for each part searched.
*/
readonly class BulkSearchResponseDTO implements \ArrayAccess, \IteratorAggregate
{
/**
* @param BulkSearchPartResultsDTO[] $partResults Array of search results for each part
*/
public function __construct(
public array $partResults
) {}
/**
* Replaces the search results for a specific part, and returns a new instance.
* The part to replaced, is identified by the part property of the new_results parameter.
* The original instance remains unchanged.
* @param BulkSearchPartResultsDTO $new_results
* @return BulkSearchResponseDTO
*/
public function replaceResultsForPart(BulkSearchPartResultsDTO $new_results): self
{
$array = $this->partResults;
$replaced = false;
foreach ($array as $index => $partResult) {
if ($partResult->part === $new_results->part) {
$array[$index] = $new_results;
$replaced = true;
break;
}
}
if (!$replaced) {
throw new \InvalidArgumentException("Part not found in existing results.");
}
return new self($array);
}
/**
* Check if any parts have search results.
*/
public function hasAnyResults(): bool
{
foreach ($this->partResults as $partResult) {
if ($partResult->hasResults()) {
return true;
}
}
return false;
}
/**
* Get the total number of search results across all parts.
*/
public function getTotalResultCount(): int
{
$count = 0;
foreach ($this->partResults as $partResult) {
$count += $partResult->getResultCount();
}
return $count;
}
/**
* Get all parts that have search results.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithResults(): array
{
return array_filter($this->partResults, fn($result) => $result->hasResults());
}
/**
* Get all parts that have errors.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithErrors(): array
{
return array_filter($this->partResults, fn($result) => $result->hasErrors());
}
/**
* Get the number of parts processed.
*/
public function getPartCount(): int
{
return count($this->partResults);
}
/**
* Get the number of parts with successful results.
*/
public function getSuccessfulPartCount(): int
{
return count($this->getPartsWithResults());
}
/**
* Merge multiple BulkSearchResponseDTO instances into one.
* @param BulkSearchResponseDTO ...$responses
* @return BulkSearchResponseDTO
*/
public static function merge(BulkSearchResponseDTO ...$responses): BulkSearchResponseDTO
{
$mergedResults = [];
foreach ($responses as $response) {
foreach ($response->partResults as $partResult) {
$mergedResults[] = $partResult;
}
}
return new BulkSearchResponseDTO($mergedResults);
}
/**
* Convert this DTO to a serializable representation suitable for storage in the database
* @return array
*/
public function toSerializableRepresentation(): array
{
$serialized = [];
foreach ($this->partResults as $partResult) {
$partData = [
'part_id' => $partResult->part->getId(),
'search_results' => [],
'errors' => $partResult->errors ?? []
];
foreach ($partResult->searchResults as $result) {
$partData['search_results'][] = [
'dto' => $result->searchResult->toNormalizedSearchResultArray(),
'source_field' => $result->sourceField ?? null,
'source_keyword' => $result->sourceKeyword ?? null,
'localPart' => $result->localPart?->getId(),
'priority' => $result->priority
];
}
$serialized[] = $partData;
}
return $serialized;
}
/**
* Creates a BulkSearchResponseDTO from a serializable representation.
* @param array $data
* @param EntityManagerInterface $entityManager
* @return BulkSearchResponseDTO
* @throws \Doctrine\ORM\Exception\ORMException
*/
public static function fromSerializableRepresentation(array $data, EntityManagerInterface $entityManager): BulkSearchResponseDTO
{
$partResults = [];
foreach ($data as $partData) {
$partResults[] = new BulkSearchPartResultsDTO(
part: $entityManager->getReference(Part::class, $partData['part_id']),
searchResults: array_map(fn($result) => new BulkSearchPartResultDTO(
searchResult: SearchResultDTO::fromNormalizedSearchResultArray($result['dto']),
sourceField: $result['source_field'] ?? null,
sourceKeyword: $result['source_keyword'] ?? null,
localPart: isset($result['localPart']) ? $entityManager->getReference(Part::class, $result['localPart']) : null,
priority: $result['priority'] ?? null
), $partData['search_results'] ?? []),
errors: $partData['errors'] ?? []
);
}
return new BulkSearchResponseDTO($partResults);
}
public function offsetExists(mixed $offset): bool
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return isset($this->partResults[$offset]);
}
public function offsetGet(mixed $offset): ?BulkSearchPartResultsDTO
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return $this->partResults[$offset] ?? null;
}
public function offsetSet(mixed $offset, mixed $value): void
{
throw new \LogicException("BulkSearchResponseDTO is immutable.");
}
public function offsetUnset(mixed $offset): void
{
throw new \LogicException('BulkSearchResponseDTO is immutable.');
}
public function getIterator(): Traversable
{
return new \ArrayIterator($this->partResults);
}
}

8
src/Services/InfoProviderSystem/DTOs/FileDTO.php
View file

@ -28,12 +28,12 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a datasheet, a 3D model, a picture or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\FileDTOTest
*/
class FileDTO
readonly class FileDTO
{
/**
* @var string The URL where to get this file
*/
public readonly string $url;
public string $url;
/**
* @param string $url The URL where to get this file
@ -41,7 +41,7 @@ class FileDTO
*/
public function __construct(
string $url,
public readonly ?string $name = null,
public ?string $name = null,
) {
//Find all occurrences of non URL safe characters and replace them with their URL encoded version.
//We only want to replace characters which can not have a valid meaning in a URL (what would break the URL).
@ -50,4 +50,4 @@ class FileDTO
}
}
}

18
src/Services/InfoProviderSystem/DTOs/ParameterDTO.php
View file

@ -28,17 +28,17 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a voltage, a current, a temperature or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\ParameterDTOTest
*/
class ParameterDTO
readonly class ParameterDTO
{
public function __construct(
public readonly string $name,
public readonly ?string $value_text = null,
public readonly ?float $value_typ = null,
public readonly ?float $value_min = null,
public readonly ?float $value_max = null,
public readonly ?string $unit = null,
public readonly ?string $symbol = null,
public readonly ?string $group = null,
public string $name,
public ?string $value_text = null,
public ?float $value_typ = null,
public ?float $value_min = null,
public ?float $value_max = null,
public ?string $unit = null,
public ?string $symbol = null,
public ?string $group = null,
) {
}

2
src/Services/InfoProviderSystem/DTOs/PartDetailDTO.php
View file

@ -70,4 +70,4 @@ class PartDetailDTO extends SearchResultDTO
footprint: $footprint,
);
}
}
}

14
src/Services/InfoProviderSystem/DTOs/PriceDTO.php
View file

@ -28,21 +28,21 @@ use Brick\Math\BigDecimal;
/**
* This DTO represents a price for a single unit in a certain discount range
*/
class PriceDTO
readonly class PriceDTO
{
private readonly BigDecimal $price_as_big_decimal;
private BigDecimal $price_as_big_decimal;
public function __construct(
/** @var float The minimum amount that needs to get ordered for this price to be valid */
public readonly float $minimum_discount_amount,
public float $minimum_discount_amount,
/** @var string The price as string (with .) */
public readonly string $price,
public string $price,
/** @var string The currency of the used ISO code of this price detail */
public readonly ?string $currency_iso_code,
public ?string $currency_iso_code,
/** @var bool If the price includes tax */
public readonly ?bool $includes_tax = true,
public ?bool $includes_tax = true,
/** @var float the price related quantity */
public readonly ?float $price_related_quantity = 1.0,
public ?float $price_related_quantity = 1.0,
)
{
$this->price_as_big_decimal = BigDecimal::of($this->price);

12
src/Services/InfoProviderSystem/DTOs/PurchaseInfoDTO.php
View file

@ -27,15 +27,15 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This DTO represents a purchase information for a part (supplier name, order number and prices).
* @see \App\Tests\Services\InfoProviderSystem\DTOs\PurchaseInfoDTOTest
*/
class PurchaseInfoDTO
readonly class PurchaseInfoDTO
{
public function __construct(
public readonly string $distributor_name,
public readonly string $order_number,
public string $distributor_name,
public string $order_number,
/** @var PriceDTO[] */
public readonly array $prices,
public array $prices,
/** @var string|null An url to the product page of the vendor */
public readonly ?string $product_url = null,
public ?string $product_url = null,
)
{
//Ensure that the prices are PriceDTO instances
@ -45,4 +45,4 @@ class PurchaseInfoDTO
}
}
}
}
}

49
src/Services/InfoProviderSystem/DTOs/SearchResultDTO.php
View file

@ -59,8 +59,8 @@ class SearchResultDTO
public readonly ?string $provider_url = null,
/** @var string|null A footprint representation of the providers page */
public readonly ?string $footprint = null,
) {
)
{
if ($preview_image_url !== null) {
//Utilize the escaping mechanism of FileDTO to ensure that the preview image URL is correctly encoded
//See issue #521: https://github.com/Part-DB/Part-DB-server/issues/521
@ -71,4 +71,47 @@ class SearchResultDTO
$this->preview_image_url = null;
}
}
}
/**
* This method creates a normalized array representation of the DTO.
* @return array
*/
public function toNormalizedSearchResultArray(): array
{
return [
'provider_key' => $this->provider_key,
'provider_id' => $this->provider_id,
'name' => $this->name,
'description' => $this->description,
'category' => $this->category,
'manufacturer' => $this->manufacturer,
'mpn' => $this->mpn,
'preview_image_url' => $this->preview_image_url,
'manufacturing_status' => $this->manufacturing_status?->value,
'provider_url' => $this->provider_url,
'footprint' => $this->footprint,
];
}
/**
* Creates a SearchResultDTO from a normalized array representation.
* @param array $data
* @return self
*/
public static function fromNormalizedSearchResultArray(array $data): self
{
return new self(
provider_key: $data['provider_key'],
provider_id: $data['provider_id'],
name: $data['name'],
description: $data['description'],
category: $data['category'] ?? null,
manufacturer: $data['manufacturer'] ?? null,
mpn: $data['mpn'] ?? null,
preview_image_url: $data['preview_image_url'] ?? null,
manufacturing_status: isset($data['manufacturing_status']) ? ManufacturingStatus::tryFrom($data['manufacturing_status']) : null,
provider_url: $data['provider_url'] ?? null,
footprint: $data['footprint'] ?? null,
);
}
}

40
src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
/**
* This interface marks a provider as a info provider which can provide information directly in batch operations
*/
interface BatchInfoProviderInterface extends InfoProviderInterface
{
/**
* Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
* This allows for a more efficient search compared to running multiple single searches.
* @param string[] $keywords
* @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
*/
public function searchByKeywordsBatch(array $keywords): array;
}

145
src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file

@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings;
use Symfony\Component\HttpFoundation\Cookie;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class LCSCProvider implements InfoProviderInterface
class LCSCProvider implements BatchInfoProviderInterface
{
private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';
@ -69,9 +69,10 @@ class LCSCProvider implements InfoProviderInterface
/**
* @param string $id
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO
*/
private function queryDetail(string $id): PartDetailDTO
private function queryDetail(string $id, bool $lightweight = false): PartDetailDTO
{
$response = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
@ -89,7 +90,7 @@ class LCSCProvider implements InfoProviderInterface
throw new \RuntimeException('Could not find product code: ' . $id);
}
return $this->getPartDetail($product);
return $this->getPartDetail($product, $lightweight);
}
/**
@ -99,30 +100,42 @@ class LCSCProvider implements InfoProviderInterface
private function getRealDatasheetUrl(?string $url): string
{
if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.lcsc\.com|www\.lcsc\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) {
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
}
return $url;
}
/**
* @param string $term
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO[]
*/
private function queryByTerm(string $term): array
private function queryByTerm(string $term, bool $lightweight = false): array
{
// Optimize: If term looks like an LCSC part number (starts with C followed by digits),
// use direct detail query instead of slower search
if (preg_match('/^C\d+$/i', trim($term))) {
try {
return [$this->queryDetail(trim($term), $lightweight)];
} catch (\Exception $e) {
// If direct lookup fails, fall back to search
// This handles cases where the C-code might not exist
}
}
$response = $this->lcscClient->request('POST', self::ENDPOINT_URL . "/search/v2/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
@ -145,11 +158,11 @@ class LCSCProvider implements InfoProviderInterface
// detailed product listing. It does so utilizing a product tip field.
// If product tip exists and there are no products in the product list try a detail query
if (count($products) === 0 && $tipProductCode !== null) {
$result[] = $this->queryDetail($tipProductCode);
$result[] = $this->queryDetail($tipProductCode, $lightweight);
}
foreach ($products as $product) {
$result[] = $this->getPartDetail($product);
$result[] = $this->getPartDetail($product, $lightweight);
}
return $result;
@ -178,7 +191,7 @@ class LCSCProvider implements InfoProviderInterface
* @param array $product
* @return PartDetailDTO
*/
private function getPartDetail(array $product): PartDetailDTO
private function getPartDetail(array $product, bool $lightweight = false): PartDetailDTO
{
// Get product images in advance
$product_images = $this->getProductImages($product['productImages'] ?? null);
@ -214,10 +227,10 @@ class LCSCProvider implements InfoProviderInterface
manufacturing_status: null,
provider_url: $this->getProductShortURL($product['productCode']),
footprint: $this->sanitizeField($footprint),
datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images,
parameters: $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
datasheets: $lightweight ? [] : $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images, // Always include images - users need to see them
parameters: $lightweight ? [] : $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $lightweight ? [] : $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
mass: $product['weight'] ?? null,
);
}
@ -286,7 +299,7 @@ class LCSCProvider implements InfoProviderInterface
*/
private function getProductShortURL(string $product_code): string
{
return 'https://www.lcsc.com/product-detail/' . $product_code .'.html';
return 'https://www.lcsc.com/product-detail/' . $product_code . '.html';
}
/**
@ -327,7 +340,7 @@ class LCSCProvider implements InfoProviderInterface
//Skip this attribute if it's empty
if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) {
continue;
continue;
}
$result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null);
@ -338,12 +351,86 @@ class LCSCProvider implements InfoProviderInterface
public function searchByKeyword(string $keyword): array
{
return $this->queryByTerm($keyword);
return $this->queryByTerm($keyword, true); // Use lightweight mode for search
}
/**
* Batch search multiple keywords asynchronously (like JavaScript Promise.all)
* @param array $keywords Array of keywords to search
* @return array Results indexed by keyword
*/
public function searchByKeywordsBatch(array $keywords): array
{
if (empty($keywords)) {
return [];
}
$responses = [];
$results = [];
// Start all requests immediately (like JavaScript promises without await)
foreach ($keywords as $keyword) {
if (preg_match('/^C\d+$/i', trim($keyword))) {
// Direct detail API call for C-codes
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'productCode' => trim($keyword),
],
]);
} else {
// Search API call for other terms
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/search/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'keyword' => $keyword,
],
]);
}
}
// Now collect all results (like .then() in JavaScript)
foreach ($responses as $keyword => $response) {
try {
$arr = $response->toArray(); // This waits for the response
$results[$keyword] = $this->processSearchResponse($arr, $keyword);
} catch (\Exception $e) {
$results[$keyword] = []; // Empty results on error
}
}
return $results;
}
private function processSearchResponse(array $arr, string $keyword): array
{
$result = [];
// Check if this looks like a detail response (direct C-code lookup)
if (isset($arr['result']['productCode'])) {
$product = $arr['result'];
$result[] = $this->getPartDetail($product, true); // lightweight mode
} else {
// This is a search response
$products = $arr['result']['productSearchResultVO']['productList'] ?? [];
$tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null;
// If no products but has tip, we'd need another API call - skip for batch mode
foreach ($products as $product) {
$result[] = $this->getPartDetail($product, true); // lightweight mode
}
}
return $result;
}
public function getDetails(string $id): PartDetailDTO
{
$tmp = $this->queryByTerm($id);
$tmp = $this->queryByTerm($id, false);
if (count($tmp) === 0) {
throw new \RuntimeException('No part found with ID ' . $id);
}

19
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
return $this->responseToDTOArray($response);
}
@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface
]
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
$tmp = $this->responseToDTOArray($response);
//Ensure that we have exactly one result