groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-09 04:29:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch 'feature/batch-info-provider-import'

Browse source
This commit is contained in:
Jan Böhmer 2025-09-21 23:14:09 +02:00
commit ed1e51f694
80 changed files with 9789 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

588
src/Controller/BulkInfoProviderImportController.php Normal file
View file

@ -0,0 +1,588 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Controller;
use App\Entity\InfoProviderSystem\BulkImportJobStatus;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJob;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Entity\UserSystem\User;
use App\Form\InfoProviderSystem\GlobalFieldMappingType;
use App\Services\InfoProviderSystem\BulkInfoProviderService;
use App\Services\InfoProviderSystem\DTOs\BulkSearchFieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultsDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
#[Route('/tools/bulk_info_provider_import')]
class BulkInfoProviderImportController extends AbstractController
{
public function __construct(
private readonly BulkInfoProviderService $bulkService,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger,
#[Autowire(param: 'partdb.bulk_import.batch_size')]
private readonly int $bulkImportBatchSize,
#[Autowire(param: 'partdb.bulk_import.max_parts_per_operation')]
private readonly int $bulkImportMaxParts
) {
}
/**
* Convert field mappings from array format to FieldMappingDTO[].
*
* @param array $fieldMappings Array of field mapping arrays
* @return BulkSearchFieldMappingDTO[] Array of FieldMappingDTO objects
*/
private function convertFieldMappingsToDto(array $fieldMappings): array
{
$dtos = [];
foreach ($fieldMappings as $mapping) {
$dtos[] = new BulkSearchFieldMappingDTO(field: $mapping['field'], providers: $mapping['providers'], priority: $mapping['priority'] ?? 1);
}
return $dtos;
}
private function createErrorResponse(string $message, int $statusCode = 400, array $context = []): JsonResponse
{
$this->logger->warning('Bulk import operation failed', array_merge([
'error' => $message,
'user' => $this->getUser()?->getUserIdentifier(),
], $context));
return $this->json([
'success' => false,
'error' => $message
], $statusCode);
}
private function validateJobAccess(int $jobId): ?BulkInfoProviderImportJob
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
$job = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)->find($jobId);
if (!$job) {
return null;
}
if ($job->getCreatedBy() !== $this->getUser()) {
return null;
}
return $job;
}
private function updatePartSearchResults(BulkInfoProviderImportJob $job, ?BulkSearchPartResultsDTO $newResults): void
{
if ($newResults === null) {
return;
}
// Only deserialize and update if we have new results
$allResults = $job->getSearchResults($this->entityManager);
// Find and update the results for this specific part
$allResults = $allResults->replaceResultsForPart($newResults);
// Save updated results back to job
$job->setSearchResults($allResults);
}
#[Route('/step1', name: 'bulk_info_provider_step1')]
public function step1(Request $request): Response
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
set_time_limit(600);
$ids = $request->query->get('ids');
if (!$ids) {
$this->addFlash('error', 'No parts selected for bulk import');
return $this->redirectToRoute('homepage');
}
$partIds = explode(',', $ids);
$partRepository = $this->entityManager->getRepository(Part::class);
$parts = $partRepository->getElementsFromIDArray($partIds);
if (empty($parts)) {
$this->addFlash('error', 'No valid parts found for bulk import');
return $this->redirectToRoute('homepage');
}
// Validate against configured maximum
if (count($parts) > $this->bulkImportMaxParts) {
$this->addFlash('error', sprintf(
'Too many parts selected (%d). Maximum allowed is %d parts per operation.',
count($parts),
$this->bulkImportMaxParts
));
return $this->redirectToRoute('homepage');
}
if (count($parts) > ($this->bulkImportMaxParts / 2)) {
$this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
}
// Generate field choices
$fieldChoices = [
'info_providers.bulk_search.field.mpn' => 'mpn',
'info_providers.bulk_search.field.name' => 'name',
];
// Add dynamic supplier fields
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$supplierKey = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
$fieldChoices["Supplier: " . $supplier->getName() . " (SPN)"] = $supplierKey . '_spn';
}
// Initialize form with useful default mappings
$initialData = [
'field_mappings' => [
['field' => 'mpn', 'providers' => [], 'priority' => 1]
],
'prefetch_details' => false
];
$form = $this->createForm(GlobalFieldMappingType::class, $initialData, [
'field_choices' => $fieldChoices
]);
$form->handleRequest($request);
$searchResults = null;
if ($form->isSubmitted() && $form->isValid()) {
$formData = $form->getData();
$fieldMappingDtos = $this->convertFieldMappingsToDto($formData['field_mappings']);
$prefetchDetails = $formData['prefetch_details'] ?? false;
$user = $this->getUser();
if (!$user instanceof User) {
throw new \RuntimeException('User must be authenticated and of type User');
}
// Validate part count against configuration limit
if (count($parts) > $this->bulkImportMaxParts) {
$this->addFlash('error', "Too many parts selected. Maximum allowed: {$this->bulkImportMaxParts}");
$partIds = array_map(fn($part) => $part->getId(), $parts);
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
// Create and save the job
$job = new BulkInfoProviderImportJob();
$job->setFieldMappings($fieldMappingDtos);
$job->setPrefetchDetails($prefetchDetails);
$job->setCreatedBy($user);
foreach ($parts as $part) {
$jobPart = new BulkInfoProviderImportJobPart($job, $part);
$job->addJobPart($jobPart);
}
$this->entityManager->persist($job);
$this->entityManager->flush();
try {
$searchResultsDto = $this->bulkService->performBulkSearch($parts, $fieldMappingDtos, $prefetchDetails);
// Save search results to job
$job->setSearchResults($searchResultsDto);
$job->markAsInProgress();
$this->entityManager->flush();
// Prefetch details if requested
if ($prefetchDetails) {
$this->bulkService->prefetchDetailsForResults($searchResultsDto);
}
return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $job->getId()]);
} catch (\Exception $e) {
$this->logger->error('Critical error during bulk import search', [
'job_id' => $job->getId(),
'error' => $e->getMessage(),
'exception' => $e
]);
$this->entityManager->remove($job);
$this->entityManager->flush();
$this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
$partIds = array_map(fn($part) => $part->getId(), $parts);
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
}
}
// Get existing in-progress jobs for current user
$existingJobs = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)
->findBy(['createdBy' => $this->getUser(), 'status' => BulkImportJobStatus::IN_PROGRESS], ['createdAt' => 'DESC'], 10);
return $this->render('info_providers/bulk_import/step1.html.twig', [
'form' => $form,
'parts' => $parts,
'search_results' => $searchResults,
'existing_jobs' => $existingJobs,
'fieldChoices' => $fieldChoices
]);
}
#[Route('/manage', name: 'bulk_info_provider_manage')]
public function manageBulkJobs(): Response
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
// Get all jobs for current user
$allJobs = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)
->findBy([], ['createdAt' => 'DESC']);
// Check and auto-complete jobs that should be completed
// Also clean up jobs with no results (failed searches)
$updatedJobs = false;
$jobsToDelete = [];
foreach ($allJobs as $job) {
if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
$job->markAsCompleted();
$updatedJobs = true;
}
// Mark jobs with no results for deletion (failed searches)
if ($job->getResultCount() === 0 && $job->isInProgress()) {
$jobsToDelete[] = $job;
}
}
// Delete failed jobs
foreach ($jobsToDelete as $job) {
$this->entityManager->remove($job);
$updatedJobs = true;
}
// Flush changes if any jobs were updated
if ($updatedJobs) {
$this->entityManager->flush();
if (!empty($jobsToDelete)) {
$this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.');
}
}
return $this->render('info_providers/bulk_import/manage.html.twig', [
'jobs' => $this->entityManager->getRepository(BulkInfoProviderImportJob::class)
->findBy([], ['createdAt' => 'DESC']) // Refetch after cleanup
]);
}
#[Route('/job/{jobId}/delete', name: 'bulk_info_provider_delete', methods: ['DELETE'])]
public function deleteJob(int $jobId): Response
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
// Only allow deletion of completed, failed, or stopped jobs
if (!$job->isCompleted() && !$job->isFailed() && !$job->isStopped()) {
return $this->json(['error' => 'Cannot delete active job'], 400);
}
$this->entityManager->remove($job);
$this->entityManager->flush();
return $this->json(['success' => true]);
}
#[Route('/job/{jobId}/stop', name: 'bulk_info_provider_stop', methods: ['POST'])]
public function stopJob(int $jobId): Response
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
// Only allow stopping of pending or in-progress jobs
if (!$job->canBeStopped()) {
return $this->json(['error' => 'Cannot stop job in current status'], 400);
}
$job->markAsStopped();
$this->entityManager->flush();
return $this->json(['success' => true]);
}
#[Route('/step2/{jobId}', name: 'bulk_info_provider_step2')]
public function step2(int $jobId): Response
{
$this->denyAccessUnlessGranted('@info_providers.create_parts');
$job = $this->entityManager->getRepository(BulkInfoProviderImportJob::class)->find($jobId);
if (!$job) {
$this->addFlash('error', 'Bulk import job not found');
return $this->redirectToRoute('bulk_info_provider_step1');
}
// Check if user owns this job
if ($job->getCreatedBy() !== $this->getUser()) {
$this->addFlash('error', 'Access denied to this bulk import job');
return $this->redirectToRoute('bulk_info_provider_step1');
}
// Get the parts and deserialize search results
$parts = $job->getJobParts()->map(fn($jobPart) => $jobPart->getPart())->toArray();
$searchResults = $job->getSearchResults($this->entityManager);
return $this->render('info_providers/bulk_import/step2.html.twig', [
'job' => $job,
'parts' => $parts,
'search_results' => $searchResults,
]);
}
#[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
public function markPartCompleted(int $jobId, int $partId): Response
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
$job->markPartAsCompleted($partId);
// Auto-complete job if all parts are done
if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
$job->markAsCompleted();
}
$this->entityManager->flush();
return $this->json([
'success' => true,
'progress' => $job->getProgressPercentage(),
'completed_count' => $job->getCompletedPartsCount(),
'total_count' => $job->getPartCount(),
'job_completed' => $job->isCompleted()
]);
}
#[Route('/job/{jobId}/part/{partId}/mark-skipped', name: 'bulk_info_provider_mark_skipped', methods: ['POST'])]
public function markPartSkipped(int $jobId, int $partId, Request $request): Response
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
$reason = $request->request->get('reason', '');
$job->markPartAsSkipped($partId, $reason);
// Auto-complete job if all parts are done
if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
$job->markAsCompleted();
}
$this->entityManager->flush();
return $this->json([
'success' => true,
'progress' => $job->getProgressPercentage(),
'completed_count' => $job->getCompletedPartsCount(),
'skipped_count' => $job->getSkippedPartsCount(),
'total_count' => $job->getPartCount(),
'job_completed' => $job->isCompleted()
]);
}
#[Route('/job/{jobId}/part/{partId}/mark-pending', name: 'bulk_info_provider_mark_pending', methods: ['POST'])]
public function markPartPending(int $jobId, int $partId): Response
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
$job->markPartAsPending($partId);
$this->entityManager->flush();
return $this->json([
'success' => true,
'progress' => $job->getProgressPercentage(),
'completed_count' => $job->getCompletedPartsCount(),
'skipped_count' => $job->getSkippedPartsCount(),
'total_count' => $job->getPartCount(),
'job_completed' => $job->isCompleted()
]);
}
#[Route('/job/{jobId}/part/{partId}/research', name: 'bulk_info_provider_research_part', methods: ['POST'])]
public function researchPart(int $jobId, int $partId): JsonResponse
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
$part = $this->entityManager->getRepository(Part::class)->find($partId);
if (!$part) {
return $this->createErrorResponse('Part not found', 404, ['part_id' => $partId]);
}
// Only refresh if the entity might be stale (optional optimization)
if ($this->entityManager->getUnitOfWork()->isScheduledForUpdate($part)) {
$this->entityManager->refresh($part);
}
try {
// Use the job's field mappings to perform the search
$fieldMappingDtos = $job->getFieldMappings();
$prefetchDetails = $job->isPrefetchDetails();
try {
$searchResultsDto = $this->bulkService->performBulkSearch([$part], $fieldMappingDtos, $prefetchDetails);
} catch (\Exception $searchException) {
// Handle "no search results found" as a normal case, not an error
if (str_contains($searchException->getMessage(), 'No search results found')) {
$searchResultsDto = null;
} else {
throw $searchException;
}
}
// Update the job's search results for this specific part efficiently
$this->updatePartSearchResults($job, $searchResultsDto[0] ?? null);
// Prefetch details if requested
if ($prefetchDetails && $searchResultsDto !== null) {
$this->bulkService->prefetchDetailsForResults($searchResultsDto);
}
$this->entityManager->flush();
// Return the new results for this part
$newResults = $searchResultsDto[0] ?? null;
return $this->json([
'success' => true,
'part_id' => $partId,
'results_count' => $newResults ? $newResults->getResultCount() : 0,
'errors_count' => $newResults ? $newResults->getErrorCount() : 0,
'message' => 'Part research completed successfully'
]);
} catch (\Exception $e) {
return $this->createErrorResponse(
'Research failed: ' . $e->getMessage(),
500,
[
'job_id' => $jobId,
'part_id' => $partId,
'exception' => $e->getMessage()
]
);
}
}
#[Route('/job/{jobId}/research-all', name: 'bulk_info_provider_research_all', methods: ['POST'])]
public function researchAllParts(int $jobId): JsonResponse
{
$job = $this->validateJobAccess($jobId);
if (!$job) {
return $this->createErrorResponse('Job not found or access denied', 404, ['job_id' => $jobId]);
}
// Get all parts that are not completed or skipped
$parts = [];
foreach ($job->getJobParts() as $jobPart) {
if (!$jobPart->isCompleted() && !$jobPart->isSkipped()) {
$parts[] = $jobPart->getPart();
}
}
if (empty($parts)) {
return $this->json([
'success' => true,
'message' => 'No parts to research',
'researched_count' => 0
]);
}
try {
$fieldMappingDtos = $job->getFieldMappings();
$prefetchDetails = $job->isPrefetchDetails();
// Process in batches to reduce memory usage for large operations
$allResults = new BulkSearchResponseDTO(partResults: []);
$batches = array_chunk($parts, $this->bulkImportBatchSize);
foreach ($batches as $batch) {
$batchResultsDto = $this->bulkService->performBulkSearch($batch, $fieldMappingDtos, $prefetchDetails);
$allResults = BulkSearchResponseDTO::merge($allResults, $batchResultsDto);
// Properly manage entity manager memory without losing state
$jobId = $job->getId();
//$this->entityManager->clear(); //TODO: This seems to cause problems with the user relation, when trying to flush later
$job = $this->entityManager->find(BulkInfoProviderImportJob::class, $jobId);
}
// Update the job's search results
$job->setSearchResults($allResults);
// Prefetch details if requested
if ($prefetchDetails) {
$this->bulkService->prefetchDetailsForResults($allResults);
}
$this->entityManager->flush();
return $this->json([
'success' => true,
'researched_count' => count($parts),
'message' => sprintf('Successfully researched %d parts', count($parts))
]);
} catch (\Exception $e) {
return $this->createErrorResponse(
'Bulk research failed: ' . $e->getMessage(),
500,
[
'job_id' => $jobId,
'part_count' => count($parts),
'exception' => $e->getMessage()
]
);
}
}
}

134
src/Controller/PartController.php
View file

@ -64,14 +64,16 @@ use Symfony\Contracts\Translation\TranslatorInterface;
use function Symfony\Component\Translation\t;
#[Route(path: '/part')]
class PartController extends AbstractController
final class PartController extends AbstractController
{
public function __construct(protected PricedetailHelper $pricedetailHelper,
protected PartPreviewGenerator $partPreviewGenerator,
public function __construct(
private readonly PricedetailHelper $pricedetailHelper,
private readonly PartPreviewGenerator $partPreviewGenerator,
private readonly TranslatorInterface $translator,
private readonly AttachmentSubmitHandler $attachmentSubmitHandler, private readonly EntityManagerInterface $em,
protected EventCommentHelper $commentHelper, private readonly PartInfoSettings $partInfoSettings)
{
private readonly AttachmentSubmitHandler $attachmentSubmitHandler,
private readonly EntityManagerInterface $em,
private readonly EventCommentHelper $commentHelper
) {
}
/**
@ -80,9 +82,16 @@ class PartController extends AbstractController
*/
#[Route(path: '/{id}/info/{timestamp}', name: 'part_info')]
#[Route(path: '/{id}', requirements: ['id' => '\d+'])]
public function show(Part $part, Request $request, TimeTravel $timeTravel, HistoryHelper $historyHelper,
DataTableFactory $dataTable, ParameterExtractor $parameterExtractor, PartLotWithdrawAddHelper $withdrawAddHelper, ?string $timestamp = null): Response
{
public function show(
Part $part,
Request $request,
TimeTravel $timeTravel,
HistoryHelper $historyHelper,
DataTableFactory $dataTable,
ParameterExtractor $parameterExtractor,
PartLotWithdrawAddHelper $withdrawAddHelper,
?string $timestamp = null
): Response {
$this->denyAccessUnlessGranted('read', $part);
$timeTravel_timestamp = null;
@ -132,7 +141,43 @@ class PartController extends AbstractController
{
$this->denyAccessUnlessGranted('edit', $part);
return $this->renderPartForm('edit', $request, $part);
// Check if this is part of a bulk import job
$jobId = $request->query->get('jobId');
$bulkJob = null;
if ($jobId) {
$bulkJob = $this->em->getRepository(\App\Entity\InfoProviderSystem\BulkInfoProviderImportJob::class)->find($jobId);
// Verify user owns this job
if ($bulkJob && $bulkJob->getCreatedBy() !== $this->getUser()) {
$bulkJob = null;
}
}
return $this->renderPartForm('edit', $request, $part, [], [
'bulk_job' => $bulkJob
]);
}
#[Route(path: '/{id}/bulk-import-complete/{jobId}', name: 'part_bulk_import_complete', methods: ['POST'])]
public function markBulkImportComplete(Part $part, int $jobId, Request $request): Response
{
$this->denyAccessUnlessGranted('edit', $part);
if (!$this->isCsrfTokenValid('bulk_complete_' . $part->getId(), $request->request->get('_token'))) {
throw $this->createAccessDeniedException('Invalid CSRF token');
}
$bulkJob = $this->em->getRepository(\App\Entity\InfoProviderSystem\BulkInfoProviderImportJob::class)->find($jobId);
if (!$bulkJob || $bulkJob->getCreatedBy() !== $this->getUser()) {
throw $this->createNotFoundException('Bulk import job not found');
}
$bulkJob->markPartAsCompleted($part->getId());
$this->em->persist($bulkJob);
$this->em->flush();
$this->addFlash('success', 'Part marked as completed in bulk import');
return $this->redirectToRoute('bulk_info_provider_step2', ['jobId' => $jobId]);
}
#[Route(path: '/{id}/delete', name: 'part_delete', methods: ['DELETE'])]
@ -140,7 +185,7 @@ class PartController extends AbstractController
{
$this->denyAccessUnlessGranted('delete', $part);
if ($this->isCsrfTokenValid('delete'.$part->getID(), $request->request->get('_token'))) {
if ($this->isCsrfTokenValid('delete' . $part->getID(), $request->request->get('_token'))) {
$this->commentHelper->setMessage($request->request->get('log_comment', null));
@ -159,11 +204,15 @@ class PartController extends AbstractController
#[Route(path: '/new', name: 'part_new')]
#[Route(path: '/{id}/clone', name: 'part_clone')]
#[Route(path: '/new_build_part/{project_id}', name: 'part_new_build_part')]
public function new(Request $request, EntityManagerInterface $em, TranslatorInterface $translator,
AttachmentSubmitHandler $attachmentSubmitHandler, ProjectBuildPartHelper $projectBuildPartHelper,
public function new(
Request $request,
EntityManagerInterface $em,
TranslatorInterface $translator,
AttachmentSubmitHandler $attachmentSubmitHandler,
ProjectBuildPartHelper $projectBuildPartHelper,
#[MapEntity(mapping: ['id' => 'id'])] ?Part $part = null,
#[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null): Response
{
#[MapEntity(mapping: ['project_id' => 'id'])] ?Project $project = null
): Response {
if ($part instanceof Part) {
//Clone part
@ -258,9 +307,14 @@ class PartController extends AbstractController
}
#[Route(path: '/{id}/from_info_provider/{providerKey}/{providerId}/update', name: 'info_providers_update_part', requirements: ['providerId' => '.+'])]
public function updateFromInfoProvider(Part $part, Request $request, string $providerKey, string $providerId,
PartInfoRetriever $infoRetriever, PartMerger $partMerger): Response
{
public function updateFromInfoProvider(
Part $part,
Request $request,
string $providerKey,
string $providerId,
PartInfoRetriever $infoRetriever,
PartMerger $partMerger
): Response {
$this->denyAccessUnlessGranted('edit', $part);
$this->denyAccessUnlessGranted('@info_providers.create_parts');
@ -274,10 +328,22 @@ class PartController extends AbstractController
$this->addFlash('notice', t('part.merge.flash.please_review'));
// Check if this is part of a bulk import job
$jobId = $request->query->get('jobId');
$bulkJob = null;
if ($jobId) {
$bulkJob = $this->em->getRepository(\App\Entity\InfoProviderSystem\BulkInfoProviderImportJob::class)->find($jobId);
// Verify user owns this job
if ($bulkJob && $bulkJob->getCreatedBy() !== $this->getUser()) {
$bulkJob = null;
}
}
return $this->renderPartForm('update_from_ip', $request, $part, [
'info_provider_dto' => $dto,
], [
'tname_before' => $old_name
'tname_before' => $old_name,
'bulk_job' => $bulkJob
]);
}
@ -312,7 +378,7 @@ class PartController extends AbstractController
} catch (AttachmentDownloadException $attachmentDownloadException) {
$this->addFlash(
'error',
$this->translator->trans('attachment.download_failed').' '.$attachmentDownloadException->getMessage()
$this->translator->trans('attachment.download_failed') . ' ' . $attachmentDownloadException->getMessage()
);
}
}
@ -353,6 +419,12 @@ class PartController extends AbstractController
return $this->redirectToRoute('part_new');
}
// Check if we're in bulk import mode and preserve jobId
$jobId = $request->query->get('jobId');
if ($jobId && isset($merge_infos['bulk_job'])) {
return $this->redirectToRoute('part_edit', ['id' => $new_part->getID(), 'jobId' => $jobId]);
}
return $this->redirectToRoute('part_edit', ['id' => $new_part->getID()]);
}
@ -371,13 +443,17 @@ class PartController extends AbstractController
$template = 'parts/edit/update_from_ip.html.twig';
}
return $this->render($template,
return $this->render(
$template,
[
'part' => $new_part,
'form' => $form,
'merge_old_name' => $merge_infos['tname_before'] ?? null,
'merge_other' => $merge_infos['other_part'] ?? null
]);
'merge_other' => $merge_infos['other_part'] ?? null,
'bulk_job' => $merge_infos['bulk_job'] ?? null,
'jobId' => $request->query->get('jobId')
]
);
}
@ -387,17 +463,17 @@ class PartController extends AbstractController
if ($this->isCsrfTokenValid('part_withraw' . $part->getID(), $request->request->get('_csfr'))) {
//Retrieve partlot from the request
$partLot = $em->find(PartLot::class, $request->request->get('lot_id'));
if(!$partLot instanceof PartLot) {
if (!$partLot instanceof PartLot) {
throw new \RuntimeException('Part lot not found!');
}
//Ensure that the partlot belongs to the part
if($partLot->getPart() !== $part) {
if ($partLot->getPart() !== $part) {
throw new \RuntimeException("The origin partlot does not belong to the part!");
}
//Try to determine the target lot (used for move actions), if the parameter is existing
$targetId = $request->request->get('target_id', null);
$targetLot = $targetId ? $em->find(PartLot::class, $targetId) : null;
$targetLot = $targetId ? $em->find(PartLot::class, $targetId) : null;
if ($targetLot && $targetLot->getPart() !== $part) {
throw new \RuntimeException("The target partlot does not belong to the part!");
}
@ -411,12 +487,12 @@ class PartController extends AbstractController
$timestamp = null;
$timestamp_str = $request->request->getString('timestamp', '');
//Try to parse the timestamp
if($timestamp_str !== '') {
if ($timestamp_str !== '') {
$timestamp = new DateTime($timestamp_str);
}
//Ensure that the timestamp is not in the future
if($timestamp !== null && $timestamp > new DateTime("+20min")) {
if ($timestamp !== null && $timestamp > new DateTime("+20min")) {
throw new \LogicException("The timestamp must not be in the future!");
}
@ -460,7 +536,7 @@ class PartController extends AbstractController
err:
//If a redirect was passed, then redirect there
if($request->request->get('_redirect')) {
if ($request->request->get('_redirect')) {
return $this->redirect($request->request->get('_redirect'));
}
//Otherwise just redirect to the part page

59
src/DataTables/Filters/Constraints/Part/BulkImportJobExistsConstraint.php Normal file
View file

@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\DataTables\Filters\Constraints\Part;
use App\DataTables\Filters\Constraints\BooleanConstraint;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use Doctrine\ORM\QueryBuilder;
class BulkImportJobExistsConstraint extends BooleanConstraint
{
public function __construct()
{
parent::__construct('bulk_import_job_exists');
}
public function apply(QueryBuilder $queryBuilder): void
{
// Do not apply a filter if value is null (filter is set to ignore)
if (!$this->isEnabled()) {
return;
}
// Use EXISTS subquery to avoid join conflicts
$existsSubquery = $queryBuilder->getEntityManager()->createQueryBuilder();
$existsSubquery->select('1')
->from(BulkInfoProviderImportJobPart::class, 'bip_exists')
->where('bip_exists.part = part.id');
if ($this->value === true) {
// Filter for parts that ARE in bulk import jobs
$queryBuilder->andWhere('EXISTS (' . $existsSubquery->getDQL() . ')');
} else {
// Filter for parts that are NOT in bulk import jobs
$queryBuilder->andWhere('NOT EXISTS (' . $existsSubquery->getDQL() . ')');
}
}
}

64
src/DataTables/Filters/Constraints/Part/BulkImportJobStatusConstraint.php Normal file
View file

@ -0,0 +1,64 @@
<?php
declare(strict_types=1);
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\DataTables\Filters\Constraints\Part;
use App\DataTables\Filters\Constraints\AbstractConstraint;
use App\DataTables\Filters\Constraints\ChoiceConstraint;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use Doctrine\ORM\QueryBuilder;
class BulkImportJobStatusConstraint extends ChoiceConstraint
{
public function __construct()
{
parent::__construct('bulk_import_job_status');
}
public function apply(QueryBuilder $queryBuilder): void
{
// Do not apply a filter if values are empty or operator is null
if (!$this->isEnabled()) {
return;
}
// Use EXISTS subquery to check if part has a job with the specified status(es)
$existsSubquery = $queryBuilder->getEntityManager()->createQueryBuilder();
$existsSubquery->select('1')
->from(BulkInfoProviderImportJobPart::class, 'bip_status')
->join('bip_status.job', 'job_status')
->where('bip_status.part = part.id');
// Add status conditions based on operator
if ($this->operator === 'ANY') {
$existsSubquery->andWhere('job_status.status IN (:job_status_values)');
$queryBuilder->andWhere('EXISTS (' . $existsSubquery->getDQL() . ')');
$queryBuilder->setParameter('job_status_values', $this->value);
} elseif ($this->operator === 'NONE') {
$existsSubquery->andWhere('job_status.status IN (:job_status_values)');
$queryBuilder->andWhere('NOT EXISTS (' . $existsSubquery->getDQL() . ')');
$queryBuilder->setParameter('job_status_values', $this->value);
}
}
}

61
src/DataTables/Filters/Constraints/Part/BulkImportPartStatusConstraint.php Normal file
View file

@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\DataTables\Filters\Constraints\Part;
use App\DataTables\Filters\Constraints\ChoiceConstraint;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use Doctrine\ORM\QueryBuilder;
class BulkImportPartStatusConstraint extends ChoiceConstraint
{
public function __construct()
{
parent::__construct('bulk_import_part_status');
}
public function apply(QueryBuilder $queryBuilder): void
{
// Do not apply a filter if values are empty or operator is null
if (!$this->isEnabled()) {
return;
}
// Use EXISTS subquery to check if part has the specified status(es)
$existsSubquery = $queryBuilder->getEntityManager()->createQueryBuilder();
$existsSubquery->select('1')
->from(BulkInfoProviderImportJobPart::class, 'bip_part_status')
->where('bip_part_status.part = part.id');
// Add status conditions based on operator
if ($this->operator === 'ANY') {
$existsSubquery->andWhere('bip_part_status.status IN (:part_status_values)');
$queryBuilder->andWhere('EXISTS (' . $existsSubquery->getDQL() . ')');
$queryBuilder->setParameter('part_status_values', $this->value);
} elseif ($this->operator === 'NONE') {
$existsSubquery->andWhere('bip_part_status.status IN (:part_status_values)');
$queryBuilder->andWhere('NOT EXISTS (' . $existsSubquery->getDQL() . ')');
$queryBuilder->setParameter('part_status_values', $this->value);
}
}
}

18
src/DataTables/Filters/PartFilter.php
View file

@ -29,6 +29,9 @@ use App\DataTables\Filters\Constraints\DateTimeConstraint;
use App\DataTables\Filters\Constraints\EntityConstraint;
use App\DataTables\Filters\Constraints\IntConstraint;
use App\DataTables\Filters\Constraints\NumberConstraint;
use App\DataTables\Filters\Constraints\Part\BulkImportJobExistsConstraint;
use App\DataTables\Filters\Constraints\Part\BulkImportJobStatusConstraint;
use App\DataTables\Filters\Constraints\Part\BulkImportPartStatusConstraint;
use App\DataTables\Filters\Constraints\Part\LessThanDesiredConstraint;
use App\DataTables\Filters\Constraints\Part\ParameterConstraint;
use App\DataTables\Filters\Constraints\Part\TagsConstraint;
@ -102,6 +105,14 @@ class PartFilter implements FilterInterface
public readonly TextConstraint $bomName;
public readonly TextConstraint $bomComment;
/*************************************************
* Bulk Import Job tab
*************************************************/
public readonly BulkImportJobExistsConstraint $inBulkImportJob;
public readonly BulkImportJobStatusConstraint $bulkImportJobStatus;
public readonly BulkImportPartStatusConstraint $bulkImportPartStatus;
public function __construct(NodesListBuilder $nodesListBuilder)
{
//Must be done for every new set of attachment filters, to ensure deterministic parameter names.
@ -130,7 +141,7 @@ class PartFilter implements FilterInterface
*/
$this->amountSum = (new IntConstraint('(
SELECT COALESCE(SUM(__partLot.amount), 0.0)
FROM '.PartLot::class.' __partLot
FROM ' . PartLot::class . ' __partLot
WHERE __partLot.part = part.id
AND __partLot.instock_unknown = false
AND (__partLot.expiration_date IS NULL OR __partLot.expiration_date > CURRENT_DATE())
@ -166,6 +177,11 @@ class PartFilter implements FilterInterface
$this->bomName = new TextConstraint('_projectBomEntries.name');
$this->bomComment = new TextConstraint('_projectBomEntries.comment');
// Bulk Import Job filters
$this->inBulkImportJob = new BulkImportJobExistsConstraint();
$this->bulkImportJobStatus = new BulkImportJobStatusConstraint();
$this->bulkImportPartStatus = new BulkImportPartStatusConstraint();
}
public function apply(QueryBuilder $queryBuilder): void

25
src/DataTables/PartsDataTable.php
View file

@ -142,23 +142,25 @@ final class PartsDataTable implements DataTableTypeInterface
'label' => $this->translator->trans('part.table.storeLocations'),
//We need to use a aggregate function to get the first store location, as we have a one-to-many relation
'orderField' => 'NATSORT(MIN(_storelocations.name))',
'render' => fn ($value, Part $context) => $this->partDataTableHelper->renderStorageLocations($context),
'render' => fn($value, Part $context) => $this->partDataTableHelper->renderStorageLocations($context),
], alias: 'storage_location')
->add('amount', TextColumn::class, [
'label' => $this->translator->trans('part.table.amount'),
'render' => fn ($value, Part $context) => $this->partDataTableHelper->renderAmount($context),
'render' => fn($value, Part $context) => $this->partDataTableHelper->renderAmount($context),
'orderField' => 'amountSum'
])
->add('minamount', TextColumn::class, [
'label' => $this->translator->trans('part.table.minamount'),
'render' => fn($value, Part $context): string => htmlspecialchars($this->amountFormatter->format($value,
$context->getPartUnit())),
'render' => fn($value, Part $context): string => htmlspecialchars($this->amountFormatter->format(
$value,
$context->getPartUnit()
)),
])
->add('partUnit', TextColumn::class, [
'label' => $this->translator->trans('part.table.partUnit'),
'orderField' => 'NATSORT(_partUnit.name)',
'render' => function($value, Part $context): string {
'render' => function ($value, Part $context): string {
$partUnit = $context->getPartUnit();
if ($partUnit === null) {
return '';
@ -167,7 +169,7 @@ final class PartsDataTable implements DataTableTypeInterface
$tmp = htmlspecialchars($partUnit->getName());
if ($partUnit->getUnit()) {
$tmp .= ' ('.htmlspecialchars($partUnit->getUnit()).')';
$tmp .= ' (' . htmlspecialchars($partUnit->getUnit()) . ')';
}
return $tmp;
}
@ -230,7 +232,7 @@ final class PartsDataTable implements DataTableTypeInterface
}
if (count($projects) > $max) {
$tmp .= ", + ".(count($projects) - $max);
$tmp .= ", + " . (count($projects) - $max);
}
return $tmp;
@ -366,7 +368,7 @@ final class PartsDataTable implements DataTableTypeInterface
$builder->addSelect(
'(
SELECT COALESCE(SUM(partLot.amount), 0.0)
FROM '.PartLot::class.' partLot
FROM ' . PartLot::class . ' partLot
WHERE partLot.part = part.id
AND partLot.instock_unknown = false
AND (partLot.expiration_date IS NULL OR partLot.expiration_date > CURRENT_DATE())
@ -423,6 +425,13 @@ final class PartsDataTable implements DataTableTypeInterface
//Do not group by many-to-* relations, as it would restrict the COUNT having clauses to be maximum 1
//$builder->addGroupBy('_projectBomEntries');
}
if (str_contains($dql, '_jobPart')) {
$builder->leftJoin('part.bulkImportJobParts', '_jobPart');
$builder->leftJoin('_jobPart.job', '_bulkImportJob');
//Do not group by many-to-* relations, as it would restrict the COUNT having clauses to be maximum 1
//$builder->addGroupBy('_jobPart');
//$builder->addGroupBy('_bulkImportJob');
}
return $builder;
}

35
src/Entity/InfoProviderSystem/BulkImportJobStatus.php Normal file
View file

@ -0,0 +1,35 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Entity\InfoProviderSystem;
use Symfony\Contracts\Translation\TranslatableInterface;
use Symfony\Contracts\Translation\TranslatorInterface;
enum BulkImportJobStatus: string
{
case PENDING = 'pending';
case IN_PROGRESS = 'in_progress';
case COMPLETED = 'completed';
case STOPPED = 'stopped';
case FAILED = 'failed';
}

32
src/Entity/InfoProviderSystem/BulkImportPartStatus.php Normal file
View file

@ -0,0 +1,32 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Entity\InfoProviderSystem;
enum BulkImportPartStatus: string
{
case PENDING = 'pending';
case COMPLETED = 'completed';
case SKIPPED = 'skipped';
case FAILED = 'failed';
}

449
src/Entity/InfoProviderSystem/BulkInfoProviderImportJob.php Normal file
View file

@ -0,0 +1,449 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Entity\InfoProviderSystem;
use App\Entity\Base\AbstractDBElement;
use App\Entity\Parts\Part;
use App\Entity\UserSystem\User;
use App\Services\InfoProviderSystem\DTOs\BulkSearchFieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\Common\Collections\Collection;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\Mapping as ORM;
#[ORM\Entity]
#[ORM\Table(name: 'bulk_info_provider_import_jobs')]
class BulkInfoProviderImportJob extends AbstractDBElement
{
#[ORM\Column(type: Types::TEXT)]
private string $name = '';
#[ORM\Column(type: Types::JSON)]
private array $fieldMappings = [];
/**
* @var BulkSearchFieldMappingDTO[] The deserialized field mappings DTOs, cached for performance
*/
private ?array $fieldMappingsDTO = null;
#[ORM\Column(type: Types::JSON)]
private array $searchResults = [];
/**
* @var BulkSearchResponseDTO|null The deserialized search results DTO, cached for performance
*/
private ?BulkSearchResponseDTO $searchResultsDTO = null;
#[ORM\Column(type: Types::STRING, length: 20, enumType: BulkImportJobStatus::class)]
private BulkImportJobStatus $status = BulkImportJobStatus::PENDING;
#[ORM\Column(type: Types::DATETIME_IMMUTABLE)]
private \DateTimeImmutable $createdAt;
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
private ?\DateTimeImmutable $completedAt = null;
#[ORM\Column(type: Types::BOOLEAN)]
private bool $prefetchDetails = false;
#[ORM\ManyToOne(targetEntity: User::class)]
#[ORM\JoinColumn(nullable: false)]
private ?User $createdBy = null;
/** @var Collection<int, BulkInfoProviderImportJobPart> */
#[ORM\OneToMany(targetEntity: BulkInfoProviderImportJobPart::class, mappedBy: 'job', cascade: ['persist', 'remove'], orphanRemoval: true)]
private Collection $jobParts;
public function __construct()
{
$this->createdAt = new \DateTimeImmutable();
$this->jobParts = new ArrayCollection();
}
public function getName(): string
{
return $this->name;
}
public function getDisplayNameKey(): string
{
return 'info_providers.bulk_import.job_name_template';
}
public function getDisplayNameParams(): array
{
return ['%count%' => $this->getPartCount()];
}
public function getFormattedTimestamp(): string
{
return $this->createdAt->format('Y-m-d H:i:s');
}
public function setName(string $name): self
{
$this->name = $name;
return $this;
}
public function getJobParts(): Collection
{
return $this->jobParts;
}
public function addJobPart(BulkInfoProviderImportJobPart $jobPart): self
{
if (!$this->jobParts->contains($jobPart)) {
$this->jobParts->add($jobPart);
$jobPart->setJob($this);
}
return $this;
}
public function removeJobPart(BulkInfoProviderImportJobPart $jobPart): self
{
if ($this->jobParts->removeElement($jobPart)) {
if ($jobPart->getJob() === $this) {
$jobPart->setJob(null);
}
}
return $this;
}
public function getPartIds(): array
{
return $this->jobParts->map(fn($jobPart) => $jobPart->getPart()->getId())->toArray();
}
public function setPartIds(array $partIds): self
{
// This method is kept for backward compatibility but should be replaced with addJobPart
// Clear existing job parts
$this->jobParts->clear();
// Add new job parts (this would need the actual Part entities, not just IDs)
// This is a simplified implementation - in practice, you'd want to pass Part entities
return $this;
}
public function addPart(Part $part): self
{
$jobPart = new BulkInfoProviderImportJobPart($this, $part);
$this->addJobPart($jobPart);
return $this;
}
/**
* @return BulkSearchFieldMappingDTO[] The deserialized field mappings
*/
public function getFieldMappings(): array
{
if ($this->fieldMappingsDTO === null) {
// Lazy load the DTOs from the raw JSON data
$this->fieldMappingsDTO = array_map(
static fn($data) => BulkSearchFieldMappingDTO::fromSerializableArray($data),
$this->fieldMappings
);
}
return $this->fieldMappingsDTO;
}
/**
* @param BulkSearchFieldMappingDTO[] $fieldMappings
* @return $this
*/
public function setFieldMappings(array $fieldMappings): self
{
//Ensure that we are dealing with the objects here
if (count($fieldMappings) > 0 && !$fieldMappings[0] instanceof BulkSearchFieldMappingDTO) {
throw new \InvalidArgumentException('Expected an array of FieldMappingDTO objects');
}
$this->fieldMappingsDTO = $fieldMappings;
$this->fieldMappings = array_map(
static fn(BulkSearchFieldMappingDTO $dto) => $dto->toSerializableArray(),
$fieldMappings
);
return $this;
}
public function getSearchResultsRaw(): array
{
return $this->searchResults;
}
public function setSearchResultsRaw(array $searchResults): self
{
$this->searchResults = $searchResults;
return $this;
}
public function setSearchResults(BulkSearchResponseDTO $searchResponse): self
{
$this->searchResultsDTO = $searchResponse;
$this->searchResults = $searchResponse->toSerializableRepresentation();
return $this;
}
public function getSearchResults(EntityManagerInterface $entityManager): BulkSearchResponseDTO
{
if ($this->searchResultsDTO === null) {
// Lazy load the DTO from the raw JSON data
$this->searchResultsDTO = BulkSearchResponseDTO::fromSerializableRepresentation($this->searchResults, $entityManager);
}
return $this->searchResultsDTO;
}
public function hasSearchResults(): bool
{
return !empty($this->searchResults);
}
public function getStatus(): BulkImportJobStatus
{
return $this->status;
}
public function setStatus(BulkImportJobStatus $status): self
{
$this->status = $status;
return $this;
}
public function getCreatedAt(): \DateTimeImmutable
{
return $this->createdAt;
}
public function getCompletedAt(): ?\DateTimeImmutable
{
return $this->completedAt;
}
public function setCompletedAt(?\DateTimeImmutable $completedAt): self
{
$this->completedAt = $completedAt;
return $this;
}
public function isPrefetchDetails(): bool
{
return $this->prefetchDetails;
}
public function setPrefetchDetails(bool $prefetchDetails): self
{
$this->prefetchDetails = $prefetchDetails;
return $this;
}
public function getCreatedBy(): User
{
return $this->createdBy;
}
public function setCreatedBy(User $createdBy): self
{
$this->createdBy = $createdBy;
return $this;
}
public function getProgress(): array
{
$progress = [];
foreach ($this->jobParts as $jobPart) {
$progressData = [
'status' => $jobPart->getStatus()->value
];
// Only include completed_at if it's not null
if ($jobPart->getCompletedAt() !== null) {
$progressData['completed_at'] = $jobPart->getCompletedAt()->format('c');
}
// Only include reason if it's not null
if ($jobPart->getReason() !== null) {
$progressData['reason'] = $jobPart->getReason();
}
$progress[$jobPart->getPart()->getId()] = $progressData;
}
return $progress;
}
public function markAsCompleted(): self
{
$this->status = BulkImportJobStatus::COMPLETED;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsFailed(): self
{
$this->status = BulkImportJobStatus::FAILED;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsStopped(): self
{
$this->status = BulkImportJobStatus::STOPPED;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsInProgress(): self
{
$this->status = BulkImportJobStatus::IN_PROGRESS;
return $this;
}
public function isPending(): bool
{
return $this->status === BulkImportJobStatus::PENDING;
}
public function isInProgress(): bool
{
return $this->status === BulkImportJobStatus::IN_PROGRESS;
}
public function isCompleted(): bool
{
return $this->status === BulkImportJobStatus::COMPLETED;
}
public function isFailed(): bool
{
return $this->status === BulkImportJobStatus::FAILED;
}
public function isStopped(): bool
{
return $this->status === BulkImportJobStatus::STOPPED;
}
public function canBeStopped(): bool
{
return $this->status === BulkImportJobStatus::PENDING || $this->status === BulkImportJobStatus::IN_PROGRESS;
}
public function getPartCount(): int
{
return $this->jobParts->count();
}
public function getResultCount(): int
{
$count = 0;
foreach ($this->searchResults as $partResult) {
$count += count($partResult['search_results'] ?? []);
}
return $count;
}
public function markPartAsCompleted(int $partId): self
{
$jobPart = $this->findJobPartByPartId($partId);
if ($jobPart) {
$jobPart->markAsCompleted();
}
return $this;
}
public function markPartAsSkipped(int $partId, string $reason = ''): self
{
$jobPart = $this->findJobPartByPartId($partId);
if ($jobPart) {
$jobPart->markAsSkipped($reason);
}
return $this;
}
public function markPartAsPending(int $partId): self
{
$jobPart = $this->findJobPartByPartId($partId);
if ($jobPart) {
$jobPart->markAsPending();
}
return $this;
}
public function isPartCompleted(int $partId): bool
{
$jobPart = $this->findJobPartByPartId($partId);
return $jobPart ? $jobPart->isCompleted() : false;
}
public function isPartSkipped(int $partId): bool
{
$jobPart = $this->findJobPartByPartId($partId);
return $jobPart ? $jobPart->isSkipped() : false;
}
public function getCompletedPartsCount(): int
{
return $this->jobParts->filter(fn($jobPart) => $jobPart->isCompleted())->count();
}
public function getSkippedPartsCount(): int
{
return $this->jobParts->filter(fn($jobPart) => $jobPart->isSkipped())->count();
}
private function findJobPartByPartId(int $partId): ?BulkInfoProviderImportJobPart
{
foreach ($this->jobParts as $jobPart) {
if ($jobPart->getPart()->getId() === $partId) {
return $jobPart;
}
}
return null;
}
public function getProgressPercentage(): float
{
$total = $this->getPartCount();
if ($total === 0) {
return 100.0;
}
$completed = $this->getCompletedPartsCount() + $this->getSkippedPartsCount();
return round(($completed / $total) * 100, 1);
}
public function isAllPartsCompleted(): bool
{
$total = $this->getPartCount();
if ($total === 0) {
return true;
}
$completed = $this->getCompletedPartsCount() + $this->getSkippedPartsCount();
return $completed >= $total;
}
}

182
src/Entity/InfoProviderSystem/BulkInfoProviderImportJobPart.php Normal file
View file

@ -0,0 +1,182 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\Entity\InfoProviderSystem;
use App\Entity\Base\AbstractDBElement;
use App\Entity\Parts\Part;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\Mapping as ORM;
#[ORM\Entity]
#[ORM\Table(name: 'bulk_info_provider_import_job_parts')]
#[ORM\UniqueConstraint(name: 'unique_job_part', columns: ['job_id', 'part_id'])]
class BulkInfoProviderImportJobPart extends AbstractDBElement
{
#[ORM\ManyToOne(targetEntity: BulkInfoProviderImportJob::class, inversedBy: 'jobParts')]
#[ORM\JoinColumn(nullable: false)]
private BulkInfoProviderImportJob $job;
#[ORM\ManyToOne(targetEntity: Part::class, inversedBy: 'bulkImportJobParts')]
#[ORM\JoinColumn(nullable: false)]
private Part $part;
#[ORM\Column(type: Types::STRING, length: 20, enumType: BulkImportPartStatus::class)]
private BulkImportPartStatus $status = BulkImportPartStatus::PENDING;
#[ORM\Column(type: Types::TEXT, nullable: true)]
private ?string $reason = null;
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
private ?\DateTimeImmutable $completedAt = null;
public function __construct(BulkInfoProviderImportJob $job, Part $part)
{
$this->job = $job;
$this->part = $part;
}
public function getJob(): BulkInfoProviderImportJob
{
return $this->job;
}
public function setJob(?BulkInfoProviderImportJob $job): self
{
$this->job = $job;
return $this;
}
public function getPart(): Part
{
return $this->part;
}
public function setPart(?Part $part): self
{
$this->part = $part;
return $this;
}
public function getStatus(): BulkImportPartStatus
{
return $this->status;
}
public function setStatus(BulkImportPartStatus $status): self
{
$this->status = $status;
return $this;
}
public function getReason(): ?string
{
return $this->reason;
}
public function setReason(?string $reason): self
{
$this->reason = $reason;
return $this;
}
public function getCompletedAt(): ?\DateTimeImmutable
{
return $this->completedAt;
}
public function setCompletedAt(?\DateTimeImmutable $completedAt): self
{
$this->completedAt = $completedAt;
return $this;
}
public function markAsCompleted(): self
{
$this->status = BulkImportPartStatus::COMPLETED;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsSkipped(string $reason = ''): self
{
$this->status = BulkImportPartStatus::SKIPPED;
$this->reason = $reason;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsFailed(string $reason = ''): self
{
$this->status = BulkImportPartStatus::FAILED;
$this->reason = $reason;
$this->completedAt = new \DateTimeImmutable();
return $this;
}
public function markAsPending(): self
{
$this->status = BulkImportPartStatus::PENDING;
$this->reason = null;
$this->completedAt = null;
return $this;
}
public function isPending(): bool
{
return $this->status === BulkImportPartStatus::PENDING;
}
public function isCompleted(): bool
{
return $this->status === BulkImportPartStatus::COMPLETED;
}
public function isSkipped(): bool
{
return $this->status === BulkImportPartStatus::SKIPPED;
}
public function isFailed(): bool
{
return $this->status === BulkImportPartStatus::FAILED;
}
}

6
src/Entity/LogSystem/LogTargetType.php
View file

@ -24,6 +24,8 @@ namespace App\Entity\LogSystem;
use App\Entity\Attachments\Attachment;
use App\Entity\Attachments\AttachmentType;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJob;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use App\Entity\LabelSystem\LabelProfile;
use App\Entity\Parameters\AbstractParameter;
use App\Entity\Parts\Category;
@ -67,6 +69,8 @@ enum LogTargetType: int
case LABEL_PROFILE = 19;
case PART_ASSOCIATION = 20;
case BULK_INFO_PROVIDER_IMPORT_JOB = 21;
case BULK_INFO_PROVIDER_IMPORT_JOB_PART = 22;
/**
* Returns the class name of the target type or null if the target type is NONE.
@ -96,6 +100,8 @@ enum LogTargetType: int
self::PARAMETER => AbstractParameter::class,
self::LABEL_PROFILE => LabelProfile::class,
self::PART_ASSOCIATION => PartAssociation::class,
self::BULK_INFO_PROVIDER_IMPORT_JOB => BulkInfoProviderImportJob::class,
self::BULK_INFO_PROVIDER_IMPORT_JOB_PART => BulkInfoProviderImportJobPart::class,
};
}

64
src/Entity/Parts/Part.php
View file

@ -22,8 +22,6 @@ declare(strict_types=1);
namespace App\Entity\Parts;
use App\ApiPlatform\Filter\TagFilter;
use Doctrine\Common\Collections\Criteria;
use ApiPlatform\Doctrine\Common\Filter\DateFilterInterface;
use ApiPlatform\Doctrine\Orm\Filter\BooleanFilter;
use ApiPlatform\Doctrine\Orm\Filter\DateFilter;
@ -40,10 +38,12 @@ use ApiPlatform\Serializer\Filter\PropertyFilter;
use App\ApiPlatform\Filter\EntityFilter;
use App\ApiPlatform\Filter\LikeFilter;
use App\ApiPlatform\Filter\PartStoragelocationFilter;
use App\ApiPlatform\Filter\TagFilter;
use App\Entity\Attachments\Attachment;
use App\Entity\Attachments\AttachmentContainingDBElement;
use App\Entity\Attachments\PartAttachment;
use App\Entity\EDA\EDAPartInfo;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use App\Entity\Parameters\ParametersTrait;
use App\Entity\Parameters\PartParameter;
use App\Entity\Parts\PartTraits\AdvancedPropertyTrait;
@ -59,6 +59,7 @@ use App\Repository\PartRepository;
use App\Validator\Constraints\UniqueObjectCollection;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\Common\Collections\Collection;
use Doctrine\Common\Collections\Criteria;
use Doctrine\ORM\Mapping as ORM;
use Symfony\Bridge\Doctrine\Validator\Constraints\UniqueEntity;
use Symfony\Component\Serializer\Annotation\Groups;
@ -83,8 +84,18 @@ use Symfony\Component\Validator\Context\ExecutionContextInterface;
#[ORM\Index(columns: ['ipn'], name: 'parts_idx_ipn')]
#[ApiResource(
operations: [
new Get(normalizationContext: ['groups' => ['part:read', 'provider_reference:read', 'api:basic:read', 'part_lot:read',
'orderdetail:read', 'pricedetail:read', 'parameter:read', 'attachment:read', 'eda_info:read'],
new Get(normalizationContext: [
'groups' => [
'part:read',
'provider_reference:read',
'api:basic:read',
'part_lot:read',
'orderdetail:read',
'pricedetail:read',
'parameter:read',
'attachment:read',
'eda_info:read'
],
'openapi_definition_name' => 'Read',
], security: 'is_granted("read", object)'),
new GetCollection(security: 'is_granted("@parts.read")'),
@ -92,7 +103,7 @@ use Symfony\Component\Validator\Context\ExecutionContextInterface;
new Patch(security: 'is_granted("edit", object)'),
new Delete(security: 'is_granted("delete", object)'),
],
normalizationContext: ['groups' => ['part:read', 'provider_reference:read', 'api:basic:read', 'part_lot:read'], 'openapi_definition_name' => 'Read'],
normalizationContext: ['groups' => ['part:read', 'provider_reference:read', 'api:basic:read', 'part_lot:read'], 'openapi_definition_name' => 'Read'],
denormalizationContext: ['groups' => ['part:write', 'api:basic:write', 'eda_info:write', 'attachment:write', 'parameter:write'], 'openapi_definition_name' => 'Write'],
)]
#[ApiFilter(PropertyFilter::class)]
@ -100,7 +111,7 @@ use Symfony\Component\Validator\Context\ExecutionContextInterface;
#[ApiFilter(PartStoragelocationFilter::class, properties: ["storage_location"])]
#[ApiFilter(LikeFilter::class, properties: ["name", "comment", "description", "ipn", "manufacturer_product_number"])]
#[ApiFilter(TagFilter::class, properties: ["tags"])]
#[ApiFilter(BooleanFilter::class, properties: ["favorite" , "needs_review"])]
#[ApiFilter(BooleanFilter::class, properties: ["favorite", "needs_review"])]
#[ApiFilter(RangeFilter::class, properties: ["mass", "minamount"])]
#[ApiFilter(DateFilter::class, strategy: DateFilterInterface::EXCLUDE_NULL)]
#[ApiFilter(OrderFilter::class, properties: ['name', 'id', 'addedDate', 'lastModified'])]
@ -160,6 +171,12 @@ class Part extends AttachmentContainingDBElement
#[Groups(['part:read'])]
protected ?\DateTimeImmutable $lastModified = null;
/**
* @var Collection<int, BulkInfoProviderImportJobPart>
*/
#[ORM\OneToMany(mappedBy: 'part', targetEntity: BulkInfoProviderImportJobPart::class, cascade: ['remove'], orphanRemoval: true)]
protected Collection $bulkImportJobParts;
public function __construct()
{
@ -172,6 +189,7 @@ class Part extends AttachmentContainingDBElement
$this->associated_parts_as_owner = new ArrayCollection();
$this->associated_parts_as_other = new ArrayCollection();
$this->bulkImportJobParts = new ArrayCollection();
//By default, the part has no provider
$this->providerReference = InfoProviderReference::noProvider();
@ -230,4 +248,38 @@ class Part extends AttachmentContainingDBElement
}
}
}
/**
* Get all bulk import job parts for this part
* @return Collection<int, BulkInfoProviderImportJobPart>
*/
public function getBulkImportJobParts(): Collection
{
return $this->bulkImportJobParts;
}
/**
* Add a bulk import job part to this part
*/
public function addBulkImportJobPart(BulkInfoProviderImportJobPart $jobPart): self
{
if (!$this->bulkImportJobParts->contains($jobPart)) {
$this->bulkImportJobParts->add($jobPart);
$jobPart->setPart($this);
}
return $this;
}
/**
* Remove a bulk import job part from this part
*/
public function removeBulkImportJobPart(BulkInfoProviderImportJobPart $jobPart): self
{
if ($this->bulkImportJobParts->removeElement($jobPart)) {
if ($jobPart->getPart() === $this) {
$jobPart->setPart(null);
}
}
return $this;
}
}

2
src/Form/AdminPages/ImportType.php
View file

@ -59,6 +59,8 @@ class ImportType extends AbstractType
'XML' => 'xml',
'CSV' => 'csv',
'YAML' => 'yaml',
'XLSX' => 'xlsx',
'XLS' => 'xls',
],
'label' => 'export.format',
'disabled' => $disabled,

6
src/Form/Filters/LogFilterType.php
View file

@ -100,7 +100,7 @@ class LogFilterType extends AbstractType
]);
$builder->add('user', UserEntityConstraintType::class, [
'label' => 'log.user',
'label' => 'log.user',
]);
$builder->add('targetType', EnumConstraintType::class, [
@ -128,11 +128,13 @@ class LogFilterType extends AbstractType
LogTargetType::PARAMETER => 'parameter.label',
LogTargetType::LABEL_PROFILE => 'label_profile.label',
LogTargetType::PART_ASSOCIATION => 'part_association.label',
LogTargetType::BULK_INFO_PROVIDER_IMPORT_JOB => 'bulk_info_provider_import_job.label',
LogTargetType::BULK_INFO_PROVIDER_IMPORT_JOB_PART => 'bulk_info_provider_import_job_part.label',
},
]);
$builder->add('targetId', NumberConstraintType::class, [
'label' => 'log.target_id',
'label' => 'log.target_id',
'min' => 1,
'step' => 1,
]);

34
src/Form/Filters/PartFilterType.php
View file

@ -22,9 +22,12 @@ declare(strict_types=1);
*/
namespace App\Form\Filters;
use App\DataTables\Filters\Constraints\Part\BulkImportPartStatusConstraint;
use App\DataTables\Filters\Constraints\Part\ParameterConstraint;
use App\DataTables\Filters\PartFilter;
use App\Entity\Attachments\AttachmentType;
use App\Entity\InfoProviderSystem\BulkImportJobStatus;
use App\Entity\InfoProviderSystem\BulkImportPartStatus;
use App\Entity\Parts\Category;
use App\Entity\Parts\Footprint;
use App\Entity\Parts\Manufacturer;
@ -33,8 +36,12 @@ use App\Entity\Parts\StorageLocation;
use App\Entity\Parts\Supplier;
use App\Entity\ProjectSystem\Project;
use App\Form\Filters\Constraints\BooleanConstraintType;
use App\Form\Filters\Constraints\BulkImportJobExistsConstraintType;
use App\Form\Filters\Constraints\BulkImportJobStatusConstraintType;
use App\Form\Filters\Constraints\BulkImportPartStatusConstraintType;
use App\Form\Filters\Constraints\ChoiceConstraintType;
use App\Form\Filters\Constraints\DateTimeConstraintType;
use App\Form\Filters\Constraints\EnumConstraintType;
use App\Form\Filters\Constraints\NumberConstraintType;
use App\Form\Filters\Constraints\ParameterConstraintType;
use App\Form\Filters\Constraints\StructuralEntityConstraintType;
@ -50,6 +57,8 @@ use Symfony\Component\Form\Extension\Core\Type\SubmitType;
use Symfony\Component\Form\FormBuilderInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;
use function Symfony\Component\Translation\t;
class PartFilterType extends AbstractType
{
public function __construct(private readonly Security $security)
@ -298,6 +307,31 @@ class PartFilterType extends AbstractType
}
/**************************************************************************
* Bulk Import Job tab
**************************************************************************/
if ($this->security->isGranted('@info_providers.create_parts')) {
$builder
->add('inBulkImportJob', BooleanConstraintType::class, [
'label' => 'part.filter.in_bulk_import_job',
])
->add('bulkImportJobStatus', EnumConstraintType::class, [
'enum_class' => BulkImportJobStatus::class,
'label' => 'part.filter.bulk_import_job_status',
'choice_label' => function (BulkImportJobStatus $value) {
return t('bulk_import.status.' . $value->value);
},
])
->add('bulkImportPartStatus', EnumConstraintType::class, [
'enum_class' => BulkImportPartStatus::class,
'label' => 'part.filter.bulk_import_part_status',
'choice_label' => function (BulkImportPartStatus $value) {
return t('bulk_import.part_status.' . $value->value);
},
])
;
}
$builder->add('submit', SubmitType::class, [
'label' => 'filter.submit',

62
src/Form/InfoProviderSystem/BulkProviderSearchType.php Normal file
View file

@ -0,0 +1,62 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use App\Entity\Parts\Part;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\Form\Extension\Core\Type\CollectionType;
use Symfony\Component\Form\Extension\Core\Type\HiddenType;
use Symfony\Component\Form\Extension\Core\Type\SubmitType;
use Symfony\Component\Form\FormBuilderInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;
class BulkProviderSearchType extends AbstractType
{
public function buildForm(FormBuilderInterface $builder, array $options): void
{
$parts = $options['parts'];
$builder->add('part_configurations', CollectionType::class, [
'entry_type' => PartProviderConfigurationType::class,
'entry_options' => [
'label' => false,
],
'allow_add' => false,
'allow_delete' => false,
'label' => false,
]);
$builder->add('submit', SubmitType::class, [
'label' => 'info_providers.bulk_search.submit'
]);
}
public function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefaults([
'parts' => [],
]);
$resolver->setRequired('parts');
}
}

75
src/Form/InfoProviderSystem/FieldToProviderMappingType.php Normal file
View file

@ -0,0 +1,75 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\Form\Extension\Core\Type\IntegerType;
use Symfony\Component\Form\FormBuilderInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;
class FieldToProviderMappingType extends AbstractType
{
public function buildForm(FormBuilderInterface $builder, array $options): void
{
$fieldChoices = $options['field_choices'] ?? [];
$builder->add('field', ChoiceType::class, [
'label' => 'info_providers.bulk_search.search_field',
'choices' => $fieldChoices,
'expanded' => false,
'multiple' => false,
'required' => false,
'placeholder' => 'info_providers.bulk_search.field.select',
]);
$builder->add('providers', ProviderSelectType::class, [
'label' => 'info_providers.bulk_search.providers',
'help' => 'info_providers.bulk_search.providers.help',
'required' => false,
]);
$builder->add('priority', IntegerType::class, [
'label' => 'info_providers.bulk_search.priority',
'help' => 'info_providers.bulk_search.priority.help',
'required' => false,
'data' => 1, // Default priority
'attr' => [
'min' => 1,
'max' => 10,
'class' => 'form-control-sm',
'style' => 'width: 80px;'
],
'constraints' => [
new \Symfony\Component\Validator\Constraints\Range(['min' => 1, 'max' => 10]),
],
]);
}
public function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefaults([
'field_choices' => [],
]);
}
}

67
src/Form/InfoProviderSystem/GlobalFieldMappingType.php Normal file
View file

@ -0,0 +1,67 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
use Symfony\Component\Form\Extension\Core\Type\CollectionType;
use Symfony\Component\Form\Extension\Core\Type\SubmitType;
use Symfony\Component\Form\FormBuilderInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;
class GlobalFieldMappingType extends AbstractType
{
public function buildForm(FormBuilderInterface $builder, array $options): void
{
$fieldChoices = $options['field_choices'] ?? [];
$builder->add('field_mappings', CollectionType::class, [
'entry_type' => FieldToProviderMappingType::class,
'entry_options' => [
'label' => false,
'field_choices' => $fieldChoices,
],
'allow_add' => true,
'allow_delete' => true,
'prototype' => true,
'label' => false,
]);
$builder->add('prefetch_details', CheckboxType::class, [
'label' => 'info_providers.bulk_import.prefetch_details',
'required' => false,
'help' => 'info_providers.bulk_import.prefetch_details_help',
]);
$builder->add('submit', SubmitType::class, [
'label' => 'info_providers.bulk_import.search.submit'
]);
}
public function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefaults([
'field_choices' => [],
]);
}
}

55
src/Form/InfoProviderSystem/PartProviderConfigurationType.php Normal file
View file

@ -0,0 +1,55 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\Form\Extension\Core\Type\HiddenType;
use Symfony\Component\Form\FormBuilderInterface;
class PartProviderConfigurationType extends AbstractType
{
public function buildForm(FormBuilderInterface $builder, array $options): void
{
$builder->add('part_id', HiddenType::class);
$builder->add('search_field', ChoiceType::class, [
'label' => 'info_providers.bulk_search.search_field',
'choices' => [
'info_providers.bulk_search.field.mpn' => 'mpn',
'info_providers.bulk_search.field.name' => 'name',
'info_providers.bulk_search.field.digikey_spn' => 'digikey_spn',
'info_providers.bulk_search.field.mouser_spn' => 'mouser_spn',
'info_providers.bulk_search.field.lcsc_spn' => 'lcsc_spn',
'info_providers.bulk_search.field.farnell_spn' => 'farnell_spn',
],
'expanded' => false,
'multiple' => false,
]);
$builder->add('providers', ProviderSelectType::class, [
'label' => 'info_providers.bulk_search.providers',
'help' => 'info_providers.bulk_search.providers.help',
]);
}
}

2
src/Form/InfoProviderSystem/ProviderSelectType.php
View file

@ -24,9 +24,7 @@ declare(strict_types=1);
namespace App\Form\InfoProviderSystem;
use App\Services\InfoProviderSystem\ProviderRegistry;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\ChoiceList\ChoiceList;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\OptionsResolver\Options;
use Symfony\Component\OptionsResolver\OptionsResolver;

14
src/Services/ElementTypeNameGenerator.php
View file

@ -22,13 +22,13 @@ declare(strict_types=1);
namespace App\Services;
use App\Entity\Attachments\AttachmentContainingDBElement;
use App\Entity\Attachments\Attachment;
use App\Entity\Attachments\AttachmentContainingDBElement;
use App\Entity\Attachments\AttachmentType;
use App\Entity\Base\AbstractDBElement;
use App\Entity\Contracts\NamedElementInterface;
use App\Entity\Parts\PartAssociation;
use App\Entity\ProjectSystem\Project;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJob;
use App\Entity\InfoProviderSystem\BulkInfoProviderImportJobPart;
use App\Entity\LabelSystem\LabelProfile;
use App\Entity\Parameters\AbstractParameter;
use App\Entity\Parts\Category;
@ -36,12 +36,14 @@ use App\Entity\Parts\Footprint;
use App\Entity\Parts\Manufacturer;
use App\Entity\Parts\MeasurementUnit;
use App\Entity\Parts\Part;
use App\Entity\Parts\PartAssociation;
use App\Entity\Parts\PartLot;
use App\Entity\Parts\StorageLocation;
use App\Entity\Parts\Supplier;
use App\Entity\PriceInformations\Currency;
use App\Entity\PriceInformations\Orderdetail;
use App\Entity\PriceInformations\Pricedetail;
use App\Entity\ProjectSystem\Project;
use App\Entity\ProjectSystem\ProjectBOMEntry;
use App\Entity\UserSystem\Group;
use App\Entity\UserSystem\User;
@ -79,6 +81,8 @@ class ElementTypeNameGenerator
AbstractParameter::class => $this->translator->trans('parameter.label'),
LabelProfile::class => $this->translator->trans('label_profile.label'),
PartAssociation::class => $this->translator->trans('part_association.label'),
BulkInfoProviderImportJob::class => $this->translator->trans('bulk_info_provider_import_job.label'),
BulkInfoProviderImportJobPart::class => $this->translator->trans('bulk_info_provider_import_job_part.label'),
];
}
@ -130,10 +134,10 @@ class ElementTypeNameGenerator
{
$type = $this->getLocalizedTypeLabel($entity);
if ($use_html) {
return '<i>'.$type.':</i> '.htmlspecialchars($entity->getName());
return '<i>' . $type . ':</i> ' . htmlspecialchars($entity->getName());
}
return $type.': '.$entity->getName();
return $type . ': ' . $entity->getName();
}

64
src/Services/EntityMergers/Mergers/PartMerger.php
View file

@ -100,7 +100,8 @@ class PartMerger implements EntityMergerInterface
return $target;
}
private function comparePartAssociations(PartAssociation $t, PartAssociation $o): bool {
private function comparePartAssociations(PartAssociation $t, PartAssociation $o): bool
{
//We compare the translation keys, as it contains info about the type and other type info
return $t->getOther() === $o->getOther()
&& $t->getTypeTranslationKey() === $o->getTypeTranslationKey();
@ -141,40 +142,39 @@ class PartMerger implements EntityMergerInterface
$owner->addAssociatedPartsAsOwner($clone);
}
// Merge orderdetails, considering same supplier+part number as duplicates
$this->mergeCollections($target, $other, 'orderdetails', function (Orderdetail $t, Orderdetail $o) {
//First check that the orderdetails infos are equal
$tmp = $t->getSupplier() === $o->getSupplier()
&& $t->getSupplierPartNr() === $o->getSupplierPartNr()
&& $t->getSupplierProductUrl(false) === $o->getSupplierProductUrl(false);
if (!$tmp) {
return false;
}
//Check if the pricedetails are equal
$t_pricedetails = $t->getPricedetails();
$o_pricedetails = $o->getPricedetails();
//Ensure that both pricedetails have the same length
if (count($t_pricedetails) !== count($o_pricedetails)) {
return false;
}
//Check if all pricedetails are equal
for ($n=0, $nMax = count($t_pricedetails); $n< $nMax; $n++) {
$t_price = $t_pricedetails->get($n);
$o_price = $o_pricedetails->get($n);
if (!$t_price->getPrice()->isEqualTo($o_price->getPrice())
|| $t_price->getCurrency() !== $o_price->getCurrency()
|| $t_price->getPriceRelatedQuantity() !== $o_price->getPriceRelatedQuantity()
|| $t_price->getMinDiscountQuantity() !== $o_price->getMinDiscountQuantity()
) {
return false;
// If supplier and part number match, merge the orderdetails
if ($t->getSupplier() === $o->getSupplier() && $t->getSupplierPartNr() === $o->getSupplierPartNr()) {
// Update URL if target doesn't have one
if (empty($t->getSupplierProductUrl(false)) && !empty($o->getSupplierProductUrl(false))) {
$t->setSupplierProductUrl($o->getSupplierProductUrl(false));
}
// Merge price details: add new ones, update empty ones, keep existing non-empty ones
foreach ($o->getPricedetails() as $otherPrice) {
$found = false;
foreach ($t->getPricedetails() as $targetPrice) {
if ($targetPrice->getMinDiscountQuantity() === $otherPrice->getMinDiscountQuantity()
&& $targetPrice->getCurrency() === $otherPrice->getCurrency()) {
// Only update price if the existing one is zero/empty (most logical)
if ($targetPrice->getPrice()->isZero()) {
$targetPrice->setPrice($otherPrice->getPrice());
$targetPrice->setPriceRelatedQuantity($otherPrice->getPriceRelatedQuantity());
}
$found = true;
break;
}
}
// Add completely new price tiers
if (!$found) {
$clonedPrice = clone $otherPrice;
$clonedPrice->setOrderdetail($t);
$t->addPricedetail($clonedPrice);
}
}
return true; // Consider them equal so the other one gets skipped
}
//If all pricedetails are equal, the orderdetails are equal
return true;
return false; // Different supplier/part number, add as new
});
//The pricedetails are not correctly assigned to the new orderdetails, so fix that
foreach ($target->getOrderdetails() as $orderdetail) {

110
src/Services/ImportExportSystem/EntityExporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use Symfony\Component\Serializer\SerializerInterface;
use function Symfony\Component\String\u;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use PhpOffice\PhpSpreadsheet\Writer\Xlsx;
use PhpOffice\PhpSpreadsheet\Writer\Xls;
/**
* Use this class to export an entity to multiple file formats.
@ -52,7 +55,7 @@ class EntityExporter
protected function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefault('format', 'csv');
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setDefault('csv_delimiter', ';');
$resolver->setAllowedTypes('csv_delimiter', 'string');
@ -88,28 +91,35 @@ class EntityExporter
$options = $resolver->resolve($options);
//Handle Excel formats by converting from CSV
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
return $this->exportToExcel($entities, $options);
}
//If include children is set, then we need to add the include_children group
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
return $this->serializer->serialize($entities, $options['format'],
return $this->serializer->serialize(
$entities,
$options['format'],
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'xml_root_node_name' => 'PartDBExport',
'partdb_export' => true,
//Skip the item normalizer, so that we dont get IRIs in the output
//Skip the item normalizer, so that we dont get IRIs in the output
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
//Handle circular references
//Handle circular references
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
}
private function handleCircularReference(object $object, string $format, array $context): string
private function handleCircularReference(object $object): string
{
if ($object instanceof AbstractStructuralDBElement) {
return $object->getFullPath("->");
@ -119,7 +129,75 @@ class EntityExporter
return $object->__toString();
}
throw new CircularReferenceException('Circular reference detected for object of type '.get_class($object));
throw new CircularReferenceException('Circular reference detected for object of type ' . get_class($object));
}
/**
* Exports entities to Excel format (xlsx or xls).
*
* @param AbstractNamedDBElement[] $entities The entities to export
* @param array $options The export options
*
* @return string The Excel file content as binary string
*/
protected function exportToExcel(array $entities, array $options): string
{
//First get CSV data using existing serializer
$groups = [$options['level']];
if ($options['include_children']) {
$groups[] = 'include_children';
}
$csvData = $this->serializer->serialize(
$entities,
'csv',
[
'groups' => $groups,
'as_collection' => true,
'csv_delimiter' => $options['csv_delimiter'],
'partdb_export' => true,
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
AbstractNormalizer::CIRCULAR_REFERENCE_HANDLER => $this->handleCircularReference(...),
]
);
//Convert CSV to Excel
$spreadsheet = new Spreadsheet();
$worksheet = $spreadsheet->getActiveSheet();
$rows = explode("\n", $csvData);
$rowIndex = 1;
foreach ($rows as $row) {
if (trim($row) === '') {
continue;
}
$columns = str_getcsv($row, $options['csv_delimiter'], '"', '\\');
$colIndex = 1;
foreach ($columns as $column) {
$cellCoordinate = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex) . $rowIndex;
$worksheet->setCellValue($cellCoordinate, $column);
$colIndex++;
}
$rowIndex++;
}
//Save to memory stream
$writer = $options['format'] === 'xlsx' ? new Xlsx($spreadsheet) : new Xls($spreadsheet);
$memFile = fopen("php://temp", 'r+b');
$writer->save($memFile);
rewind($memFile);
$content = stream_get_contents($memFile);
fclose($memFile);
if ($content === false) {
throw new \RuntimeException('Failed to read Excel content from memory stream.');
}
return $content;
}
/**
@ -156,19 +234,15 @@ class EntityExporter
//Determine the content type for the response
//Plain text should work for all types
$content_type = 'text/plain';
//Try to use better content types based on the format
$format = $options['format'];
switch ($format) {
case 'xml':
$content_type = 'application/xml';
break;
case 'json':
$content_type = 'application/json';
break;
}
$content_type = match ($format) {
'xml' => 'application/xml',
'json' => 'application/json',
'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'xls' => 'application/vnd.ms-excel',
default => 'text/plain',
};
$response->headers->set('Content-Type', $content_type);
//If view option is not specified, then download the file.
@ -186,7 +260,7 @@ class EntityExporter
$level = $options['level'];
$filename = 'export_'.$entity_name.'_'.$level.'.'.$format;
$filename = "export_{$entity_name}_{$level}.{$format}";
//Sanitize the filename
$filename = FilenameSanatizer::sanitizeFilename($filename);

139
src/Services/ImportExportSystem/EntityImporter.php
View file

@ -38,6 +38,9 @@ use Symfony\Component\HttpFoundation\File\File;
use Symfony\Component\OptionsResolver\OptionsResolver;
use Symfony\Component\Serializer\SerializerInterface;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use Psr\Log\LoggerInterface;
/**
* @see \App\Tests\Services\ImportExportSystem\EntityImporterTest
@ -50,7 +53,7 @@ class EntityImporter
*/
private const ENCODINGS = ["ASCII", "UTF-8", "ISO-8859-1", "ISO-8859-15", "Windows-1252", "UTF-16", "UTF-32"];
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator)
public function __construct(protected SerializerInterface $serializer, protected EntityManagerInterface $em, protected ValidatorInterface $validator, protected LoggerInterface $logger)
{
}
@ -102,7 +105,7 @@ class EntityImporter
foreach ($names as $name) {
//Count indentation level (whitespace characters at the beginning of the line)
$identSize = strlen($name)-strlen(ltrim($name));
$identSize = strlen($name) - strlen(ltrim($name));
//If the line is intended more than the last line, we have a new parent element
if ($identSize > end($indentations)) {
@ -195,16 +198,20 @@ class EntityImporter
}
//The [] behind class_name denotes that we expect an array.
$entities = $this->serializer->deserialize($data, $options['class'].'[]', $options['format'],
$entities = $this->serializer->deserialize(
$data,
$options['class'] . '[]',
$options['format'],
[
'groups' => $groups,
'csv_delimiter' => $options['csv_delimiter'],
'create_unknown_datastructures' => $options['create_unknown_datastructures'],
'path_delimiter' => $options['path_delimiter'],
'partdb_import' => true,
//Disable API Platform normalizer, as we don't want to use it here
//Disable API Platform normalizer, as we don't want to use it here
SkippableItemNormalizer::DISABLE_ITEM_NORMALIZER => true,
]);
]
);
//Ensure we have an array of entity elements.
if (!is_array($entities)) {
@ -279,7 +286,7 @@ class EntityImporter
'path_delimiter' => '->', //The delimiter used to separate the path elements in the name of a structural element
]);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml']);
$resolver->setAllowedValues('format', ['csv', 'json', 'xml', 'yaml', 'xlsx', 'xls']);
$resolver->setAllowedTypes('csv_delimiter', 'string');
$resolver->setAllowedTypes('preserve_children', 'bool');
$resolver->setAllowedTypes('class', 'string');
@ -335,6 +342,33 @@ class EntityImporter
*/
public function importFile(File $file, array $options = [], array &$errors = []): array
{
$resolver = new OptionsResolver();
$this->configureOptions($resolver);
$options = $resolver->resolve($options);
if (in_array($options['format'], ['xlsx', 'xls'], true)) {
$this->logger->info('Converting Excel file to CSV', [
'filename' => $file->getFilename(),
'format' => $options['format'],
'delimiter' => $options['csv_delimiter']
]);
$csvData = $this->convertExcelToCsv($file, $options['csv_delimiter']);
$options['format'] = 'csv';
$this->logger->debug('Excel to CSV conversion completed', [
'csv_length' => strlen($csvData),
'csv_lines' => substr_count($csvData, "\n") + 1
]);
// Log the converted CSV for debugging (first 1000 characters)
$this->logger->debug('Converted CSV preview', [
'csv_preview' => substr($csvData, 0, 1000) . (strlen($csvData) > 1000 ? '...' : '')
]);
return $this->importString($csvData, $options, $errors);
}
return $this->importString($file->getContent(), $options, $errors);
}
@ -354,10 +388,103 @@ class EntityImporter
'xml' => 'xml',
'csv', 'tsv' => 'csv',
'yaml', 'yml' => 'yaml',
'xlsx' => 'xlsx',
'xls' => 'xls',
default => null,
};
}
/**
* Converts Excel file to CSV format using PhpSpreadsheet.
*
* @param File $file The Excel file to convert
* @param string $delimiter The CSV delimiter to use
*
* @return string The CSV data as string
*/
protected function convertExcelToCsv(File $file, string $delimiter = ';'): string
{
try {
$this->logger->debug('Loading Excel file', ['path' => $file->getPathname()]);
$spreadsheet = IOFactory::load($file->getPathname());
$worksheet = $spreadsheet->getActiveSheet();
$csvData = [];
$highestRow = $worksheet->getHighestRow();
$highestColumn = $worksheet->getHighestColumn();
$this->logger->debug('Excel file dimensions', [
'rows' => $highestRow,
'columns_detected' => $highestColumn,
'worksheet_title' => $worksheet->getTitle()
]);
$highestColumnIndex = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($highestColumn);
for ($row = 1; $row <= $highestRow; $row++) {
$rowData = [];
// Read all columns using numeric index
for ($colIndex = 1; $colIndex <= $highestColumnIndex; $colIndex++) {
$col = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($colIndex);
try {
$cellValue = $worksheet->getCell("{$col}{$row}")->getCalculatedValue();
$rowData[] = $cellValue ?? '';
} catch (\Exception $e) {
$this->logger->warning('Error reading cell value', [
'cell' => "{$col}{$row}",
'error' => $e->getMessage()
]);
$rowData[] = '';
}
}
$csvRow = implode($delimiter, array_map(function ($value) use ($delimiter) {
$value = (string) $value;
if (strpos($value, $delimiter) !== false || strpos($value, '"') !== false || strpos($value, "\n") !== false) {
return '"' . str_replace('"', '""', $value) . '"';
}
return $value;
}, $rowData));
$csvData[] = $csvRow;
// Log first few rows for debugging
if ($row <= 3) {
$this->logger->debug("Row {$row} converted", [
'original_data' => $rowData,
'csv_row' => $csvRow,
'first_cell_raw' => $worksheet->getCell("A{$row}")->getValue(),
'first_cell_calculated' => $worksheet->getCell("A{$row}")->getCalculatedValue()
]);
}
}
$result = implode("\n", $csvData);
$this->logger->info('Excel to CSV conversion successful', [
'total_rows' => count($csvData),
'total_characters' => strlen($result)
]);
$this->logger->debug('Full CSV data', [
'csv_data' => $result
]);
return $result;
} catch (\Exception $e) {
$this->logger->error('Failed to convert Excel to CSV', [
'file' => $file->getFilename(),
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
throw $e;
}
}
/**
* This functions corrects the parent setting based on the children value of the parent.
*

380
src/Services/InfoProviderSystem/BulkInfoProviderService.php Normal file
View file

@ -0,0 +1,380 @@
<?php
declare(strict_types=1);
namespace App\Services\InfoProviderSystem;
use App\Entity\Parts\Part;
use App\Entity\Parts\Supplier;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchResponseDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchFieldMappingDTO;
use App\Services\InfoProviderSystem\DTOs\BulkSearchPartResultsDTO;
use App\Services\InfoProviderSystem\Providers\BatchInfoProviderInterface;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpClient\Exception\ClientException;
final class BulkInfoProviderService
{
/** @var array<string, Supplier|null> Cache for normalized supplier names */
private array $supplierCache = [];
public function __construct(
private readonly PartInfoRetriever $infoRetriever,
private readonly ExistingPartFinder $existingPartFinder,
private readonly ProviderRegistry $providerRegistry,
private readonly EntityManagerInterface $entityManager,
private readonly LoggerInterface $logger
) {}
/**
* Perform bulk search across multiple parts and providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mappings defining search strategy
* @param bool $prefetchDetails Whether to prefetch detailed information for results
* @return BulkSearchResponseDTO Structured response containing all search results
* @throws \InvalidArgumentException If no valid parts provided
* @throws \RuntimeException If no search results found for any parts
*/
public function performBulkSearch(array $parts, array $fieldMappings, bool $prefetchDetails = false): BulkSearchResponseDTO
{
if (empty($parts)) {
throw new \InvalidArgumentException('No valid parts found for bulk import');
}
$partResults = [];
$hasAnyResults = false;
// Group providers by batch capability
$batchProviders = [];
$regularProviders = [];
foreach ($fieldMappings as $mapping) {
foreach ($mapping->providers as $providerKey) {
if (!is_string($providerKey)) {
$this->logger->error('Invalid provider key type', [
'providerKey' => $providerKey,
'type' => gettype($providerKey)
]);
continue;
}
$provider = $this->providerRegistry->getProviderByKey($providerKey);
if ($provider instanceof BatchInfoProviderInterface) {
$batchProviders[$providerKey] = $provider;
} else {
$regularProviders[$providerKey] = $provider;
}
}
}
// Process batch providers first (more efficient)
$batchResults = $this->processBatchProviders($parts, $fieldMappings, $batchProviders);
// Process regular providers
$regularResults = $this->processRegularProviders($parts, $fieldMappings, $regularProviders, $batchResults);
// Combine and format results for each part
foreach ($parts as $part) {
$searchResults = [];
// Get results from batch and regular processing
$allResults = array_merge(
$batchResults[$part->getId()] ?? [],
$regularResults[$part->getId()] ?? []
);
if (!empty($allResults)) {
$hasAnyResults = true;
$searchResults = $this->formatSearchResults($allResults);
}
$partResults[] = new BulkSearchPartResultsDTO(
part: $part,
searchResults: $searchResults,
errors: []
);
}
if (!$hasAnyResults) {
throw new \RuntimeException('No search results found for any of the selected parts');
}
$response = new BulkSearchResponseDTO($partResults);
// Prefetch details if requested
if ($prefetchDetails) {
$this->prefetchDetailsForResults($response);
}
return $response;
}
/**
* Process parts using batch-capable info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, BatchInfoProviderInterface> $batchProviders Batch providers indexed by key
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processBatchProviders(array $parts, array $fieldMappings, array $batchProviders): array
{
$batchResults = [];
foreach ($batchProviders as $providerKey => $provider) {
$keywords = $this->collectKeywordsForProvider($parts, $fieldMappings, $providerKey);
if (empty($keywords)) {
continue;
}
try {
$providerResults = $provider->searchByKeywordsBatch($keywords);
// Map results back to parts
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && isset($providerResults[$keyword])) {
foreach ($providerResults[$keyword] as $dto) {
$batchResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
}
}
}
} catch (\Exception $e) {
$this->logger->error('Batch search failed for provider ' . $providerKey, [
'error' => $e->getMessage(),
'provider' => $providerKey
]);
}
}
return $batchResults;
}
/**
* Process parts using regular (non-batch) info providers.
*
* @param Part[] $parts Array of parts to search for
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param array<string, InfoProviderInterface> $regularProviders Regular providers indexed by key
* @param array<int, BulkSearchPartResultDTO[]> $excludeResults Results to exclude (from batch processing)
* @return array<int, BulkSearchPartResultDTO[]> Results indexed by part ID
*/
private function processRegularProviders(array $parts, array $fieldMappings, array $regularProviders, array $excludeResults): array
{
$regularResults = [];
foreach ($parts as $part) {
$regularResults[$part->getId()] = [];
// Skip if we already have batch results for this part
if (!empty($excludeResults[$part->getId()] ?? [])) {
continue;
}
foreach ($fieldMappings as $mapping) {
$providers = array_intersect($mapping->providers, array_keys($regularProviders));
if (empty($providers)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if (!$keyword) {
continue;
}
try {
$dtos = $this->infoRetriever->searchByKeyword($keyword, $providers);
foreach ($dtos as $dto) {
$regularResults[$part->getId()][] = new BulkSearchPartResultDTO(
searchResult: $dto,
sourceField: $mapping->field,
sourceKeyword: $keyword,
localPart: $this->existingPartFinder->findFirstExisting($dto),
priority: $mapping->priority
);
}
} catch (ClientException $e) {
$this->logger->error('Regular search failed', [
'part_id' => $part->getId(),
'field' => $mapping->field,
'error' => $e->getMessage()
]);
}
}
}
return $regularResults;
}
/**
* Collect unique keywords for a specific provider from all parts and field mappings.
*
* @param Part[] $parts Array of parts to collect keywords from
* @param BulkSearchFieldMappingDTO[] $fieldMappings Array of field mapping configurations
* @param string $providerKey The provider key to collect keywords for
* @return string[] Array of unique keywords
*/
private function collectKeywordsForProvider(array $parts, array $fieldMappings, string $providerKey): array
{
$keywords = [];
foreach ($parts as $part) {
foreach ($fieldMappings as $mapping) {
if (!in_array($providerKey, $mapping->providers, true)) {
continue;
}
$keyword = $this->getKeywordFromField($part, $mapping->field);
if ($keyword && !in_array($keyword, $keywords, true)) {
$keywords[] = $keyword;
}
}
}
return $keywords;
}
private function getKeywordFromField(Part $part, string $field): ?string
{
return match ($field) {
'mpn' => $part->getManufacturerProductNumber(),
'name' => $part->getName(),
default => $this->getSupplierPartNumber($part, $field)
};
}
private function getSupplierPartNumber(Part $part, string $field): ?string
{
if (!str_ends_with($field, '_spn')) {
return null;
}
$supplierKey = substr($field, 0, -4);
$supplier = $this->getSupplierByNormalizedName($supplierKey);
if (!$supplier) {
return null;
}
$orderDetail = $part->getOrderdetails()->filter(
fn($od) => $od->getSupplier()?->getId() === $supplier->getId()
)->first();
return $orderDetail !== false ? $orderDetail->getSupplierpartnr() : null;
}
/**
* Get supplier by normalized name with caching to prevent N+1 queries.
*
* @param string $normalizedKey The normalized supplier key to search for
* @return Supplier|null The matching supplier or null if not found
*/
private function getSupplierByNormalizedName(string $normalizedKey): ?Supplier
{
// Check cache first
if (isset($this->supplierCache[$normalizedKey])) {
return $this->supplierCache[$normalizedKey];
}
// Use efficient database query with PHP normalization
// Since DQL doesn't support REPLACE, we'll load all suppliers once and cache the normalization
if (empty($this->supplierCache)) {
$this->loadSuppliersIntoCache();
}
$supplier = $this->supplierCache[$normalizedKey] ?? null;
// Cache the result (including null results to prevent repeated queries)
$this->supplierCache[$normalizedKey] = $supplier;
return $supplier;
}
/**
* Load all suppliers into cache with normalized names to avoid N+1 queries.
*/
private function loadSuppliersIntoCache(): void
{
/** @var Supplier[] $suppliers */
$suppliers = $this->entityManager->getRepository(Supplier::class)->findAll();
foreach ($suppliers as $supplier) {
$normalizedName = strtolower(str_replace([' ', '-', '_'], '_', $supplier->getName()));
$this->supplierCache[$normalizedName] = $supplier;
}
}
/**
* Format and deduplicate search results.
*
* @param BulkSearchPartResultDTO[] $bulkResults Array of bulk search results
* @return BulkSearchPartResultDTO[] Array of formatted search results with metadata
*/
private function formatSearchResults(array $bulkResults): array
{
// Sort by priority and remove duplicates
usort($bulkResults, fn($a, $b) => $a->priority <=> $b->priority);
$uniqueResults = [];
$seenKeys = [];
foreach ($bulkResults as $result) {
$key = "{$result->searchResult->provider_key}|{$result->searchResult->provider_id}";
if (!in_array($key, $seenKeys, true)) {
$seenKeys[] = $key;
$uniqueResults[] = $result;
}
}
return $uniqueResults;
}
/**
* Prefetch detailed information for search results.
*
* @param BulkSearchResponseDTO $searchResults Search results (supports both new DTO and legacy array format)
*/
public function prefetchDetailsForResults(BulkSearchResponseDTO $searchResults): void
{
$prefetchCount = 0;
// Handle both new DTO format and legacy array format for backwards compatibility
foreach ($searchResults->partResults as $partResult) {
foreach ($partResult->searchResults as $result) {
$dto = $result->searchResult;
try {
$this->infoRetriever->getDetails($dto->provider_key, $dto->provider_id);
$prefetchCount++;
} catch (\Exception $e) {
$this->logger->warning('Failed to prefetch details for provider part', [
'provider_key' => $dto->provider_key,
'provider_id' => $dto->provider_id,
'error' => $e->getMessage()
]);
}
}
}
$this->logger->info("Prefetched details for {$prefetchCount} search results");
}
}

91
src/Services/InfoProviderSystem/DTOs/BulkSearchFieldMappingDTO.php Normal file
View file

@ -0,0 +1,91 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
/**
* Represents a mapping between a part field and the info providers that should search in that field.
*/
readonly class BulkSearchFieldMappingDTO
{
/**
* @param string $field The field to search in (e.g., 'mpn', 'name', or supplier-specific fields like 'digikey_spn')
* @param string[] $providers Array of provider keys to search with (e.g., ['digikey', 'farnell'])
* @param int $priority Priority for this field mapping (1-10, lower numbers = higher priority)
*/
public function __construct(
public string $field,
public array $providers,
public int $priority = 1
) {
if ($priority < 1 || $priority > 10) {
throw new \InvalidArgumentException('Priority must be between 1 and 10');
}
}
/**
* Create a FieldMappingDTO from legacy array format.
* @param array{field: string, providers: string[], priority?: int} $data
*/
public static function fromSerializableArray(array $data): self
{
return new self(
field: $data['field'],
providers: $data['providers'] ?? [],
priority: $data['priority'] ?? 1
);
}
/**
* Convert this DTO to the legacy array format for backwards compatibility.
* @return array{field: string, providers: string[], priority: int}
*/
public function toSerializableArray(): array
{
return [
'field' => $this->field,
'providers' => $this->providers,
'priority' => $this->priority,
];
}
/**
* Check if this field mapping is for a supplier part number field.
*/
public function isSupplierPartNumberField(): bool
{
return str_ends_with($this->field, '_spn');
}
/**
* Get the supplier key from a supplier part number field.
* Returns null if this is not a supplier part number field.
*/
public function getSupplierKey(): ?string
{
if (!$this->isSupplierPartNumberField()) {
return null;
}
return substr($this->field, 0, -4);
}
}

44
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultDTO.php Normal file
View file

@ -0,0 +1,44 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents a single search result from bulk search with additional context information, like how the part was found.
*/
readonly class BulkSearchPartResultDTO
{
public function __construct(
/** The base search result DTO containing provider data */
public SearchResultDTO $searchResult,
/** The field that was used to find this result */
public ?string $sourceField = null,
/** The actual keyword that was searched for */
public ?string $sourceKeyword = null,
/** Local part that matches this search result, if any */
public ?Part $localPart = null,
/** Priority for this search result */
public int $priority = 1
) {}
}

83
src/Services/InfoProviderSystem/DTOs/BulkSearchPartResultsDTO.php Normal file
View file

@ -0,0 +1,83 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
/**
* Represents the search results for a single part from bulk info provider search.
* It contains multiple search results, that match the part.
*/
readonly class BulkSearchPartResultsDTO
{
/**
* @param Part $part The part that was searched for
* @param BulkSearchPartResultDTO[] $searchResults Array of search results found for this part
* @param string[] $errors Array of error messages encountered during search
*/
public function __construct(
public Part $part,
public array $searchResults = [],
public array $errors = []
) {}
/**
* Check if this part has any search results.
*/
public function hasResults(): bool
{
return !empty($this->searchResults);
}
/**
* Check if this part has any errors.
*/
public function hasErrors(): bool
{
return !empty($this->errors);
}
/**
* Get the number of search results for this part.
*/
public function getResultCount(): int
{
return count($this->searchResults);
}
public function getErrorCount(): int
{
return count($this->errors);
}
/**
* Get search results sorted by priority (ascending).
* @return BulkSearchPartResultDTO[]
*/
public function getResultsSortedByPriority(): array
{
$results = $this->searchResults;
usort($results, static fn(BulkSearchPartResultDTO $a, BulkSearchPartResultDTO $b) => $a->priority <=> $b->priority);
return $results;
}
}

231
src/Services/InfoProviderSystem/DTOs/BulkSearchResponseDTO.php Normal file
View file

@ -0,0 +1,231 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\DTOs;
use App\Entity\Parts\Part;
use Doctrine\ORM\EntityManagerInterface;
use Traversable;
/**
* Represents the complete response from a bulk info provider search operation.
* It contains a list of PartSearchResultDTOs, one for each part searched.
*/
readonly class BulkSearchResponseDTO implements \ArrayAccess, \IteratorAggregate
{
/**
* @param BulkSearchPartResultsDTO[] $partResults Array of search results for each part
*/
public function __construct(
public array $partResults
) {}
/**
* Replaces the search results for a specific part, and returns a new instance.
* The part to replaced, is identified by the part property of the new_results parameter.
* The original instance remains unchanged.
* @param BulkSearchPartResultsDTO $new_results
* @return BulkSearchResponseDTO
*/
public function replaceResultsForPart(BulkSearchPartResultsDTO $new_results): self
{
$array = $this->partResults;
$replaced = false;
foreach ($array as $index => $partResult) {
if ($partResult->part === $new_results->part) {
$array[$index] = $new_results;
$replaced = true;
break;
}
}
if (!$replaced) {
throw new \InvalidArgumentException("Part not found in existing results.");
}
return new self($array);
}
/**
* Check if any parts have search results.
*/
public function hasAnyResults(): bool
{
foreach ($this->partResults as $partResult) {
if ($partResult->hasResults()) {
return true;
}
}
return false;
}
/**
* Get the total number of search results across all parts.
*/
public function getTotalResultCount(): int
{
$count = 0;
foreach ($this->partResults as $partResult) {
$count += $partResult->getResultCount();
}
return $count;
}
/**
* Get all parts that have search results.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithResults(): array
{
return array_filter($this->partResults, fn($result) => $result->hasResults());
}
/**
* Get all parts that have errors.
* @return BulkSearchPartResultsDTO[]
*/
public function getPartsWithErrors(): array
{
return array_filter($this->partResults, fn($result) => $result->hasErrors());
}
/**
* Get the number of parts processed.
*/
public function getPartCount(): int
{
return count($this->partResults);
}
/**
* Get the number of parts with successful results.
*/
public function getSuccessfulPartCount(): int
{
return count($this->getPartsWithResults());
}
/**
* Merge multiple BulkSearchResponseDTO instances into one.
* @param BulkSearchResponseDTO ...$responses
* @return BulkSearchResponseDTO
*/
public static function merge(BulkSearchResponseDTO ...$responses): BulkSearchResponseDTO
{
$mergedResults = [];
foreach ($responses as $response) {
foreach ($response->partResults as $partResult) {
$mergedResults[] = $partResult;
}
}
return new BulkSearchResponseDTO($mergedResults);
}
/**
* Convert this DTO to a serializable representation suitable for storage in the database
* @return array
*/
public function toSerializableRepresentation(): array
{
$serialized = [];
foreach ($this->partResults as $partResult) {
$partData = [
'part_id' => $partResult->part->getId(),
'search_results' => [],
'errors' => $partResult->errors ?? []
];
foreach ($partResult->searchResults as $result) {
$partData['search_results'][] = [
'dto' => $result->searchResult->toNormalizedSearchResultArray(),
'source_field' => $result->sourceField ?? null,
'source_keyword' => $result->sourceKeyword ?? null,
'localPart' => $result->localPart?->getId(),
'priority' => $result->priority
];
}
$serialized[] = $partData;
}
return $serialized;
}
/**
* Creates a BulkSearchResponseDTO from a serializable representation.
* @param array $data
* @param EntityManagerInterface $entityManager
* @return BulkSearchResponseDTO
* @throws \Doctrine\ORM\Exception\ORMException
*/
public static function fromSerializableRepresentation(array $data, EntityManagerInterface $entityManager): BulkSearchResponseDTO
{
$partResults = [];
foreach ($data as $partData) {
$partResults[] = new BulkSearchPartResultsDTO(
part: $entityManager->getReference(Part::class, $partData['part_id']),
searchResults: array_map(fn($result) => new BulkSearchPartResultDTO(
searchResult: SearchResultDTO::fromNormalizedSearchResultArray($result['dto']),
sourceField: $result['source_field'] ?? null,
sourceKeyword: $result['source_keyword'] ?? null,
localPart: isset($result['localPart']) ? $entityManager->getReference(Part::class, $result['localPart']) : null,
priority: $result['priority'] ?? null
), $partData['search_results'] ?? []),
errors: $partData['errors'] ?? []
);
}
return new BulkSearchResponseDTO($partResults);
}
public function offsetExists(mixed $offset): bool
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return isset($this->partResults[$offset]);
}
public function offsetGet(mixed $offset): ?BulkSearchPartResultsDTO
{
if (!is_int($offset)) {
throw new \InvalidArgumentException("Offset must be an integer.");
}
return $this->partResults[$offset] ?? null;
}
public function offsetSet(mixed $offset, mixed $value): void
{
throw new \LogicException("BulkSearchResponseDTO is immutable.");
}
public function offsetUnset(mixed $offset): void
{
throw new \LogicException('BulkSearchResponseDTO is immutable.');
}
public function getIterator(): Traversable
{
return new \ArrayIterator($this->partResults);
}
}

8
src/Services/InfoProviderSystem/DTOs/FileDTO.php
View file

@ -28,12 +28,12 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a datasheet, a 3D model, a picture or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\FileDTOTest
*/
class FileDTO
readonly class FileDTO
{
/**
* @var string The URL where to get this file
*/
public readonly string $url;
public string $url;
/**
* @param string $url The URL where to get this file
@ -41,7 +41,7 @@ class FileDTO
*/
public function __construct(
string $url,
public readonly ?string $name = null,
public ?string $name = null,
) {
//Find all occurrences of non URL safe characters and replace them with their URL encoded version.
//We only want to replace characters which can not have a valid meaning in a URL (what would break the URL).
@ -50,4 +50,4 @@ class FileDTO
}
}
}

18
src/Services/InfoProviderSystem/DTOs/ParameterDTO.php
View file

@ -28,17 +28,17 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This could be a voltage, a current, a temperature or similar.
* @see \App\Tests\Services\InfoProviderSystem\DTOs\ParameterDTOTest
*/
class ParameterDTO
readonly class ParameterDTO
{
public function __construct(
public readonly string $name,
public readonly ?string $value_text = null,
public readonly ?float $value_typ = null,
public readonly ?float $value_min = null,
public readonly ?float $value_max = null,
public readonly ?string $unit = null,
public readonly ?string $symbol = null,
public readonly ?string $group = null,
public string $name,
public ?string $value_text = null,
public ?float $value_typ = null,
public ?float $value_min = null,
public ?float $value_max = null,
public ?string $unit = null,
public ?string $symbol = null,
public ?string $group = null,
) {
}

2
src/Services/InfoProviderSystem/DTOs/PartDetailDTO.php
View file

@ -70,4 +70,4 @@ class PartDetailDTO extends SearchResultDTO
footprint: $footprint,
);
}
}
}

14
src/Services/InfoProviderSystem/DTOs/PriceDTO.php
View file

@ -28,21 +28,21 @@ use Brick\Math\BigDecimal;
/**
* This DTO represents a price for a single unit in a certain discount range
*/
class PriceDTO
readonly class PriceDTO
{
private readonly BigDecimal $price_as_big_decimal;
private BigDecimal $price_as_big_decimal;
public function __construct(
/** @var float The minimum amount that needs to get ordered for this price to be valid */
public readonly float $minimum_discount_amount,
public float $minimum_discount_amount,
/** @var string The price as string (with .) */
public readonly string $price,
public string $price,
/** @var string The currency of the used ISO code of this price detail */
public readonly ?string $currency_iso_code,
public ?string $currency_iso_code,
/** @var bool If the price includes tax */
public readonly ?bool $includes_tax = true,
public ?bool $includes_tax = true,
/** @var float the price related quantity */
public readonly ?float $price_related_quantity = 1.0,
public ?float $price_related_quantity = 1.0,
)
{
$this->price_as_big_decimal = BigDecimal::of($this->price);

12
src/Services/InfoProviderSystem/DTOs/PurchaseInfoDTO.php
View file

@ -27,15 +27,15 @@ namespace App\Services\InfoProviderSystem\DTOs;
* This DTO represents a purchase information for a part (supplier name, order number and prices).
* @see \App\Tests\Services\InfoProviderSystem\DTOs\PurchaseInfoDTOTest
*/
class PurchaseInfoDTO
readonly class PurchaseInfoDTO
{
public function __construct(
public readonly string $distributor_name,
public readonly string $order_number,
public string $distributor_name,
public string $order_number,
/** @var PriceDTO[] */
public readonly array $prices,
public array $prices,
/** @var string|null An url to the product page of the vendor */
public readonly ?string $product_url = null,
public ?string $product_url = null,
)
{
//Ensure that the prices are PriceDTO instances
@ -45,4 +45,4 @@ class PurchaseInfoDTO
}
}
}
}
}

49
src/Services/InfoProviderSystem/DTOs/SearchResultDTO.php
View file

@ -59,8 +59,8 @@ class SearchResultDTO
public readonly ?string $provider_url = null,
/** @var string|null A footprint representation of the providers page */
public readonly ?string $footprint = null,
) {
)
{
if ($preview_image_url !== null) {
//Utilize the escaping mechanism of FileDTO to ensure that the preview image URL is correctly encoded
//See issue #521: https://github.com/Part-DB/Part-DB-server/issues/521
@ -71,4 +71,47 @@ class SearchResultDTO
$this->preview_image_url = null;
}
}
}
/**
* This method creates a normalized array representation of the DTO.
* @return array
*/
public function toNormalizedSearchResultArray(): array
{
return [
'provider_key' => $this->provider_key,
'provider_id' => $this->provider_id,
'name' => $this->name,
'description' => $this->description,
'category' => $this->category,
'manufacturer' => $this->manufacturer,
'mpn' => $this->mpn,
'preview_image_url' => $this->preview_image_url,
'manufacturing_status' => $this->manufacturing_status?->value,
'provider_url' => $this->provider_url,
'footprint' => $this->footprint,
];
}
/**
* Creates a SearchResultDTO from a normalized array representation.
* @param array $data
* @return self
*/
public static function fromNormalizedSearchResultArray(array $data): self
{
return new self(
provider_key: $data['provider_key'],
provider_id: $data['provider_id'],
name: $data['name'],
description: $data['description'],
category: $data['category'] ?? null,
manufacturer: $data['manufacturer'] ?? null,
mpn: $data['mpn'] ?? null,
preview_image_url: $data['preview_image_url'] ?? null,
manufacturing_status: isset($data['manufacturing_status']) ? ManufacturingStatus::tryFrom($data['manufacturing_status']) : null,
provider_url: $data['provider_url'] ?? null,
footprint: $data['footprint'] ?? null,
);
}
}

40
src/Services/InfoProviderSystem/Providers/BatchInfoProviderInterface.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
/**
* This interface marks a provider as a info provider which can provide information directly in batch operations
*/
interface BatchInfoProviderInterface extends InfoProviderInterface
{
/**
* Search for multiple keywords in a single batch operation and return the results, ordered by the keywords.
* This allows for a more efficient search compared to running multiple single searches.
* @param string[] $keywords
* @return array<string, SearchResultDTO[]> An associative array where the key is the keyword and the value is the search results for that keyword
*/
public function searchByKeywordsBatch(array $keywords): array;
}

145
src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file

@ -33,7 +33,7 @@ use App\Settings\InfoProviderSystem\LCSCSettings;
use Symfony\Component\HttpFoundation\Cookie;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class LCSCProvider implements InfoProviderInterface
class LCSCProvider implements BatchInfoProviderInterface
{
private const ENDPOINT_URL = 'https://wmsc.lcsc.com/ftps/wm';
@ -69,9 +69,10 @@ class LCSCProvider implements InfoProviderInterface
/**
* @param string $id
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO
*/
private function queryDetail(string $id): PartDetailDTO
private function queryDetail(string $id, bool $lightweight = false): PartDetailDTO
{
$response = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
@ -89,7 +90,7 @@ class LCSCProvider implements InfoProviderInterface
throw new \RuntimeException('Could not find product code: ' . $id);
}
return $this->getPartDetail($product);
return $this->getPartDetail($product, $lightweight);
}
/**
@ -99,30 +100,42 @@ class LCSCProvider implements InfoProviderInterface
private function getRealDatasheetUrl(?string $url): string
{
if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.lcsc\.com|www\.lcsc\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) {
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
if (preg_match("/^https:\/\/datasheet\.lcsc\.com\/lcsc\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) {
$url = 'https://www.lcsc.com/datasheet/lcsc_datasheet_' . $rewriteMatches[1];
}
$response = $this->lcscClient->request('GET', $url, [
'headers' => [
'Referer' => 'https://www.lcsc.com/product-detail/_' . $matches[2] . '.html'
],
]);
if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.lcsc\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) {
//HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused
//See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934
$jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}');
$url = $jsonObj->previewPdfUrl;
}
}
return $url;
}
/**
* @param string $term
* @param bool $lightweight If true, skip expensive operations like datasheet resolution
* @return PartDetailDTO[]
*/
private function queryByTerm(string $term): array
private function queryByTerm(string $term, bool $lightweight = false): array
{
// Optimize: If term looks like an LCSC part number (starts with C followed by digits),
// use direct detail query instead of slower search
if (preg_match('/^C\d+$/i', trim($term))) {
try {
return [$this->queryDetail(trim($term), $lightweight)];
} catch (\Exception $e) {
// If direct lookup fails, fall back to search
// This handles cases where the C-code might not exist
}
}
$response = $this->lcscClient->request('POST', self::ENDPOINT_URL . "/search/v2/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
@ -145,11 +158,11 @@ class LCSCProvider implements InfoProviderInterface
// detailed product listing. It does so utilizing a product tip field.
// If product tip exists and there are no products in the product list try a detail query
if (count($products) === 0 && $tipProductCode !== null) {
$result[] = $this->queryDetail($tipProductCode);
$result[] = $this->queryDetail($tipProductCode, $lightweight);
}
foreach ($products as $product) {
$result[] = $this->getPartDetail($product);
$result[] = $this->getPartDetail($product, $lightweight);
}
return $result;
@ -178,7 +191,7 @@ class LCSCProvider implements InfoProviderInterface
* @param array $product
* @return PartDetailDTO
*/
private function getPartDetail(array $product): PartDetailDTO
private function getPartDetail(array $product, bool $lightweight = false): PartDetailDTO
{
// Get product images in advance
$product_images = $this->getProductImages($product['productImages'] ?? null);
@ -214,10 +227,10 @@ class LCSCProvider implements InfoProviderInterface
manufacturing_status: null,
provider_url: $this->getProductShortURL($product['productCode']),
footprint: $this->sanitizeField($footprint),
datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images,
parameters: $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
datasheets: $lightweight ? [] : $this->getProductDatasheets($product['pdfUrl'] ?? null),
images: $product_images, // Always include images - users need to see them
parameters: $lightweight ? [] : $this->attributesToParameters($product['paramVOList'] ?? []),
vendor_infos: $lightweight ? [] : $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []),
mass: $product['weight'] ?? null,
);
}
@ -286,7 +299,7 @@ class LCSCProvider implements InfoProviderInterface
*/
private function getProductShortURL(string $product_code): string
{
return 'https://www.lcsc.com/product-detail/' . $product_code .'.html';
return 'https://www.lcsc.com/product-detail/' . $product_code . '.html';
}
/**
@ -327,7 +340,7 @@ class LCSCProvider implements InfoProviderInterface
//Skip this attribute if it's empty
if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) {
continue;
continue;
}
$result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null);
@ -338,12 +351,86 @@ class LCSCProvider implements InfoProviderInterface
public function searchByKeyword(string $keyword): array
{
return $this->queryByTerm($keyword);
return $this->queryByTerm($keyword, true); // Use lightweight mode for search
}
/**
* Batch search multiple keywords asynchronously (like JavaScript Promise.all)
* @param array $keywords Array of keywords to search
* @return array Results indexed by keyword
*/
public function searchByKeywordsBatch(array $keywords): array
{
if (empty($keywords)) {
return [];
}
$responses = [];
$results = [];
// Start all requests immediately (like JavaScript promises without await)
foreach ($keywords as $keyword) {
if (preg_match('/^C\d+$/i', trim($keyword))) {
// Direct detail API call for C-codes
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/product/detail", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'productCode' => trim($keyword),
],
]);
} else {
// Search API call for other terms
$responses[$keyword] = $this->lcscClient->request('GET', self::ENDPOINT_URL . "/search/global", [
'headers' => [
'Cookie' => new Cookie('currencyCode', $this->settings->currency)
],
'query' => [
'keyword' => $keyword,
],
]);
}
}
// Now collect all results (like .then() in JavaScript)
foreach ($responses as $keyword => $response) {
try {
$arr = $response->toArray(); // This waits for the response
$results[$keyword] = $this->processSearchResponse($arr, $keyword);
} catch (\Exception $e) {
$results[$keyword] = []; // Empty results on error
}
}
return $results;
}
private function processSearchResponse(array $arr, string $keyword): array
{
$result = [];
// Check if this looks like a detail response (direct C-code lookup)
if (isset($arr['result']['productCode'])) {
$product = $arr['result'];
$result[] = $this->getPartDetail($product, true); // lightweight mode
} else {
// This is a search response
$products = $arr['result']['productSearchResultVO']['productList'] ?? [];
$tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null;
// If no products but has tip, we'd need another API call - skip for batch mode
foreach ($products as $product) {
$result[] = $this->getPartDetail($product, true); // lightweight mode
}
}
return $result;
}
public function getDetails(string $id): PartDetailDTO
{
$tmp = $this->queryByTerm($id);
$tmp = $this->queryByTerm($id, false);
if (count($tmp) === 0) {
throw new \RuntimeException('No part found with ID ' . $id);
}

19
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -132,6 +132,15 @@ class MouserProvider implements InfoProviderInterface
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
return $this->responseToDTOArray($response);
}
@ -169,6 +178,16 @@ class MouserProvider implements InfoProviderInterface
]
],
]);
// Check for API errors before processing response
if ($response->getStatusCode() !== 200) {
throw new \RuntimeException(sprintf(
'Mouser API returned HTTP %d: %s',
$response->getStatusCode(),
$response->getContent(false)
));
}
$tmp = $this->responseToDTOArray($response);
//Ensure that we have exactly one result

14
src/Services/Parts/PartsTableActionHandler.php
View file

@ -30,13 +30,11 @@ use App\Entity\Parts\Manufacturer;
use App\Entity\Parts\MeasurementUnit;
use App\Entity\Parts\Part;
use App\Entity\Parts\PartLot;
use App\Repository\PartRepository;
use Doctrine\ORM\EntityManagerInterface;
use InvalidArgumentException;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Symfony\Component\Security\Core\Exception\AccessDeniedException;
use Symfony\Contracts\Translation\TranslatableInterface;
use function Symfony\Component\Translation\t;
@ -100,7 +98,7 @@ implode(',', array_map(static fn (PartLot $lot) => $lot->getID(), $part->getPart
//When action starts with "export_" we have to redirect to the export controller
$matches = [];
if (preg_match('/^export_(json|yaml|xml|csv)$/', $action, $matches)) {
if (preg_match('/^export_(json|yaml|xml|csv|xlsx)$/', $action, $matches)) {
$ids = implode(',', array_map(static fn (Part $part) => $part->getID(), $selected_parts));
$level = match ($target_id) {
2 => 'extended',
@ -119,6 +117,16 @@ implode(',', array_map(static fn (PartLot $lot) => $lot->getID(), $part->getPart
);
}
if ($action === 'bulk_info_provider_import') {
$ids = implode(',', array_map(static fn (Part $part) => $part->getID(), $selected_parts));
return new RedirectResponse(
$this->urlGenerator->generate('bulk_info_provider_step1', [
'ids' => $ids,
'_redirect' => $redirect_url
])
);
}
//Iterate over the parts and apply the action to it:
foreach ($selected_parts as $part) {

5
src/Services/Trees/ToolsTreeBuilder.php
View file

@ -138,6 +138,11 @@ class ToolsTreeBuilder
$this->translator->trans('info_providers.search.title'),
$this->urlGenerator->generate('info_providers_search')
))->setIcon('fa-treeview fa-fw fa-solid fa-cloud-arrow-down');
$nodes[] = (new TreeViewNode(
$this->translator->trans('info_providers.bulk_import.manage_jobs'),
$this->urlGenerator->generate('bulk_info_provider_manage')
))->setIcon('fa-treeview fa-fw fa-solid fa-tasks');
}
return $nodes;