groove/Part-DB-server
1
0
Fork 0
mirror of https://github.com/Part-DB/Part-DB-server.git synced 2025-12-16 07:59:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'master' into settings-bundle

Browse source
This commit is contained in:
Jan Böhmer 2025-06-15 18:39:49 +02:00
commit 442457f11b
131 changed files with 12759 additions and 6750 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/Services/InfoProviderSystem/DTOs/ParameterDTO.php
View file

@ -72,9 +72,9 @@ class ParameterDTO
group: $group);
}
//If the attribute contains "..." or a tilde we assume it is a range
if (preg_match('/(\.{3}|~)/', $value) === 1) {
$parts = preg_split('/\s*(\.{3}|~)\s*/', $value);
//If the attribute contains ".." or "..." or a tilde we assume it is a range
if (preg_match('/(\.{2,3}|~)/', $value) === 1) {
$parts = preg_split('/\s*(\.{2,3}|~)\s*/', $value);
if (count($parts) === 2) {
//Try to extract number and unit from value (allow leading +)
if ($unit === null || trim($unit) === '') {

35
src/Services/InfoProviderSystem/DTOtoEntityConverter.php
View file

@ -178,9 +178,21 @@ final class DTOtoEntityConverter
//Set the provider reference on the part
$entity->setProviderReference(InfoProviderReference::fromPartDTO($dto));
$param_groups = [];
//Add parameters
foreach ($dto->parameters ?? [] as $parameter) {
$entity->addParameter($this->convertParameter($parameter));
$new_param = $this->convertParameter($parameter);
$key = $new_param->getName() . '##' . $new_param->getGroup();
//If there is already an parameter with the same name and group, rename the new parameter, by suffixing a number
if (count($param_groups[$key] ?? []) > 0) {
$new_param->setName($new_param->getName() . ' (' . (count($param_groups[$key]) + 1) . ')');
}
$param_groups[$key][] = $new_param;
$entity->addParameter($new_param);
}
//Add preview image
@ -196,6 +208,8 @@ final class DTOtoEntityConverter
$entity->setMasterPictureAttachment($preview_image);
}
$attachments_grouped = [];
//Add other images
$images = $this->files_unique($dto->images ?? []);
foreach ($images as $image) {
@ -204,14 +218,29 @@ final class DTOtoEntityConverter
continue;
}
$entity->addAttachment($this->convertFile($image, $image_type));
$attachment = $this->convertFile($image, $image_type);
$attachments_grouped[$attachment->getName()][] = $attachment;
if (count($attachments_grouped[$attachment->getName()] ?? []) > 1) {
$attachment->setName($attachment->getName() . ' (' . (count($attachments_grouped[$attachment->getName()]) + 1) . ')');
}
$entity->addAttachment($attachment);
}
//Add datasheets
$datasheet_type = $this->getDatasheetType();
$datasheets = $this->files_unique($dto->datasheets ?? []);
foreach ($datasheets as $datasheet) {
$entity->addAttachment($this->convertFile($datasheet, $datasheet_type));
$attachment = $this->convertFile($datasheet, $datasheet_type);
$attachments_grouped[$attachment->getName()][] = $attachment;
if (count($attachments_grouped[$attachment->getName()] ?? []) > 1) {
$attachment->setName($attachment->getName() . ' (' . (count($attachments_grouped[$attachment->getName()])) . ')');
}
$entity->addAttachment($attachment);
}
//Add orderdetails and prices

21
src/Services/InfoProviderSystem/PartInfoRetriever.php
View file

@ -27,6 +27,7 @@ use App\Entity\Parts\Part;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Contracts\Cache\CacheInterface;
use Symfony\Contracts\Cache\ItemInterface;
@ -34,10 +35,12 @@ final class PartInfoRetriever
{
private const CACHE_DETAIL_EXPIRATION = 60 * 60 * 24 * 4; // 4 days
private const CACHE_RESULT_EXPIRATION = 60 * 60 * 24 * 7; // 7 days
private const CACHE_RESULT_EXPIRATION = 60 * 60 * 24 * 4; // 7 days
public function __construct(private readonly ProviderRegistry $provider_registry,
private readonly DTOtoEntityConverter $dto_to_entity_converter, private readonly CacheInterface $partInfoCache)
private readonly DTOtoEntityConverter $dto_to_entity_converter, private readonly CacheInterface $partInfoCache,
#[Autowire(param: "kernel.debug")]
private readonly bool $debugMode = false)
{
}
@ -56,6 +59,11 @@ final class PartInfoRetriever
$provider = $this->provider_registry->getProviderByKey($provider);
}
//Ensure that the provider is active
if (!$provider->isActive()) {
throw new \RuntimeException("The provider with key {$provider->getProviderKey()} is not active!");
}
if (!$provider instanceof InfoProviderInterface) {
throw new \InvalidArgumentException("The provider must be either a provider key or a provider instance!");
}
@ -77,7 +85,7 @@ final class PartInfoRetriever
$escaped_keyword = urlencode($keyword);
return $this->partInfoCache->get("search_{$provider->getProviderKey()}_{$escaped_keyword}", function (ItemInterface $item) use ($provider, $keyword) {
//Set the expiration time
$item->expiresAfter(self::CACHE_RESULT_EXPIRATION);
$item->expiresAfter(!$this->debugMode ? self::CACHE_RESULT_EXPIRATION : 1);
return $provider->searchByKeyword($keyword);
});
@ -94,11 +102,16 @@ final class PartInfoRetriever
{
$provider = $this->provider_registry->getProviderByKey($provider_key);
//Ensure that the provider is active
if (!$provider->isActive()) {
throw new \RuntimeException("The provider with key $provider_key is not active!");
}
//Generate key and escape reserved characters from the provider id
$escaped_part_id = urlencode($part_id);
return $this->partInfoCache->get("details_{$provider_key}_{$escaped_part_id}", function (ItemInterface $item) use ($provider, $part_id) {
//Set the expiration time
$item->expiresAfter(self::CACHE_DETAIL_EXPIRATION);
$item->expiresAfter(!$this->debugMode ? self::CACHE_DETAIL_EXPIRATION : 1);
return $provider->getDetails($part_id);
});

121
src/Services/InfoProviderSystem/Providers/DigikeyProvider.php
View file

@ -108,12 +108,15 @@ class DigikeyProvider implements InfoProviderInterface
{
$request = [
'Keywords' => $keyword,
'RecordCount' => 50,
'RecordStartPosition' => 0,
'ExcludeMarketPlaceProducts' => 'true',
'Limit' => 50,
'Offset' => 0,
'FilterOptionsRequest' => [
'MarketPlaceFilter' => 'ExcludeMarketPlace',
],
];
$response = $this->digikeyClient->request('POST', '/Search/v3/Products/Keyword', [
//$response = $this->digikeyClient->request('POST', '/Search/v3/Products/Keyword', [
$response = $this->digikeyClient->request('POST', '/products/v4/search/keyword', [
'json' => $request,
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
@ -124,18 +127,21 @@ class DigikeyProvider implements InfoProviderInterface
$result = [];
$products = $response_array['Products'];
foreach ($products as $product) {
$result[] = new SearchResultDTO(
provider_key: $this->getProviderKey(),
provider_id: $product['DigiKeyPartNumber'],
name: $product['ManufacturerPartNumber'],
description: $product['DetailedDescription'] ?? $product['ProductDescription'],
category: $this->getCategoryString($product),
manufacturer: $product['Manufacturer']['Value'] ?? null,
mpn: $product['ManufacturerPartNumber'],
preview_image_url: $product['PrimaryPhoto'] ?? null,
manufacturing_status: $this->productStatusToManufacturingStatus($product['ProductStatus']),
provider_url: $product['ProductUrl'],
);
foreach ($product['ProductVariations'] as $variation) {
$result[] = new SearchResultDTO(
provider_key: $this->getProviderKey(),
provider_id: $variation['DigiKeyProductNumber'],
name: $product['ManufacturerProductNumber'],
description: $product['Description']['DetailedDescription'] ?? $product['Description']['ProductDescription'],
category: $this->getCategoryString($product),
manufacturer: $product['Manufacturer']['Name'] ?? null,
mpn: $product['ManufacturerProductNumber'],
preview_image_url: $product['PhotoUrl'] ?? null,
manufacturing_status: $this->productStatusToManufacturingStatus($product['ProductStatus']['Id']),
provider_url: $product['ProductUrl'],
footprint: $variation['PackageType']['Name'], //Use the footprint field, to show the user the package type (Tape & Reel, etc., as digikey has many different package types)
);
}
}
return $result;
@ -143,62 +149,79 @@ class DigikeyProvider implements InfoProviderInterface
public function getDetails(string $id): PartDetailDTO
{
$response = $this->digikeyClient->request('GET', '/Search/v3/Products/' . urlencode($id), [
$response = $this->digikeyClient->request('GET', '/products/v4/search/' . urlencode($id) . '/productdetails', [
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
$product = $response->toArray();
$response_array = $response->toArray();
$product = $response_array['Product'];
$footprint = null;
$parameters = $this->parametersToDTOs($product['Parameters'] ?? [], $footprint);
$media = $this->mediaToDTOs($product['MediaLinks']);
$media = $this->mediaToDTOs($id);
// Get the price_breaks of the selected variation
$price_breaks = [];
foreach ($product['ProductVariations'] as $variation) {
if ($variation['DigiKeyProductNumber'] == $id) {
$price_breaks = $variation['StandardPricing'] ?? [];
break;
}
}
return new PartDetailDTO(
provider_key: $this->getProviderKey(),
provider_id: $product['DigiKeyPartNumber'],
name: $product['ManufacturerPartNumber'],
description: $product['DetailedDescription'] ?? $product['ProductDescription'],
provider_id: $id,
name: $product['ManufacturerProductNumber'],
description: $product['Description']['DetailedDescription'] ?? $product['Description']['ProductDescription'],
category: $this->getCategoryString($product),
manufacturer: $product['Manufacturer']['Value'] ?? null,
mpn: $product['ManufacturerPartNumber'],
preview_image_url: $product['PrimaryPhoto'] ?? null,
manufacturing_status: $this->productStatusToManufacturingStatus($product['ProductStatus']),
manufacturer: $product['Manufacturer']['Name'] ?? null,
mpn: $product['ManufacturerProductNumber'],
preview_image_url: $product['PhotoUrl'] ?? null,
manufacturing_status: $this->productStatusToManufacturingStatus($product['ProductStatus']['Id']),
provider_url: $product['ProductUrl'],
footprint: $footprint,
datasheets: $media['datasheets'],
images: $media['images'],
parameters: $parameters,
vendor_infos: $this->pricingToDTOs($product['StandardPricing'] ?? [], $product['DigiKeyPartNumber'], $product['ProductUrl']),
vendor_infos: $this->pricingToDTOs($price_breaks, $id, $product['ProductUrl']),
);
}
/**
* Converts the product status from the Digikey API to the manufacturing status used in Part-DB
* @param string|null $dk_status
* @param int|null $dk_status
* @return ManufacturingStatus|null
*/
private function productStatusToManufacturingStatus(?string $dk_status): ?ManufacturingStatus
private function productStatusToManufacturingStatus(?int $dk_status): ?ManufacturingStatus
{
// The V4 can use strings to get the status, but if you have changed the PROVIDER_DIGIKEY_LANGUAGE it will not match.
// Using the Id instead which should be fixed.
//
// The API is not well documented and the ID are not there yet, so were extracted using "trial and error".
// The 'Preliminary' id was not found in several categories so I was unable to extract it. Disabled for now.
return match ($dk_status) {
null => null,
'Active' => ManufacturingStatus::ACTIVE,
'Obsolete' => ManufacturingStatus::DISCONTINUED,
'Discontinued at Digi-Key', 'Last Time Buy' => ManufacturingStatus::EOL,
'Not For New Designs' => ManufacturingStatus::NRFND,
'Preliminary' => ManufacturingStatus::ANNOUNCED,
0 => ManufacturingStatus::ACTIVE,
1 => ManufacturingStatus::DISCONTINUED,
2, 4 => ManufacturingStatus::EOL,
7 => ManufacturingStatus::NRFND,
//'Preliminary' => ManufacturingStatus::ANNOUNCED,
default => ManufacturingStatus::NOT_SET,
};
}
private function getCategoryString(array $product): string
{
$category = $product['Category']['Value'];
$sub_category = $product['Family']['Value'];
$category = $product['Category']['Name'];
$sub_category = current($product['Category']['ChildCategories']);
//Replace the ' - ' category separator with ' -> '
$sub_category = str_replace(' - ', ' -> ', $sub_category);
if ($sub_category) {
//Replace the ' - ' category separator with ' -> '
$category = $category . ' -> ' . str_replace(' - ', ' -> ', $sub_category["Name"]);
}
return $category . ' -> ' . $sub_category;
return $category;
}
/**
@ -215,18 +238,18 @@ class DigikeyProvider implements InfoProviderInterface
foreach ($parameters as $parameter) {
if ($parameter['ParameterId'] === 1291) { //Meaning "Manufacturer given footprint"
$footprint_name = $parameter['Value'];
$footprint_name = $parameter['ValueText'];
}
if (in_array(trim((string) $parameter['Value']), ['', '-'], true)) {
if (in_array(trim((string) $parameter['ValueText']), ['', '-'], true)) {
continue;
}
//If the parameter was marked as text only, then we do not try to parse it as a numerical value
if (in_array($parameter['ParameterId'], self::TEXT_ONLY_PARAMETERS, true)) {
$results[] = new ParameterDTO(name: $parameter['Parameter'], value_text: $parameter['Value']);
$results[] = new ParameterDTO(name: $parameter['ParameterText'], value_text: $parameter['ValueText']);
} else { //Otherwise try to parse it as a numerical value
$results[] = ParameterDTO::parseValueIncludingUnit($parameter['Parameter'], $parameter['Value']);
$results[] = ParameterDTO::parseValueIncludingUnit($parameter['ParameterText'], $parameter['ValueText']);
}
}
@ -254,16 +277,22 @@ class DigikeyProvider implements InfoProviderInterface
}
/**
* @param array $media_links
* @param string $id The Digikey product number, to get the media for
* @return FileDTO[][]
* @phpstan-return array<string, FileDTO[]>
*/
private function mediaToDTOs(array $media_links): array
private function mediaToDTOs(string $id): array
{
$datasheets = [];
$images = [];
foreach ($media_links as $media_link) {
$response = $this->digikeyClient->request('GET', '/products/v4/search/' . urlencode($id) . '/media', [
'auth_bearer' => $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME)
]);
$media_array = $response->toArray();
foreach ($media_array['MediaLinks'] as $media_link) {
$file = new FileDTO(url: $media_link['Url'], name: $media_link['Title']);
switch ($media_link['MediaType']) {

29
src/Services/InfoProviderSystem/Providers/Element14Provider.php
View file

@ -29,14 +29,13 @@ use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO;
use App\Settings\InfoProviderSystem\Element14Settings;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class Element14Provider implements InfoProviderInterface
{
private const ENDPOINT_URL = 'https://api.element14.com/catalog/products';
private const API_VERSION_NUMBER = '1.2';
private const API_VERSION_NUMBER = '1.4';
private const NUMBER_OF_RESULTS = 20;
public const DISTRIBUTOR_NAME = 'Farnell';
@ -44,9 +43,19 @@ class Element14Provider implements InfoProviderInterface
private const COMPLIANCE_ATTRIBUTES = ['euEccn', 'hazardous', 'MSL', 'productTraceability', 'rohsCompliant',
'rohsPhthalatesCompliant', 'SVHC', 'tariffCode', 'usEccn', 'hazardCode'];
private readonly HttpClientInterface $element14Client;
public function __construct(private readonly HttpClientInterface $element14Client, private readonly Element14Settings $settings)
{
/* We use the mozilla CA from the composer ca bundle directly, as some debian systems seems to have problems
* with the SSL.COM CA, element14 uses. See https://github.com/Part-DB/Part-DB-server/issues/866
*
* This is a workaround until the issue is resolved in debian (or never).
* As this only affects this provider, this should have no negative impact and the CA bundle is still secure.
*/
$this->element14Client = $element14Client->withOptions([
'cafile' => CaBundle::getBundledCaBundlePath(),
]);
}
public function getProviderInfo(): array
@ -84,7 +93,7 @@ class Element14Provider implements InfoProviderInterface
'resultsSettings.responseGroup' => 'large',
'callInfo.apiKey' => $this->settings->apiKey,
'callInfo.responseDataFormat' => 'json',
'callInfo.version' => self::API_VERSION_NUMBER,
'versionNumber' => self::API_VERSION_NUMBER,
],
]);
@ -108,10 +117,12 @@ class Element14Provider implements InfoProviderInterface
mpn: $product['translatedManufacturerPartNumber'],
preview_image_url: $this->toImageUrl($product['image'] ?? null),
manufacturing_status: $this->releaseStatusCodeToManufacturingStatus($product['releaseStatusCode'] ?? null),
provider_url: $this->generateProductURL($product['sku']),
provider_url: $product['productURL'],
notes: $product['productOverview']['description'] ?? null,
datasheets: $this->parseDataSheets($product['datasheets'] ?? null),
parameters: $this->attributesToParameters($product['attributes'] ?? null),
vendor_infos: $this->pricesToVendorInfo($product['sku'], $product['prices'] ?? [])
vendor_infos: $this->pricesToVendorInfo($product['sku'], $product['prices'] ?? [], $product['productURL']),
);
}
@ -120,7 +131,7 @@ class Element14Provider implements InfoProviderInterface
private function generateProductURL($sku): string
{
return 'https://' . $this->settings->storeId . '/' . $sku;
return 'https://' . $this->store_id . '/' . $sku;
}
/**
@ -162,7 +173,7 @@ class Element14Provider implements InfoProviderInterface
* @param array $prices
* @return array
*/
private function pricesToVendorInfo(string $sku, array $prices): array
private function pricesToVendorInfo(string $sku, array $prices, string $product_url): array
{
$price_dtos = [];
@ -180,7 +191,7 @@ class Element14Provider implements InfoProviderInterface
distributor_name: self::DISTRIBUTOR_NAME,
order_number: $sku,
prices: $price_dtos,
product_url: $this->generateProductURL($sku)
product_url: $product_url
)
];
}

10
src/Services/InfoProviderSystem/Providers/MouserProvider.php
View file

@ -92,6 +92,7 @@ class MouserProvider implements InfoProviderInterface
From the startingRecord, the number of records specified will be returned up to the end of the recordset.
This is useful for paging through the complete recordset of parts matching keyword.
searchOptions string
Optional.
If not provided, the default is None.
@ -174,11 +175,16 @@ class MouserProvider implements InfoProviderInterface
throw new \RuntimeException('No part found with ID '.$id);
}
//Manually filter out the part with the correct ID
$tmp = array_filter($tmp, fn(PartDetailDTO $part) => $part->provider_id === $id);
if (count($tmp) === 0) {
throw new \RuntimeException('No part found with ID '.$id);
}
if (count($tmp) > 1) {
throw new \RuntimeException('Multiple parts found with ID '.$id . ' ('.count($tmp).' found). This is basically a bug in Mousers API response. See issue #616.');
throw new \RuntimeException('Multiple parts found with ID '.$id);
}
return $tmp[0];
return reset($tmp);
}
public function getCapabilities(): array

2
src/Services/InfoProviderSystem/Providers/OEMSecretsProvider.php
View file

@ -1218,7 +1218,7 @@ class OEMSecretsProvider implements InfoProviderInterface
* - 'value_min' => string|null The minimum value in a range, if applicable.
* - 'value_max' => string|null The maximum value in a range, if applicable.
*/
private function customSplitIntoValueAndUnit(string $value1, string $value2 = null): array
private function customSplitIntoValueAndUnit(string $value1, ?string $value2 = null): array
{
// Separate numbers and units (basic parsing handling)
$unit = null;

249
src/Services/InfoProviderSystem/Providers/PollinProvider.php Normal file
View file

@ -0,0 +1,249 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Entity\Parts\ManufacturingStatus;
use App\Entity\Parts\Part;
use App\Services\InfoProviderSystem\DTOs\FileDTO;
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class PollinProvider implements InfoProviderInterface
{
public function __construct(private readonly HttpClientInterface $client,
#[Autowire(env: 'bool:PROVIDER_POLLIN_ENABLED')]
private readonly bool $enabled = true,
)
{
}
public function getProviderInfo(): array
{
return [
'name' => 'Pollin',
'description' => 'Webscraping from pollin.de to get part information',
'url' => 'https://www.pollin.de/',
'disabled_help' => 'Set PROVIDER_POLLIN_ENABLED env to 1'
];
}
public function getProviderKey(): string
{
return 'pollin';
}
public function isActive(): bool
{
return $this->enabled;
}
public function searchByKeyword(string $keyword): array
{
$response = $this->client->request('GET', 'https://www.pollin.de/search', [
'query' => [
'search' => $keyword
]
]);
$content = $response->getContent();
//If the response has us redirected to the product page, then just return the single item
if ($response->getInfo('redirect_count') > 0) {
return [$this->parseProductPage($content)];
}
$dom = new Crawler($content);
$results = [];
//Iterate over each div.product-box
$dom->filter('div.product-box')->each(function (Crawler $node) use (&$results) {
$results[] = new SearchResultDTO(
provider_key: $this->getProviderKey(),
provider_id: $node->filter('meta[itemprop="productID"]')->attr('content'),
name: $node->filter('a.product-name')->text(),
description: '',
preview_image_url: $node->filter('img.product-image')->attr('src'),
manufacturing_status: $this->mapAvailability($node->filter('link[itemprop="availability"]')->attr('href')),
provider_url: $node->filter('a.product-name')->attr('href')
);
});
return $results;
}
private function mapAvailability(string $availabilityURI): ManufacturingStatus
{
return match( $availabilityURI) {
'http://schema.org/InStock' => ManufacturingStatus::ACTIVE,
'http://schema.org/OutOfStock' => ManufacturingStatus::DISCONTINUED,
default => ManufacturingStatus::NOT_SET
};
}
public function getDetails(string $id): PartDetailDTO
{
//Ensure that $id is numeric
if (!is_numeric($id)) {
throw new \InvalidArgumentException("The id must be numeric!");
}
$response = $this->client->request('GET', 'https://www.pollin.de/search', [
'query' => [
'search' => $id
]
]);
//The response must have us redirected to the product page
if ($response->getInfo('redirect_count') > 0) {
throw new \RuntimeException("Could not resolve the product page for the given id!");
}
$content = $response->getContent();
return $this->parseProductPage($content);
}
private function parseProductPage(string $content): PartDetailDTO
{
$dom = new Crawler($content);
$productPageUrl = $dom->filter('meta[property="product:product_link"]')->attr('content');
$orderId = trim($dom->filter('span[itemprop="sku"]')->text()); //Text is important here
//Calculate the mass
$massStr = $dom->filter('meta[itemprop="weight"]')->attr('content');
//Remove the unit
$massStr = str_replace('kg', '', $massStr);
//Convert to float and convert to grams
$mass = (float) $massStr * 1000;
//Parse purchase info
$purchaseInfo = new PurchaseInfoDTO('Pollin', $orderId, $this->parsePrices($dom), $productPageUrl);
return new PartDetailDTO(
provider_key: $this->getProviderKey(),
provider_id: $orderId,
name: trim($dom->filter('meta[property="og:title"]')->attr('content')),
description: $dom->filter('meta[property="og:description"]')->attr('content'),
category: $this->parseCategory($dom),
manufacturer: $dom->filter('meta[property="product:brand"]')->count() > 0 ? $dom->filter('meta[property="product:brand"]')->attr('content') : null,
preview_image_url: $dom->filter('meta[property="og:image"]')->attr('content'),
manufacturing_status: $this->mapAvailability($dom->filter('link[itemprop="availability"]')->attr('href')),
provider_url: $productPageUrl,
notes: $this->parseNotes($dom),
datasheets: $this->parseDatasheets($dom),
parameters: $this->parseParameters($dom),
vendor_infos: [$purchaseInfo],
mass: $mass,
);
}
private function parseDatasheets(Crawler $dom): array
{
//Iterate over each a element withing div.pol-product-detail-download-files
$datasheets = [];
$dom->filter('div.pol-product-detail-download-files a')->each(function (Crawler $node) use (&$datasheets) {
$datasheets[] = new FileDTO($node->attr('href'), $node->text());
});
return $datasheets;
}
private function parseParameters(Crawler $dom): array
{
$parameters = [];
//Iterate over each tr.properties-row inside table.product-detail-properties-table
$dom->filter('table.product-detail-properties-table tr.properties-row')->each(function (Crawler $node) use (&$parameters) {
$parameters[] = ParameterDTO::parseValueIncludingUnit(
name: rtrim($node->filter('th.properties-label')->text(), ':'),
value: trim($node->filter('td.properties-value')->text())
);
});
return $parameters;
}
private function parseCategory(Crawler $dom): string
{
$category = '';
//Iterate over each li.breadcrumb-item inside ol.breadcrumb
$dom->filter('ol.breadcrumb li.breadcrumb-item')->each(function (Crawler $node) use (&$category) {
//Skip if it has breadcrumb-item-home class
if (str_contains($node->attr('class'), 'breadcrumb-item-home')) {
return;
}
$category .= $node->text() . ' -> ';
});
//Remove the last ' -> '
return substr($category, 0, -4);
}
private function parseNotes(Crawler $dom): string
{
//Concat product highlights and product description
return $dom->filter('div.product-detail-top-features')->html('') . '<br><br>' . $dom->filter('div.product-detail-description-text')->html('');
}
private function parsePrices(Crawler $dom): array
{
//TODO: Properly handle multiple prices, for now we just look at the price for one piece
//We assume the currency is always the same
$currency = $dom->filter('meta[property="product:price:currency"]')->attr('content');
//If there is meta[property=highPrice] then use this as the price
if ($dom->filter('meta[itemprop="highPrice"]')->count() > 0) {
$price = $dom->filter('meta[itemprop="highPrice"]')->attr('content');
} else {
$price = $dom->filter('meta[property="product:price:amount"]')->attr('content');
}
return [
new PriceDTO(1.0, $price, $currency)
];
}
public function getCapabilities(): array
{
return [
ProviderCapabilities::BASIC,
ProviderCapabilities::PICTURE,
ProviderCapabilities::PRICE,
ProviderCapabilities::DATASHEET
];
}
}

285
src/Services/InfoProviderSystem/Providers/ReicheltProvider.php Normal file
View file

@ -0,0 +1,285 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\FileDTO;
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class ReicheltProvider implements InfoProviderInterface
{
public const DISTRIBUTOR_NAME = "Reichelt";
public function __construct(private readonly HttpClientInterface $client,
#[Autowire(env: "bool:PROVIDER_REICHELT_ENABLED")]
private readonly bool $enabled = true,
#[Autowire(env: "PROVIDER_REICHELT_LANGUAGE")]
private readonly string $language = "en",
#[Autowire(env: "PROVIDER_REICHELT_COUNTRY")]
private readonly string $country = "DE",
#[Autowire(env: "PROVIDER_REICHELT_INCLUDE_VAT")]
private readonly bool $includeVAT = false,
#[Autowire(env: "PROVIDER_REICHELT_CURRENCY")]
private readonly string $currency = "EUR",
)
{
}
public function getProviderInfo(): array
{
return [
'name' => 'Reichelt',
'description' => 'Webscraping from reichelt.com to get part information',
'url' => 'https://www.reichelt.com/',
'disabled_help' => 'Set PROVIDER_REICHELT_ENABLED env to 1'
];
}
public function getProviderKey(): string
{
return 'reichelt';
}
public function isActive(): bool
{
return $this->enabled;
}
public function searchByKeyword(string $keyword): array
{
$response = $this->client->request('GET', sprintf($this->getBaseURL() . '/shop/search/%s', $keyword));
$html = $response->getContent();
//Parse the HTML and return the results
$dom = new Crawler($html);
//Iterate over all div.al_gallery_article elements
$results = [];
$dom->filter('div.al_gallery_article')->each(function (Crawler $element) use (&$results) {
//Extract product id from data-product attribute
$artId = json_decode($element->attr('data-product'), true, 2, JSON_THROW_ON_ERROR)['artid'];
$productID = $element->filter('meta[itemprop="productID"]')->attr('content');
$name = $element->filter('meta[itemprop="name"]')->attr('content');
$sku = $element->filter('meta[itemprop="sku"]')->attr('content');
//Try to extract a picture URL:
$pictureURL = $element->filter("div.al_artlogo img")->attr('src');
$results[] = new SearchResultDTO(
provider_key: $this->getProviderKey(),
provider_id: $artId,
name: $productID,
description: $name,
category: null,
manufacturer: $sku,
preview_image_url: $pictureURL,
provider_url: $element->filter('a.al_artinfo_link')->attr('href')
);
});
return $results;
}
public function getDetails(string $id): PartDetailDTO
{
//Check that the ID is a number
if (!is_numeric($id)) {
throw new \InvalidArgumentException("Invalid ID");
}
//Use this endpoint to resolve the artID to a product page
$response = $this->client->request('GET',
sprintf(
'https://www.reichelt.com/?ACTION=514&id=74&article=%s&LANGUAGE=%s&CCOUNTRY=%s',
$id,
strtoupper($this->language),
strtoupper($this->country)
)
);
$json = $response->toArray();
//Retrieve the product page from the response
$productPage = $this->getBaseURL() . '/shop/product' . $json[0]['article_path'];
$response = $this->client->request('GET', $productPage, [
'query' => [
'CCTYPE' => $this->includeVAT ? 'private' : 'business',
'currency' => $this->currency,
],
]);
$html = $response->getContent();
$dom = new Crawler($html);
//Extract the product notes
$notes = $dom->filter('p[itemprop="description"]')->html();
//Extract datasheets
$datasheets = [];
$dom->filter('div.articleDatasheet a')->each(function (Crawler $element) use (&$datasheets) {
$datasheets[] = new FileDTO($element->attr('href'), $element->filter('span')->text());
});
//Determine price for one unit
$priceString = $dom->filter('meta[itemprop="price"]')->attr('content');
$currency = $dom->filter('meta[itemprop="priceCurrency"]')->attr('content', 'EUR');
//Create purchase info
$purchaseInfo = new PurchaseInfoDTO(
distributor_name: self::DISTRIBUTOR_NAME,
order_number: $json[0]['article_artnr'],
prices: array_merge(
[new PriceDTO(1.0, $priceString, $currency, $this->includeVAT)]
, $this->parseBatchPrices($dom, $currency)),
product_url: $productPage
);
//Create part object
return new PartDetailDTO(
provider_key: $this->getProviderKey(),
provider_id: $id,
name: $json[0]['article_artnr'],
description: $json[0]['article_besch'],
category: $this->parseCategory($dom),
manufacturer: $json[0]['manufacturer_name'],
mpn: $this->parseMPN($dom),
preview_image_url: $json[0]['article_picture'],
provider_url: $productPage,
notes: $notes,
datasheets: $datasheets,
parameters: $this->parseParameters($dom),
vendor_infos: [$purchaseInfo]
);
}
private function parseMPN(Crawler $dom): string
{
//Find the small element directly after meta[itemprop="url"] element
$element = $dom->filter('meta[itemprop="url"] + small');
//If the text contains GTIN text, take the small element afterwards
if (str_contains($element->text(), 'GTIN')) {
$element = $dom->filter('meta[itemprop="url"] + small + small');
}
//The MPN is contained in the span inside the element
return $element->filter('span')->text();
}
private function parseBatchPrices(Crawler $dom, string $currency): array
{
//Iterate over each a.inline-block element in div.discountValue
$prices = [];
$dom->filter('div.discountValue a.inline-block')->each(function (Crawler $element) use (&$prices, $currency) {
//The minimum amount is the number in the span.block element
$minAmountText = $element->filter('span.block')->text();
//Extract a integer from the text
$matches = [];
if (!preg_match('/\d+/', $minAmountText, $matches)) {
return;
}
$minAmount = (int) $matches[0];
//The price is the text of the p.productPrice element
$priceString = $element->filter('p.productPrice')->text();
//Replace comma with dot
$priceString = str_replace(',', '.', $priceString);
//Strip any non-numeric characters
$priceString = preg_replace('/[^0-9.]/', '', $priceString);
$prices[] = new PriceDTO($minAmount, $priceString, $currency, $this->includeVAT);
});
return $prices;
}
private function parseCategory(Crawler $dom): string
{
// Look for ol.breadcrumb and iterate over the li elements
$category = '';
$dom->filter('ol.breadcrumb li.triangle-left')->each(function (Crawler $element) use (&$category) {
//Do not include the .breadcrumb-showmore element
if ($element->attr('id') === 'breadcrumb-showmore') {
return;
}
$category .= $element->text() . ' -> ';
});
//Remove the trailing ' -> '
$category = substr($category, 0, -4);
return $category;
}
/**
* @param Crawler $dom
* @return ParameterDTO[]
*/
private function parseParameters(Crawler $dom): array
{
$parameters = [];
//Iterate over each ul.articleTechnicalData which contains the specifications of each group
$dom->filter('ul.articleTechnicalData')->each(function (Crawler $groupElement) use (&$parameters) {
$groupName = $groupElement->filter('li.articleTechnicalHeadline')->text();
//Iterate over each second li in ul.articleAttribute, which contains the specifications
$groupElement->filter('ul.articleAttribute li:nth-child(2n)')->each(function (Crawler $specElement) use (&$parameters, $groupName) {
$parameters[] = ParameterDTO::parseValueIncludingUnit(
name: $specElement->previousAll()->text(),
value: $specElement->text(),
group: $groupName
);
});
});
return $parameters;
}
private function getBaseURL(): string
{
//Without the trailing slash
return 'https://www.reichelt.com/' . strtolower($this->country) . '/' . strtolower($this->language);
}
public function getCapabilities(): array
{
return [
ProviderCapabilities::BASIC,
ProviderCapabilities::PICTURE,
ProviderCapabilities::DATASHEET,
ProviderCapabilities::PRICE,
];
}
}

10
src/Services/InfoProviderSystem/Providers/TMEClient.php
View file

@ -51,6 +51,16 @@ class TMEClient
return !($this->settings->apiToken === '' || $this->settings->apiSecret === '');
}
/**
* Returns true if the client is using a private (account related token) instead of a deprecated anonymous token
* to authenticate with TME.
* @return bool
*/
public function isUsingPrivateToken(): bool
{
//Private tokens are longer than anonymous ones (50 instead of 45 characters)
return strlen($this->token) > 45;
}
/**
* Generates the signature for the given action and parameters.

10
src/Services/InfoProviderSystem/Providers/TMEProvider.php
View file

@ -37,9 +37,15 @@ class TMEProvider implements InfoProviderInterface
private const VENDOR_NAME = 'TME';
private readonly bool $get_gross_prices;
public function __construct(private readonly TMEClient $tmeClient, private readonly TMESettings $settings)
{
//If we have a private token, set get_gross_prices to false, as it is automatically determined by the account type then
if ($this->tmeClient->isUsingPrivateToken()) {
$this->get_gross_prices = false;
} else {
$this->get_gross_prices = $get_gross_prices;
}
}
public function getProviderInfo(): array
@ -185,7 +191,7 @@ class TMEProvider implements InfoProviderInterface
'Country' => $this->settings->country,
'Language' => $this->settings->language,
'Currency' => $this->settings->currency,
'GrossPrices' => $this->settings->grossPrices,
'GrossPrices' => $this->get_gross_prices,
'SymbolList' => [$id],
]);