From a57d46d57b2badab8727125391b72635016bf59d Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Fri, 21 Feb 2025 09:32:54 +0100 Subject: [PATCH 1/6] Create BuerklinProvider based on LCSCProvider --- .docker/symfony.conf | 1 + docs/usage/information_provider_system.md | 19 + .../Providers/BuerklinProvider.php | 366 ++++++++++++++++++ 3 files changed, 386 insertions(+) create mode 100644 src/Services/InfoProviderSystem/Providers/BuerklinProvider.php diff --git a/.docker/symfony.conf b/.docker/symfony.conf index b5229bf6..0525e820 100644 --- a/.docker/symfony.conf +++ b/.docker/symfony.conf @@ -46,6 +46,7 @@ PassEnv PROVIDER_OEMSECRETS_KEY PROVIDER_OEMSECRETS_COUNTRY_CODE PROVIDER_OEMSECRETS_CURRENCY PROVIDER_OEMSECRETS_ZERO_PRICE PROVIDER_OEMSECRETS_SET_PARAM PROVIDER_OEMSECRETS_SORT_CRITERIA PassEnv PROVIDER_REICHELT_ENABLED PROVIDER_REICHELT_CURRENCY PROVIDER_REICHELT_COUNTRY PROVIDER_REICHELT_LANGUAGE PROVIDER_REICHELT_INCLUDE_VAT PassEnv PROVIDER_POLLIN_ENABLED + PassEnv PROVIDER_BUERKLIN_CLIENT_ID PROVIDER_BUERKLIN_SECRET PROVIDER_BUERKLIN_USERNAME PROVIDER_BUERKLIN_PASSWORD PROVIDER_BUERKLIN_CURRENCY PROVIDER_BUERKLIN_LANGUAGE PassEnv EDA_KICAD_CATEGORY_DEPTH # For most configuration files from conf-available/, which are diff --git a/docs/usage/information_provider_system.md b/docs/usage/information_provider_system.md index 015a9eb3..0407bc0e 100644 --- a/docs/usage/information_provider_system.md +++ b/docs/usage/information_provider_system.md @@ -255,6 +255,25 @@ This is not an official API and could break at any time. So use it at your own r The following env configuration options are available: * `PROVIDER_POLLIN_ENABLED`: Set this to `1` to enable the Pollin provider +### Buerklin + +The Buerklin provider uses the [Buerklin API](https://www.buerklin.com/en/services/eprocurement/) to search for parts and get information. +To use it you have to request access to the API. +You will get an e-mail with the client ID and client secret, which you have to put in the Part-DB env configuration (see below). + +Please note that the Buerklin API is limited to 100 requests/minute per IP address and +access to the Authentication server is limited to 10 requests/minute per IP address + +The following env configuration options are available: + +* `PROVIDER_BUERKLIN_CLIENT_ID`: The client ID you got from Buerklin (mandatory) +* `PROVIDER_BUERKLIN_SECRET`: The client secret you got from Buerklin (mandatory) +* `PROVIDER_BUERKLIN_USERNAME`: The username you got from Buerklin (mandatory) +* `PROVIDER_BUERKLIN_PASSWORD`: The password you got from Buerklin (mandatory) +* `PROVIDER_BUERKLIN_CURRENCY`: The currency you want to get prices in if available (optional, 3 letter ISO-code, + default: `EUR`). +* `PROVIDER_BUERKLIN_LANGUAGE`: The language you want to get the descriptions in. Possible values: `de` = German, `en` = English. (optional, default: `de`) + ### Custom provider To create a custom provider, you have to create a new class implementing the `InfoProviderInterface` interface. As long diff --git a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php new file mode 100644 index 00000000..7e265b08 --- /dev/null +++ b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php @@ -0,0 +1,366 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem\Providers; + +use App\Services\InfoProviderSystem\DTOs\FileDTO; +use App\Services\InfoProviderSystem\DTOs\ParameterDTO; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\PriceDTO; +use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; +use Symfony\Component\HttpFoundation\Cookie; +use Symfony\Contracts\HttpClient\HttpClientInterface; + +class BuerklinProvider implements InfoProviderInterface +{ + + private const ENDPOINT_URL = 'https://buerklin.com/buerklinws/v2/buerklin/'; + + public const DISTRIBUTOR_NAME = 'Buerklin'; + + public function __construct(private readonly HttpClientInterface $buerklinClient, private readonly string $currency, private readonly bool $enabled = true) + { + + } + + public function getProviderInfo(): array + { + return [ + 'name' => 'Buerklin', + 'description' => 'This provider uses the Buerklin API to search for parts.', + 'url' => 'https://www.buerklin.com/', + 'disabled_help' => 'Set PROVIDER_BUERKLIN_ENABLED to 1 (or true) in your environment variable config.' + ]; + } + + public function getProviderKey(): string + { + return 'buerklin'; + } + + // This provider is always active + public function isActive(): bool + { + return $this->enabled; + } + + /** + * @param string $id + * @return PartDetailDTO + */ + private function queryDetail(string $id): PartDetailDTO + { + $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "/products", [ + 'headers' => [ + 'Cookie' => new Cookie('currencyCode', $this->currency) + ], + 'query' => [ + 'sku' => $id, + ], + ]); + + $arr = $response->toArray(); + $product = $arr['result'] ?? null; + + if ($product === null) { + throw new \RuntimeException('Could not find product code: ' . $id); + } + + return $this->getPartDetail($product); + } + + /** + * @param string $url + * @return String + */ + private function getRealDatasheetUrl(?string $url): string + { + if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.buerklin\.com|www\.buerklin\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) { + if (preg_match("/^https:\/\/datasheet\.buerklin\.com\/buerklin\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) { + $url = 'https://www.buerklin.com/datasheet/buerklin_datasheet_' . $rewriteMatches[1]; + } + $response = $this->buerklinClient->request('GET', $url, [ + 'headers' => [ + 'Referer' => 'https://www.buerklin.com/product-detail/_' . $matches[2] . '.html' + ], + ]); + if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.buerklin\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) { + //HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused + //See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934 + $jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}'); + $url = $jsonObj->previewPdfUrl; + } + } + return $url; + } + + /** + * @param string $term + * @return PartDetailDTO[] + */ + private function queryByTerm(string $term): array + { + $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "/search/global", [ + 'headers' => [ + 'Cookie' => new Cookie('currencyCode', $this->currency) + ], + 'query' => [ + 'keyword' => $term, + ], + ]); + + $arr = $response->toArray(); + + // Get products list + $products = $arr['result']['productSearchResultVO']['productList'] ?? []; + // Get product tip + $tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null; + + $result = []; + + // Buerklin does not display Buerklin codes in the search, instead taking you directly to the + // detailed product listing. It does so utilizing a product tip field. + // If product tip exists and there are no products in the product list try a detail query + if (count($products) === 0 && $tipProductCode !== null) { + $result[] = $this->queryDetail($tipProductCode); + } + + foreach ($products as $product) { + $result[] = $this->getPartDetail($product); + } + + return $result; + } + + /** + * Sanitizes a field by removing any HTML tags and other unwanted characters + * @param string|null $field + * @return string|null + */ + private function sanitizeField(?string $field): ?string + { + if ($field === null) { + return null; + } + + return strip_tags($field); + } + + + /** + * Takes a deserialized json object of the product and returns a PartDetailDTO + * @param array $product + * @return PartDetailDTO + */ + private function getPartDetail(array $product): PartDetailDTO + { + // Get product images in advance + $product_images = $this->getProductImages($product['productImages'] ?? null); + $product['productImageUrl'] ??= null; + + // If the product does not have a product image but otherwise has attached images, use the first one. + if (count($product_images) > 0) { + $product['productImageUrl'] ??= $product_images[0]->url; + } + + // Buerklin puts HTML in footprints and descriptions sometimes randomly + $footprint = $product["encapStandard"] ?? null; + //If the footprint just consists of a dash, we'll assume it's empty + if ($footprint === '-') { + $footprint = null; + } + + //Build category by concatenating the catalogName and parentCatalogName + $category = $product['parentCatalogName'] ?? null; + if (isset($product['catalogName'])) { + $category = ($category ?? '') . ' -> ' . $product['catalogName']; + + // Replace the / with a -> for better readability + $category = str_replace('/', ' -> ', $category); + } + + return new PartDetailDTO( + provider_key: $this->getProviderKey(), + provider_id: $product['productCode'], + name: $product['productModel'], + description: $this->sanitizeField($product['productIntroEn']), + category: $this->sanitizeField($category ?? null), + manufacturer: $this->sanitizeField($product['brandNameEn'] ?? null), + mpn: $this->sanitizeField($product['productModel'] ?? null), + preview_image_url: $product['productImageUrl'], + manufacturing_status: null, + provider_url: $this->getProductShortURL($product['productCode']), + footprint: $this->sanitizeField($footprint), + datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null), + images: $product_images, + parameters: $this->attributesToParameters($product['paramVOList'] ?? []), + vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []), + mass: $product['weight'] ?? null, + ); + } + + /** + * Converts the price array to a VendorInfoDTO array to be used in the PartDetailDTO + * @param string $sku + * @param string $url + * @param array $prices + * @return array + */ + private function pricesToVendorInfo(string $sku, string $url, array $prices): array + { + $price_dtos = []; + + foreach ($prices as $price) { + $price_dtos[] = new PriceDTO( + minimum_discount_amount: $price['ladder'], + price: $price['productPrice'], + currency_iso_code: $this->getUsedCurrency($price['currencySymbol']), + includes_tax: false, + ); + } + + return [ + new PurchaseInfoDTO( + distributor_name: self::DISTRIBUTOR_NAME, + order_number: $sku, + prices: $price_dtos, + product_url: $url, + ) + ]; + } + + /** + * Converts Buerklin currency symbol to an ISO code. + * @param string $currency + * @return string + */ + private function getUsedCurrency(string $currency): string + { + //Decide based on the currency symbol + return match ($currency) { + 'US$', '$' => 'USD', + '€' => 'EUR', + 'A$' => 'AUD', + 'C$' => 'CAD', + '£' => 'GBP', + 'HK$' => 'HKD', + 'JP¥' => 'JPY', + 'RM' => 'MYR', + 'S$' => 'SGD', + '₽' => 'RUB', + 'kr' => 'SEK', + 'kr.' => 'DKK', + '₹' => 'INR', + //Fallback to the configured currency + default => $this->currency, + }; + } + + /** + * Returns a valid Buerklin product short URL from product code + * @param string $product_code + * @return string + */ + private function getProductShortURL(string $product_code): string + { + return 'https://www.buerklin.com/product-detail/' . $product_code .'.html'; + } + + /** + * Returns a product datasheet FileDTO array from a single pdf url + * @param string $url + * @return FileDTO[] + */ + private function getProductDatasheets(?string $url): array + { + if ($url === null) { + return []; + } + + $realUrl = $this->getRealDatasheetUrl($url); + + return [new FileDTO($realUrl, null)]; + } + + /** + * Returns a FileDTO array with a list of product images + * @param array|null $images + * @return FileDTO[] + */ + private function getProductImages(?array $images): array + { + return array_map(static fn($image) => new FileDTO($image), $images ?? []); + } + + /** + * @param array|null $attributes + * @return ParameterDTO[] + */ + private function attributesToParameters(?array $attributes): array + { + $result = []; + + foreach ($attributes as $attribute) { + + //Skip this attribute if it's empty + if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) { + continue; + } + + $result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null); + } + + return $result; + } + + public function searchByKeyword(string $keyword): array + { + return $this->queryByTerm($keyword); + } + + public function getDetails(string $id): PartDetailDTO + { + $tmp = $this->queryByTerm($id); + if (count($tmp) === 0) { + throw new \RuntimeException('No part found with ID ' . $id); + } + + if (count($tmp) > 1) { + throw new \RuntimeException('Multiple parts found with ID ' . $id); + } + + return $tmp[0]; + } + + public function getCapabilities(): array + { + return [ + ProviderCapabilities::BASIC, + ProviderCapabilities::PICTURE, + ProviderCapabilities::DATASHEET, + ProviderCapabilities::PRICE, + ProviderCapabilities::FOOTPRINT, + ]; + } +} From 7a0a4aef51c8024df301a42747ce7ee6e61296f9 Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Sun, 23 Feb 2025 00:39:45 +0100 Subject: [PATCH 2/6] Update GET URLs for Buerklin --- .../InfoProviderSystem/Providers/BuerklinProvider.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php index 7e265b08..bd1b2db3 100644 --- a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php +++ b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php @@ -102,7 +102,7 @@ class BuerklinProvider implements InfoProviderInterface } $response = $this->buerklinClient->request('GET', $url, [ 'headers' => [ - 'Referer' => 'https://www.buerklin.com/product-detail/_' . $matches[2] . '.html' + 'Referer' => 'https://www.buerklin.com/de/p/' . $matches[2] . '/' ], ]); if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.buerklin\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) { @@ -121,7 +121,7 @@ class BuerklinProvider implements InfoProviderInterface */ private function queryByTerm(string $term): array { - $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "/search/global", [ + $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "products/search/?curr=$this->currency&language=en&pageSize=50¤tPage=0&query=Laser&sort=relevance", [ 'headers' => [ 'Cookie' => new Cookie('currencyCode', $this->currency) ], @@ -284,7 +284,7 @@ class BuerklinProvider implements InfoProviderInterface */ private function getProductShortURL(string $product_code): string { - return 'https://www.buerklin.com/product-detail/' . $product_code .'.html'; + return 'https://www.buerklin.com/de/p/' . $product_code .'/'; } /** From de5fad7326f0bd1a3271b9e12cd36709196ba3ab Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Mon, 24 Feb 2025 09:38:42 +0100 Subject: [PATCH 3/6] Add getToken function analog to Octopart --- .../Providers/BuerklinProvider.php | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php index 7e265b08..9b32b613 100644 --- a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php +++ b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php @@ -30,6 +30,7 @@ use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; use App\Services\InfoProviderSystem\DTOs\PriceDTO; use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; use Symfony\Component\HttpFoundation\Cookie; +use Symfony\Component\HttpClient\HttpOptions; use Symfony\Contracts\HttpClient\HttpClientInterface; class BuerklinProvider implements InfoProviderInterface @@ -38,19 +39,43 @@ class BuerklinProvider implements InfoProviderInterface private const ENDPOINT_URL = 'https://buerklin.com/buerklinws/v2/buerklin/'; public const DISTRIBUTOR_NAME = 'Buerklin'; + private const OAUTH_APP_NAME = 'ip_buerklin_oauth'; - public function __construct(private readonly HttpClientInterface $buerklinClient, private readonly string $currency, private readonly bool $enabled = true) + public function __construct(private readonly HttpClientInterface $httpClient, + private readonly OAuthTokenManager $authTokenManager, private readonly CacheItemPoolInterface $partInfoCache, + private readonly string $clientId, private readonly string $secret, + private readonly int $username, private readonly bool $password, + private readonly string $currency, private readonly string $language) { } + /** + * Gets the latest OAuth token for the Buerklin API, or creates a new one if none is available + * @return string + */ + private function getToken(): string + { + //Check if we already have a token saved for this app, otherwise we have to retrieve one via OAuth + if (!$this->authTokenManager->hasToken(self::OAUTH_APP_NAME)) { + $this->authTokenManager->retrieveClientCredentialsToken(self::OAUTH_APP_NAME); + } + + $tmp = $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME); + if ($tmp === null) { + throw new \RuntimeException('Could not retrieve OAuth token for Buerklin'); + } + + return $tmp; + } + public function getProviderInfo(): array { return [ 'name' => 'Buerklin', 'description' => 'This provider uses the Buerklin API to search for parts.', 'url' => 'https://www.buerklin.com/', - 'disabled_help' => 'Set PROVIDER_BUERKLIN_ENABLED to 1 (or true) in your environment variable config.' + 'disabled_help' => 'Set the environment variables PROVIDER_BUERKLIN_CLIENT_ID, PROVIDER_BUERKLIN_SECRET, PROVIDER_BUERKLIN_USERNAME and PROVIDER_BUERKLIN_PASSWORD.' ]; } @@ -62,7 +87,8 @@ class BuerklinProvider implements InfoProviderInterface // This provider is always active public function isActive(): bool { - return $this->enabled; + //The client ID has to be set and a token has to be available (user clicked connect) + return $this->clientId !== '' && $this->secret !== '' && $this->username !== '' && $this->password !== ''; } /** From d87b09453c8c6cdf09e2df8ed55e166974926d7b Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Mon, 24 Feb 2025 09:46:50 +0100 Subject: [PATCH 4/6] Remove line break in docs --- docs/usage/information_provider_system.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/usage/information_provider_system.md b/docs/usage/information_provider_system.md index 0407bc0e..fd40e77f 100644 --- a/docs/usage/information_provider_system.md +++ b/docs/usage/information_provider_system.md @@ -270,8 +270,7 @@ The following env configuration options are available: * `PROVIDER_BUERKLIN_SECRET`: The client secret you got from Buerklin (mandatory) * `PROVIDER_BUERKLIN_USERNAME`: The username you got from Buerklin (mandatory) * `PROVIDER_BUERKLIN_PASSWORD`: The password you got from Buerklin (mandatory) -* `PROVIDER_BUERKLIN_CURRENCY`: The currency you want to get prices in if available (optional, 3 letter ISO-code, - default: `EUR`). +* `PROVIDER_BUERKLIN_CURRENCY`: The currency you want to get prices in if available (optional, 3 letter ISO-code, default: `EUR`). * `PROVIDER_BUERKLIN_LANGUAGE`: The language you want to get the descriptions in. Possible values: `de` = German, `en` = English. (optional, default: `de`) ### Custom provider From ac8329781102e58f39e6d0a8b4cfb60c6d46c89a Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Mon, 24 Feb 2025 15:24:56 +0100 Subject: [PATCH 5/6] Remove trailing / in ENDPOINT_URL Use Autowire to use values of environment variables Remove unwanted Code from LCSC-Provider Map json response to DTO variables --- .../Providers/BuerklinProvider.php | 250 +++++++----------- 1 file changed, 97 insertions(+), 153 deletions(-) diff --git a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php index fa7cb712..7e70cc5f 100644 --- a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php +++ b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php @@ -29,23 +29,36 @@ use App\Services\InfoProviderSystem\DTOs\ParameterDTO; use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; use App\Services\InfoProviderSystem\DTOs\PriceDTO; use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; -use Symfony\Component\HttpFoundation\Cookie; +use App\Services\OAuth\OAuthTokenManager; +use Psr\Cache\CacheItemPoolInterface; +use Symfony\Component\DependencyInjection\Attribute\Autowire; use Symfony\Component\HttpClient\HttpOptions; use Symfony\Contracts\HttpClient\HttpClientInterface; class BuerklinProvider implements InfoProviderInterface { - private const ENDPOINT_URL = 'https://buerklin.com/buerklinws/v2/buerklin/'; + private const ENDPOINT_URL = 'https://buerklin.com/buerklinws/v2/buerklin'; public const DISTRIBUTOR_NAME = 'Buerklin'; private const OAUTH_APP_NAME = 'ip_buerklin_oauth'; - public function __construct(private readonly HttpClientInterface $httpClient, - private readonly OAuthTokenManager $authTokenManager, private readonly CacheItemPoolInterface $partInfoCache, - private readonly string $clientId, private readonly string $secret, - private readonly int $username, private readonly bool $password, - private readonly string $currency, private readonly string $language) + public function __construct(private readonly HttpClientInterface $client, + private readonly OAuthTokenManager $authTokenManager, + private readonly CacheItemPoolInterface $partInfoCache, + #[Autowire(env: "string:PROVIDER_BUERKLIN_CLIENT_ID")] + private readonly string $clientId = "", + #[Autowire(env: "string:PROVIDER_BUERKLIN_SECRET")] + private readonly string $secret = "", + #[Autowire(env: "string:PROVIDER_BUERKLIN_USERNAME")] + private readonly string $username = "", + #[Autowire(env: "string:PROVIDER_BUERKLIN_PASSWORD")] + private readonly string $password = "", + #[Autowire(env: "PROVIDER_BUERKLIN_LANGUAGE")] + private readonly string $language = "en", + #[Autowire(env: "PROVIDER_BUERKLIN_CURRENCY")] + private readonly string $currency = "EUR" + ) { } @@ -69,6 +82,30 @@ class BuerklinProvider implements InfoProviderInterface return $tmp; } + /** + * Make a http get request to the Buerklin API + * @return array + */ + private function makeAPICall(string $query, ?array $variables = null): array + { + if ($variables === []) { + $variables = null; + } + + $options = (new HttpOptions()) + ->setJson(['query' => $query, 'variables' => $variables]) + ->setAuthBearer($this->getToken()) + ; + + $response = $this->client->request( + 'GET', + self::ENDPOINT_URL, + $options->toArray(), + ); + + return $response->toArray(true); + } + public function getProviderInfo(): array { return [ @@ -97,10 +134,7 @@ class BuerklinProvider implements InfoProviderInterface */ private function queryDetail(string $id): PartDetailDTO { - $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "/products", [ - 'headers' => [ - 'Cookie' => new Cookie('currencyCode', $this->currency) - ], + $response = $this->client->request('GET', self::ENDPOINT_URL . "/products", [ 'query' => [ 'sku' => $id, ], @@ -116,69 +150,6 @@ class BuerklinProvider implements InfoProviderInterface return $this->getPartDetail($product); } - /** - * @param string $url - * @return String - */ - private function getRealDatasheetUrl(?string $url): string - { - if ($url !== null && trim($url) !== '' && preg_match("/^https:\/\/(datasheet\.buerklin\.com|www\.buerklin\.com\/datasheet)\/.*(C\d+)\.pdf$/", $url, $matches) > 0) { - if (preg_match("/^https:\/\/datasheet\.buerklin\.com\/buerklin\/(.*\.pdf)$/", $url, $rewriteMatches) > 0) { - $url = 'https://www.buerklin.com/datasheet/buerklin_datasheet_' . $rewriteMatches[1]; - } - $response = $this->buerklinClient->request('GET', $url, [ - 'headers' => [ - 'Referer' => 'https://www.buerklin.com/de/p/' . $matches[2] . '/' - ], - ]); - if (preg_match('/(previewPdfUrl): ?("[^"]+wmsc\.buerklin\.com[^"]+\.pdf")/', $response->getContent(), $matches) > 0) { - //HACKY: The URL string contains escaped characters like \u002F, etc. To decode it, the JSON decoding is reused - //See https://github.com/Part-DB/Part-DB-server/pull/582#issuecomment-2033125934 - $jsonObj = json_decode('{"' . $matches[1] . '": ' . $matches[2] . '}'); - $url = $jsonObj->previewPdfUrl; - } - } - return $url; - } - - /** - * @param string $term - * @return PartDetailDTO[] - */ - private function queryByTerm(string $term): array - { - $response = $this->buerklinClient->request('GET', self::ENDPOINT_URL . "products/search/?curr=$this->currency&language=en&pageSize=50¤tPage=0&query=Laser&sort=relevance", [ - 'headers' => [ - 'Cookie' => new Cookie('currencyCode', $this->currency) - ], - 'query' => [ - 'keyword' => $term, - ], - ]); - - $arr = $response->toArray(); - - // Get products list - $products = $arr['result']['productSearchResultVO']['productList'] ?? []; - // Get product tip - $tipProductCode = $arr['result']['tipProductDetailUrlVO']['productCode'] ?? null; - - $result = []; - - // Buerklin does not display Buerklin codes in the search, instead taking you directly to the - // detailed product listing. It does so utilizing a product tip field. - // If product tip exists and there are no products in the product list try a detail query - if (count($products) === 0 && $tipProductCode !== null) { - $result[] = $this->queryDetail($tipProductCode); - } - - foreach ($products as $product) { - $result[] = $this->getPartDetail($product); - } - - return $result; - } - /** * Sanitizes a field by removing any HTML tags and other unwanted characters * @param string|null $field @@ -193,7 +164,6 @@ class BuerklinProvider implements InfoProviderInterface return strip_tags($field); } - /** * Takes a deserialized json object of the product and returns a PartDetailDTO * @param array $product @@ -202,46 +172,38 @@ class BuerklinProvider implements InfoProviderInterface private function getPartDetail(array $product): PartDetailDTO { // Get product images in advance - $product_images = $this->getProductImages($product['productImages'] ?? null); + $product_images = $this->getProductImages($product['images'] ?? null); $product['productImageUrl'] ??= null; - // If the product does not have a product image but otherwise has attached images, use the first one. + // If the product does not have a product image but otherwise has attached images, use the first one which should be thumbnail. if (count($product_images) > 0) { - $product['productImageUrl'] ??= $product_images[0]->url; + $product['productImageUrl'] ??= self::ENDPOINT_URL . $product_images[0]->url; } - // Buerklin puts HTML in footprints and descriptions sometimes randomly - $footprint = $product["encapStandard"] ?? null; - //If the footprint just consists of a dash, we'll assume it's empty - if ($footprint === '-') { - $footprint = null; - } - - //Build category by concatenating the catalogName and parentCatalogName - $category = $product['parentCatalogName'] ?? null; - if (isset($product['catalogName'])) { - $category = ($category ?? '') . ' -> ' . $product['catalogName']; - - // Replace the / with a -> for better readability - $category = str_replace('/', ' -> ', $category); + // Find the footprint in classifications->features. en: name='Design'; de: name='Bauform' + foreach ($product[classifications][features] as $feature) { + if($feature[name]=='Design'||$feature[name]=='Bauform') + { + $footprint = $feature["featureValues"]["value"]; + } } return new PartDetailDTO( provider_key: $this->getProviderKey(), - provider_id: $product['productCode'], - name: $product['productModel'], - description: $this->sanitizeField($product['productIntroEn']), - category: $this->sanitizeField($category ?? null), - manufacturer: $this->sanitizeField($product['brandNameEn'] ?? null), - mpn: $this->sanitizeField($product['productModel'] ?? null), + provider_id: $product['code'], + name: $product['name'], + description: $this->sanitizeField($product['description']), + category: $this->sanitizeField($product['classifications'][0]['name'] ?? null), + manufacturer: $this->sanitizeField($product['manufacturer'] ?? null), + mpn: $this->sanitizeField($product['manufacturerProductId'] ?? null), preview_image_url: $product['productImageUrl'], manufacturing_status: null, - provider_url: $this->getProductShortURL($product['productCode']), - footprint: $this->sanitizeField($footprint), - datasheets: $this->getProductDatasheets($product['pdfUrl'] ?? null), + provider_url: $this->getProductShortURL($product['code']), + footprint: $footprint ?? null, + datasheets: null, //datasheet urls not found in API responses images: $product_images, - parameters: $this->attributesToParameters($product['paramVOList'] ?? []), - vendor_infos: $this->pricesToVendorInfo($product['productCode'], $this->getProductShortURL($product['productCode']), $product['productPriceList'] ?? []), + parameters: $this->attributesToParameters($product['classifications']['features'] ?? []), + vendor_infos: $this->pricesToVendorInfo($product['code'], $this->getProductShortURL($product['code']), $product['productPriceList'] ?? []), mass: $product['weight'] ?? null, ); } @@ -259,9 +221,9 @@ class BuerklinProvider implements InfoProviderInterface foreach ($prices as $price) { $price_dtos[] = new PriceDTO( - minimum_discount_amount: $price['ladder'], - price: $price['productPrice'], - currency_iso_code: $this->getUsedCurrency($price['currencySymbol']), + minimum_discount_amount: $price['minQuantity'], + price: $price['value'], + currency_iso_code: $price['currencyIso'], includes_tax: false, ); } @@ -276,33 +238,6 @@ class BuerklinProvider implements InfoProviderInterface ]; } - /** - * Converts Buerklin currency symbol to an ISO code. - * @param string $currency - * @return string - */ - private function getUsedCurrency(string $currency): string - { - //Decide based on the currency symbol - return match ($currency) { - 'US$', '$' => 'USD', - '€' => 'EUR', - 'A$' => 'AUD', - 'C$' => 'CAD', - '£' => 'GBP', - 'HK$' => 'HKD', - 'JP¥' => 'JPY', - 'RM' => 'MYR', - 'S$' => 'SGD', - '₽' => 'RUB', - 'kr' => 'SEK', - 'kr.' => 'DKK', - '₹' => 'INR', - //Fallback to the configured currency - default => $this->currency, - }; - } - /** * Returns a valid Buerklin product short URL from product code * @param string $product_code @@ -313,22 +248,6 @@ class BuerklinProvider implements InfoProviderInterface return 'https://www.buerklin.com/de/p/' . $product_code .'/'; } - /** - * Returns a product datasheet FileDTO array from a single pdf url - * @param string $url - * @return FileDTO[] - */ - private function getProductDatasheets(?string $url): array - { - if ($url === null) { - return []; - } - - $realUrl = $this->getRealDatasheetUrl($url); - - return [new FileDTO($realUrl, null)]; - } - /** * Returns a FileDTO array with a list of product images * @param array|null $images @@ -350,11 +269,11 @@ class BuerklinProvider implements InfoProviderInterface foreach ($attributes as $attribute) { //Skip this attribute if it's empty - if (in_array(trim((string) $attribute['paramValueEn']), ['', '-'], true)) { + if (in_array(trim((string) $attribute['featureValues']['value']), ['', '-'], true)) { continue; } - $result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null); + $result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['name'], value: $attribute['featureValues']['value'], group: null); } return $result; @@ -362,12 +281,37 @@ class BuerklinProvider implements InfoProviderInterface public function searchByKeyword(string $keyword): array { - return $this->queryByTerm($keyword); + $response = $this->client->request('GET', self::ENDPOINT_URL . "products/search/", [ + 'auth_bearer' => $this->getToken(), + 'query' => [ + 'curr' => $this->currency, + 'language' => $this->language, + 'pageSize' => '50', + 'currentPage' => '1', + 'query' => $term, + 'sort' => 'relevance' + ], + ]); + + $arr = $response->toArray(); + + // Get products list + $products = $arr['products'] ?? []; + + $result = []; + + foreach ($products as $product) { + $result[] = $this->getPartDetail($product); + } + + return $result; } public function getDetails(string $id): PartDetailDTO { - $tmp = $this->queryByTerm($id); + $tmp = $this->searchByKeyword($id); + + if (count($tmp) === 0) { throw new \RuntimeException('No part found with ID ' . $id); } From 43801bbcde30525bb5e5d138dd8af90fead62697 Mon Sep 17 00:00:00 2001 From: Marc Kreidler Date: Tue, 25 Feb 2025 16:40:20 +0100 Subject: [PATCH 6/6] =?UTF-8?q?Fix=20variable=20reference=20errors=20($ter?= =?UTF-8?q?m=20=E2=86=92=20$keyword)=20Ensure=20array=20keys=20exist=20bef?= =?UTF-8?q?ore=20accessing=20them=20Optimize=20API=20calls=20to=20prevent?= =?UTF-8?q?=20unnecessary=20requests=20Improve=20error=20handling=20for=20?= =?UTF-8?q?better=20debugging=20Enhance=20readability=20and=20maintainabil?= =?UTF-8?q?ity=20of=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Providers/BuerklinProvider.php | 154 ++++++++---------- 1 file changed, 67 insertions(+), 87 deletions(-) diff --git a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php index 7e70cc5f..01fe9ca8 100644 --- a/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php +++ b/src/Services/InfoProviderSystem/Providers/BuerklinProvider.php @@ -69,43 +69,40 @@ class BuerklinProvider implements InfoProviderInterface */ private function getToken(): string { - //Check if we already have a token saved for this app, otherwise we have to retrieve one via OAuth - if (!$this->authTokenManager->hasToken(self::OAUTH_APP_NAME)) { - $this->authTokenManager->retrieveClientCredentialsToken(self::OAUTH_APP_NAME); + if ($this->authTokenManager->hasToken(self::OAUTH_APP_NAME)) { + return $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME); } - - $tmp = $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME); - if ($tmp === null) { + + $this->authTokenManager->retrieveClientCredentialsToken(self::OAUTH_APP_NAME); + $token = $this->authTokenManager->getAlwaysValidTokenString(self::OAUTH_APP_NAME); + + if ($token === null) { throw new \RuntimeException('Could not retrieve OAuth token for Buerklin'); } - - return $tmp; + + return $token; } + /** * Make a http get request to the Buerklin API * @return array */ - private function makeAPICall(string $query, ?array $variables = null): array + private function makeAPICall(string $endpoint, array $queryParams = []): array { - if ($variables === []) { - $variables = null; + try { + $response = $this->client->request('GET', self::ENDPOINT_URL . $endpoint, [ + 'auth_bearer' => $this->getToken(), + 'query' => $queryParams, + ]); + + return $response->toArray(); + } catch (\Exception $e) { + throw new \RuntimeException("Buerklin API request failed: " . $e->getMessage()); } - - $options = (new HttpOptions()) - ->setJson(['query' => $query, 'variables' => $variables]) - ->setAuthBearer($this->getToken()) - ; - - $response = $this->client->request( - 'GET', - self::ENDPOINT_URL, - $options->toArray(), - ); - - return $response->toArray(true); } + public function getProviderInfo(): array { return [ @@ -134,11 +131,7 @@ class BuerklinProvider implements InfoProviderInterface */ private function queryDetail(string $id): PartDetailDTO { - $response = $this->client->request('GET', self::ENDPOINT_URL . "/products", [ - 'query' => [ - 'sku' => $id, - ], - ]); + $response = $this->makeAPICall('/products', ['sku' => $id]); $arr = $response->toArray(); $product = $arr['result'] ?? null; @@ -181,10 +174,12 @@ class BuerklinProvider implements InfoProviderInterface } // Find the footprint in classifications->features. en: name='Design'; de: name='Bauform' - foreach ($product[classifications][features] as $feature) { - if($feature[name]=='Design'||$feature[name]=='Bauform') - { - $footprint = $feature["featureValues"]["value"]; + if (isset($product['classifications']['features'])) { + foreach ($product['classifications']['features'] as $feature) { + if (isset($feature['name']) && ($feature['name'] === 'Design' || $feature['name'] === 'Bauform')) { + $footprint = $feature['featureValues']['value'] ?? null; + break; + } } } @@ -217,27 +212,24 @@ class BuerklinProvider implements InfoProviderInterface */ private function pricesToVendorInfo(string $sku, string $url, array $prices): array { - $price_dtos = []; - - foreach ($prices as $price) { - $price_dtos[] = new PriceDTO( - minimum_discount_amount: $price['minQuantity'], - price: $price['value'], - currency_iso_code: $price['currencyIso'], - includes_tax: false, - ); - } + $priceDTOs = array_map(fn($price) => new PriceDTO( + minimum_discount_amount: $price['minQuantity'], + price: $price['value'], + currency_iso_code: $price['currencyIso'], + includes_tax: false + ), $prices); return [ new PurchaseInfoDTO( distributor_name: self::DISTRIBUTOR_NAME, order_number: $sku, - prices: $price_dtos, + prices: $priceDTOs, product_url: $url, ) ]; } + /** * Returns a valid Buerklin product short URL from product code * @param string $product_code @@ -265,62 +257,50 @@ class BuerklinProvider implements InfoProviderInterface private function attributesToParameters(?array $attributes): array { $result = []; - + foreach ($attributes as $attribute) { - - //Skip this attribute if it's empty - if (in_array(trim((string) $attribute['featureValues']['value']), ['', '-'], true)) { - continue; + if (!isset($attribute['name'], $attribute['featureValues']['value'])) { + continue; } - - $result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['name'], value: $attribute['featureValues']['value'], group: null); + + $value = trim((string)$attribute['featureValues']['value']); + if ($value === '' || $value === '-') { + continue; + } + + $result[] = ParameterDTO::parseValueIncludingUnit( + name: $attribute['name'], + value: $value, + group: null + ); } - + return $result; } + public function searchByKeyword(string $keyword): array { - $response = $this->client->request('GET', self::ENDPOINT_URL . "products/search/", [ - 'auth_bearer' => $this->getToken(), - 'query' => [ - 'curr' => $this->currency, - 'language' => $this->language, - 'pageSize' => '50', - 'currentPage' => '1', - 'query' => $term, - 'sort' => 'relevance' - ], - ]); - - $arr = $response->toArray(); - - // Get products list - $products = $arr['products'] ?? []; - - $result = []; - - foreach ($products as $product) { - $result[] = $this->getPartDetail($product); - } - - return $result; + $response = $this->makeAPICall('/products/search', [ + 'curr' => $this->currency, + 'language' => $this->language, + 'pageSize' => '50', + 'currentPage' => '1', + 'keyword' => $keyword, + 'sort' => 'relevance']); + + return array_map(fn($product) => $this->getPartDetail($product), $response['products'] ?? []); } public function getDetails(string $id): PartDetailDTO { - $tmp = $this->searchByKeyword($id); - - - if (count($tmp) === 0) { - throw new \RuntimeException('No part found with ID ' . $id); + $response = $this->makeAPICall("/products", ['sku' => $id]); + + if (empty($response['result'])) { + throw new \RuntimeException("No part found with ID $id"); } - - if (count($tmp) > 1) { - throw new \RuntimeException('Multiple parts found with ID ' . $id); - } - - return $tmp[0]; + + return $this->getPartDetail($response['result']); } public function getCapabilities(): array