From 47dbb5783887fd8568595db417a490b189b4b8a0 Mon Sep 17 00:00:00 2001 From: samy kamkar Date: Sat, 3 Oct 2026 14:16:16 -0700 Subject: [PATCH] added an info provider for pololu.com Pololu has no API, so like the Pollin and Reichelt providers this one reads the website: the products-only search for keywords and item numbers, and the product page for name, description, brand, status, category, all pictures, price breaks and stock. Specifications and file downloads (dimension diagrams, 3D models, datasheets) live on separate tabs, which are only requested when the product page links them, so a lookup needs at most three page requests. The Pololu item number is the provider id and order number. It is only used as MPN for Pololu's own products, as the shop also sells other brands. Pololu is a small shop, so the page requests are spaced by a configurable delay (PROVIDER_POLOLU_REQUEST_DELAY) and the provider stops sending requests for an hour once the site answers with 403, 429 or 503. --- docs/usage/information_provider_system.md | 17 + .../Providers/PololuProvider.php | 579 ++++++++++++++++++ .../InfoProviderSettings.php | 3 + .../InfoProviderSystem/PololuSettings.php | 51 ++ .../Providers/PololuProviderTest.php | 367 +++++++++++ translations/messages.en.xlf | 24 + 6 files changed, 1041 insertions(+) create mode 100644 src/Services/InfoProviderSystem/Providers/PololuProvider.php create mode 100644 src/Settings/InfoProviderSystem/PololuSettings.php create mode 100644 tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php diff --git a/docs/usage/information_provider_system.md b/docs/usage/information_provider_system.md index 5d734c64e..d81055e9d 100644 --- a/docs/usage/information_provider_system.md +++ b/docs/usage/information_provider_system.md @@ -295,6 +295,23 @@ This is not an official API and could break at any time. So use it at your own r The following env configuration options are available: * `PROVIDER_POLLIN_ENABLED`: Set this to `1` to enable the Pollin provider +### Pololu + +The Pololu provider uses webscraping from [pololu.com](https://www.pololu.com/) to get part information. +This is not an official API and could break at any time. So use it at your own risk. + +You can search by keyword or by the Pololu item number (e.g. `2130`). Besides the basic infos, the provider returns +all product pictures, the price breaks (in USD), the available stock, the files from the "Resources" tab (datasheets, +dimension diagrams, 3D models, drill guides, ...) and the specifications from the "Specs" tab as parameters. + +Getting the details of a product needs up to three page requests (product page, specs and resources). To be polite to +the shop, the provider waits a configurable time between two page requests, and stops sending requests for some time +when the website answers with an error indicating that requests are blocked or rate limited (HTTP 403, 429 or 503). + +The following env configuration options are available: +* `PROVIDER_POLOLU_ENABLED`: Set this to `1` to enable the Pololu provider +* `PROVIDER_POLOLU_REQUEST_DELAY`: The minimum time in seconds between two page requests to pololu.com (optional, default: `1`) + ### Buerklin The Buerklin provider uses the [Buerklin API](https://www.buerklin.com/en/services/eprocurement/) to search for parts and get information. diff --git a/src/Services/InfoProviderSystem/Providers/PololuProvider.php b/src/Services/InfoProviderSystem/Providers/PololuProvider.php new file mode 100644 index 000000000..7d15cbd73 --- /dev/null +++ b/src/Services/InfoProviderSystem/Providers/PololuProvider.php @@ -0,0 +1,579 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem\Providers; + +use App\Entity\Parts\ManufacturingStatus; +use App\Services\InfoProviderSystem\DTOs\FileDTO; +use App\Services\InfoProviderSystem\DTOs\ParameterDTO; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\PriceDTO; +use App\Services\InfoProviderSystem\DTOs\ProviderInfoDTO; +use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Settings\InfoProviderSystem\PololuSettings; +use Psr\Cache\CacheItemPoolInterface; +use Symfony\Component\DomCrawler\Crawler; +use Symfony\Contracts\HttpClient\HttpClientInterface; +use Symfony\Contracts\HttpClient\ResponseInterface; + +/** + * Retrieves part information from pololu.com by scraping the product pages. + * + * The provider id is the Pololu item number, which is also part of the product URL (/product/). + * The specifications and the downloadable files are on separate tabs of a product (/product//specs and + * /product//resources), which are only requested if the product page says that they are not empty. + */ +class PololuProvider implements InfoProviderInterface, URLHandlerInfoProviderInterface +{ + public const PROVIDER_KEY = 'pololu'; + + public const DISTRIBUTOR_NAME = 'Pololu'; + + private const BASE_URL = 'https://www.pololu.com'; + + /** @var string All prices on pololu.com are in US dollars */ + private const CURRENCY = 'USD'; + + /** @var int[] The HTTP status codes which indicate that the website does not want our requests (anymore) */ + private const BLOCKED_STATUS_CODES = [403, 429, 503]; + + /** @var int How long (in seconds) no requests are sent anymore, after the website has blocked a request */ + private const BLOCKED_PAUSE = 3600; + + private const CACHE_KEY_BLOCKED = 'pololu_blocked'; + private const CACHE_KEY_LAST_REQUEST = 'pololu_last_request'; + + /** @var string Links outside the file downloads list are only used, if they point to a file with one of these extensions */ + private const FILE_EXTENSIONS_REGEX = '/\.(pdf|zip|step|stp|igs|iges|stl|dxf|dwg|sldprt|easm|eprt|brd|sch|hex|bin)$/i'; + + public function __construct(private readonly HttpClientInterface $client, + private readonly PololuSettings $settings, + private readonly CacheItemPoolInterface $partInfoCache, + ) + { + } + + public function getProviderInfo(): ProviderInfoDTO + { + return new ProviderInfoDTO( + key: self::PROVIDER_KEY, + name: 'Pololu', + description: 'Webscraping from pololu.com to get part information', + url: 'https://www.pololu.com/', + disabledHelp: 'Enable the provider in provider settings', + settingsClass: PololuSettings::class, + capabilities: [ + ProviderCapabilities::BASIC, + ProviderCapabilities::PICTURE, + ProviderCapabilities::PRICE, + ProviderCapabilities::DATASHEET, + ProviderCapabilities::PARAMETERS, + ProviderCapabilities::STOCK_LEVEL, + ], + ); + } + + public function isActive(): bool + { + return $this->settings->enabled; + } + + /** + * Requests the given page of pololu.com. The requests are paced, and if the website blocks a request, no + * further requests are sent for some time. + * @param string $path The path of the page, starting with a slash + * @param array $query + */ + private function request(string $path, array $query = []): ResponseInterface + { + if ($this->partInfoCache->getItem(self::CACHE_KEY_BLOCKED)->isHit()) { + throw new \RuntimeException('pololu.com has blocked a previous request. No requests are sent to it for some time, try again later.'); + } + + $this->waitForNextRequest(); + + $response = $this->client->request('GET', self::BASE_URL . $path, [ + 'query' => $query, + ]); + + if (in_array($response->getStatusCode(), self::BLOCKED_STATUS_CODES, true)) { + $item = $this->partInfoCache->getItem(self::CACHE_KEY_BLOCKED); + $item->set(time()); + $item->expiresAfter(self::BLOCKED_PAUSE); + $this->partInfoCache->save($item); + + throw new \RuntimeException(sprintf('pololu.com has blocked the request (HTTP status %d). No requests are sent to it for some time, try again later.', $response->getStatusCode())); + } + + return $response; + } + + /** + * Sleeps until the configured delay since the last request has passed. The time of the last request is stored + * in the cache, so the delay is also respected between different lookups. + */ + private function waitForNextRequest(): void + { + $delay = $this->settings->requestDelay; + $item = $this->partInfoCache->getItem(self::CACHE_KEY_LAST_REQUEST); + + if ($delay > 0 && $item->isHit()) { + $wait = (float) $item->get() + $delay - microtime(true); + if ($wait > 0) { + usleep((int) (min($wait, $delay) * 1_000_000)); + } + } + + $item->set(microtime(true)); + $item->expiresAfter(60); + $this->partInfoCache->save($item); + } + + public function searchByKeyword(string $keyword, array $options = []): array + { + $response = $this->request('/search', [ + 'query' => $keyword, + '[search_type]' => 'products', + ]); + + $content = $response->getContent(); + + //If we were redirected to a product page, then just return the single item + if (preg_match('#/product/(\d+)#', (string) $response->getInfo('url'), $matches)) { + return [$this->parseProductPage($content, $matches[1])]; + } + + $dom = new Crawler($content); + + $results = []; + + //Each result is a dt (picture and title) followed by a dd (text excerpt) + $dom->filter('dl#search > dt')->each(function (Crawler $node) use (&$results) { + $link = $node->filter('a[href*="/product/"]')->last(); + if ($link->count() === 0 || !preg_match('#/product/(\d+)#', $link->attr('href') ?? '', $matches)) { + return; + } + $id = $matches[1]; + + $image = $node->filter('img'); + $excerpt = $node->nextAll()->first(); + + $results[] = new SearchResultDTO( + provider_key: self::PROVIDER_KEY, + provider_id: $id, + //The title has the format "Pololu item 2130: DRV8833 Dual Motor Driver Carrier" + name: preg_replace('/^Pololu item\s*#?\s*\d+:\s*/', '', $link->text()), + description: $excerpt->count() > 0 && $excerpt->nodeName() === 'dd' ? $excerpt->text() : '', + preview_image_url: $image->count() > 0 ? $image->attr('src') : null, + provider_url: $this->getProductURL($id), + ); + }); + + return $results; + } + + public function getDetails(string $id, array $options = []): PartDetailDTO + { + //Ensure that $id is an item number + if (!ctype_digit($id)) { + throw new \InvalidArgumentException("The id must be a Pololu item number!"); + } + + $content = $this->request('/product/' . $id)->getContent(); + $dom = new Crawler($content); + + //The specifications and the files are on their own pages. The tabs of an empty page are not linked. + $specs = null; + if ($dom->filter('table.tabs a[href$="/specs"]')->count() > 0) { + $specs = $this->request('/product/' . $id . '/specs')->getContent(); + } + + $resources = null; + if ($dom->filter('table.tabs a[href$="/resources"]')->count() > 0) { + $resources = $this->request('/product/' . $id . '/resources')->getContent(); + } + + return $this->parseProductPage($content, $id, $specs, $resources); + } + + private function getProductURL(string $id): string + { + return self::BASE_URL . '/product/' . $id; + } + + /** + * @param string $content The HTML of the product page + * @param string $id The Pololu item number + * @param string|null $specs The HTML of the specs tab of the product, if available + * @param string|null $resources The HTML of the resources tab of the product, if available + */ + private function parseProductPage(string $content, string $id, ?string $specs = null, ?string $resources = null): PartDetailDTO + { + $dom = new Crawler($content); + + $productPageUrl = $this->getProductURL($id); + + $stock = $dom->filter('[data-available-stock]'); + + $purchaseInfo = new PurchaseInfoDTO( + distributor_name: self::DISTRIBUTOR_NAME, + order_number: $id, + prices: $this->parsePrices($dom), + product_url: $productPageUrl, + available_amount: $stock->count() > 0 && is_numeric($stock->attr('data-available-stock')) ? (float) $stock->attr('data-available-stock') : null, + ); + + $brand = $this->parseOrderFormValue($dom, 'Brand'); + //Pololu sells also products of other brands, for which the item number is no manufacturer part number + $isPololuProduct = $brand === null || strcasecmp($brand, 'Pololu') === 0; + + $images = $this->parseImages($dom); + $parameters = $specs !== null ? $this->parseParameters(new Crawler($specs)) : []; + + return new PartDetailDTO( + provider_key: self::PROVIDER_KEY, + provider_id: $id, + name: $dom->filter('h1#page_title')->text(), + description: $dom->filter('div#short_description')->text(''), + category: $this->parseCategory($dom), + manufacturer: $brand ?? 'Pololu', + mpn: $isPololuProduct ? $id : null, + preview_image_url: $images !== [] ? $images[0]->url : null, + manufacturing_status: $this->mapStatus($this->parseOrderFormValue($dom, 'Status')), + provider_url: $productPageUrl, + notes: $this->parseNotes($dom), + datasheets: $resources !== null ? $this->parseFiles(new Crawler($resources)) : [], + images: $images, + parameters: $parameters, + vendor_infos: [$purchaseInfo], + mass: $this->parseMass($parameters), + manufacturer_product_url: $isPololuProduct ? $productPageUrl : null, + ); + } + + /** + * Returns the value of a labeled field (like "Brand:" or "Status:") of the order form. + */ + private function parseOrderFormValue(Crawler $dom, string $label): ?string + { + $value = null; + + $dom->filter('span.label')->each(function (Crawler $node) use (&$value, $label) { + if ($value !== null || rtrim($this->cleanText($node->text()), ':') !== $label) { + return; + } + + $valueNode = $node->nextAll()->filter('span.value')->first(); + if ($valueNode->count() > 0) { + $value = $this->cleanText($valueNode->text()); + } + }); + + return $value === '' ? null : $value; + } + + /** + * Normalizes the whitespaces (including non-breaking spaces) of the given text + */ + private function cleanText(string $text): string + { + return trim(preg_replace('/[\s\x{00A0}]+/u', ' ', $text) ?? $text); + } + + private function mapStatus(?string $status): ManufacturingStatus + { + if ($status === null) { + return ManufacturingStatus::NOT_SET; + } + + $status = strtolower($status); + + return match (true) { + str_contains($status, 'discontinued'), str_contains($status, 'obsolete') => ManufacturingStatus::DISCONTINUED, + str_contains($status, 'end of life'), str_contains($status, 'last time buy') => ManufacturingStatus::EOL, + str_contains($status, 'not recommended') => ManufacturingStatus::NRFND, + str_contains($status, 'active') => ManufacturingStatus::ACTIVE, + default => ManufacturingStatus::NOT_SET, + }; + } + + private function parseCategory(Crawler $dom): ?string + { + $categories = $dom->filter('#breadcrumbs a')->each(fn(Crawler $node) => $this->cleanText($node->text())); + + return $categories === [] ? null : implode(' -> ', $categories); + } + + /** + * @return PriceDTO[] + */ + private function parsePrices(Crawler $dom): array + { + $prices = []; + + //Each row of the table consists of the minimum order amount and the unit price + $dom->filter('table.pricing tr')->each(function (Crawler $row) use (&$prices) { + $cells = $row->filter('td'); + if ($cells->count() < 2) { + return; + } + + $amount = str_replace(',', '', $this->cleanText($cells->eq(0)->text())); + //If the product is on sale, the cell contains the regular price followed by the reduced one + if (!is_numeric($amount) || !preg_match_all('/\d+(?:\.\d+)?/', str_replace(',', '', $cells->eq(1)->text()), $matches)) { + return; + } + + $prices[] = new PriceDTO( + minimum_discount_amount: (float) $amount, + price: end($matches[0]), + currency_iso_code: self::CURRENCY, + includes_tax: false, + ); + }); + + return $prices; + } + + /** + * @return FileDTO[] + */ + private function parseImages(Crawler $dom): array + { + $images = []; + + //The main picture carries the list of all pictures of the product for the gallery + $main = $dom->filter('#main_picture_img'); + if ($main->count() > 0) { + $gallery = json_decode($main->attr('data-gallery-pictures') ?? '', true); + foreach (is_array($gallery) ? $gallery : [] as $picture) { + $url = $picture['url_full'] ?? $picture['url_medium'] ?? null; + if (!is_string($url) || isset($images[$url])) { + continue; + } + + $caption = $this->cleanText(html_entity_decode(strip_tags((string) ($picture['caption'] ?? '')), ENT_QUOTES | ENT_HTML5)); + $images[$url] = new FileDTO($url, $caption !== '' ? $caption : null); + } + } + + if ($images === []) { + $ogImage = $dom->filter('meta[property="og:image"]'); + if ($ogImage->count() > 0 && $ogImage->attr('content')) { + return [new FileDTO($ogImage->attr('content'))]; + } + } + + return array_values($images); + } + + private function parseNotes(Crawler $dom): string + { + $tabPage = $dom->filter('div.tab_page'); + if ($tabPage->count() === 0) { + return ''; + } + + /** @var \DOMElement $root */ + $root = $tabPage->getNode(0)->cloneNode(true); + + //Remove the pictures (they are returned as images), scripts and embedded videos + $remove = []; + foreach (['table', 'script', 'style', 'iframe', 'img'] as $tag) { + foreach ($root->getElementsByTagName($tag) as $element) { + if ($tag !== 'table' || str_contains($element->getAttribute('class'), 'picture_with_caption')) { + $remove[] = $element; + } + } + } + foreach ($remove as $element) { + $element->parentNode?->removeChild($element); + } + + //Make the links absolute, so they work outside of pololu.com + foreach ($root->getElementsByTagName('a') as $link) { + $link->removeAttribute('onclick'); + if ($link->hasAttribute('href')) { + $link->setAttribute('href', $this->absoluteURL($link->getAttribute('href'))); + } + } + + $html = ''; + foreach ($root->childNodes as $child) { + $html .= $root->ownerDocument->saveHTML($child); + } + + //The removed pictures leave empty paragraphs behind + return trim(preg_replace('#

\s*

#', '', $html) ?? $html); + } + + private function absoluteURL(string $url): string + { + if (str_starts_with($url, '//')) { + return 'https:' . $url; + } + if (str_starts_with($url, '/')) { + return self::BASE_URL . $url; + } + + return $url; + } + + /** + * Parses the specs tab of a product: Each table of specifications is preceded by a heading, which is used as group. + * @return ParameterDTO[] + */ + private function parseParameters(Crawler $dom): array + { + $parameters = []; + $group = null; + + $dom->filter('div.tab_page')->children()->each(function (Crawler $node) use (&$parameters, &$group) { + if ($node->nodeName() === 'h2') { + $group = $this->cleanText($node->text()); + return; + } + + if ($node->nodeName() !== 'table' || !str_contains($node->attr('class') ?? '', 'specifications')) { + return; + } + + $node->filter('tr')->each(function (Crawler $row) use (&$parameters, $group) { + $header = $row->filter('th'); + $cell = $row->filter('td'); + if ($header->count() === 0 || $cell->count() === 0) { + return; + } + + $name = rtrim($this->cleanText($header->text()), ':'); + $value = $this->textWithoutFootnotes($cell); + if ($name === '' || $value === '') { + return; + } + + //Yes/no specifications are named like a question ("Reverse voltage protection?") and answered with Y or N + if (str_ends_with($name, '?')) { + $value = match ($value) { + 'Y' => 'Yes', + 'N' => 'No', + default => $value, + }; + } + + //Only split values like "2.7 V" into number and unit, but not markings like "0J3762" + if (preg_match('/^-?\d+(?:\.\d+)?\s*[^\d\s]*$/u', $value)) { + $parameters[] = ParameterDTO::parseValueIncludingUnit(name: $name, value: $value, group: $group); + } else { + $parameters[] = new ParameterDTO(name: $name, value_text: $value, group: $group); + } + }); + }); + + return $parameters; + } + + /** + * Returns the text of the given node without the footnote references (like the "1" in "1.0 g¹") + */ + private function textWithoutFootnotes(Crawler $node): string + { + /** @var \DOMElement $element */ + $element = $node->getNode(0)->cloneNode(true); + + foreach (iterator_to_array($element->getElementsByTagName('sup')) as $footnote) { + $footnote->parentNode?->removeChild($footnote); + } + + return $this->cleanText($element->textContent); + } + + /** + * Returns the weight in grams, if the specifications contain it + * @param ParameterDTO[] $parameters + */ + private function parseMass(array $parameters): ?float + { + foreach ($parameters as $parameter) { + if (strcasecmp($parameter->name, 'Weight') !== 0 || $parameter->value_typ === null) { + continue; + } + + return match ($parameter->unit) { + 'g' => $parameter->value_typ, + 'kg' => $parameter->value_typ * 1000, + 'mg' => $parameter->value_typ / 1000, + 'oz' => round($parameter->value_typ * 28.3495, 2), + 'lb', 'lbs' => round($parameter->value_typ * 453.592, 2), + default => null, + }; + } + + return null; + } + + /** + * Parses the resources tab of a product. All file downloads (datasheets, dimension diagrams, 3D models, ...) are + * returned, of the other resources (recommended links, guides, ...) only the ones which link directly to a file. + * @return FileDTO[] + */ + private function parseFiles(Crawler $dom): array + { + $files = []; + + $dom->filter('ul.resources')->each(function (Crawler $list) use (&$files) { + $isFileList = str_contains($list->attr('class') ?? '', 'attached_files'); + + $list->filter('li.resource .content_title a[href]')->each(function (Crawler $link) use (&$files, $isFileList) { + $url = $this->absoluteURL($link->attr('href') ?? ''); + if (!str_starts_with($url, 'http') || isset($files[$url])) { + return; + } + + if (!$isFileList && !preg_match(self::FILE_EXTENSIONS_REGEX, (string) parse_url($url, PHP_URL_PATH))) { + return; + } + + $files[$url] = new FileDTO($url, $this->cleanText($link->text())); + }); + }); + + return array_values($files); + } + + public function getHandledDomains(): array + { + return ['pololu.com']; + } + + public function getIDFromURL(string $url): ?string + { + //URL like: https://www.pololu.com/product/2130 or https://www.pololu.com/product/2130/resources + $matches = []; + if (preg_match('#/product/(\d+)(?:[/?\#]|$)#', $url, $matches)) { + return $matches[1]; + } + + return null; + } +} diff --git a/src/Settings/InfoProviderSystem/InfoProviderSettings.php b/src/Settings/InfoProviderSystem/InfoProviderSettings.php index 372443c6a..7e354d2cd 100644 --- a/src/Settings/InfoProviderSystem/InfoProviderSettings.php +++ b/src/Settings/InfoProviderSystem/InfoProviderSettings.php @@ -72,6 +72,9 @@ class InfoProviderSettings #[EmbeddedSettings] public ?PollinSettings $pollin = null; + #[EmbeddedSettings] + public ?PololuSettings $pololu = null; + #[EmbeddedSettings] public ?BuerklinSettings $buerklin = null; diff --git a/src/Settings/InfoProviderSystem/PololuSettings.php b/src/Settings/InfoProviderSystem/PololuSettings.php new file mode 100644 index 000000000..a19a5a30a --- /dev/null +++ b/src/Settings/InfoProviderSystem/PololuSettings.php @@ -0,0 +1,51 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Settings\InfoProviderSystem; + +use App\Settings\SettingsIcon; +use Jbtronics\SettingsBundle\Metadata\EnvVarMode; +use Jbtronics\SettingsBundle\Settings\Settings; +use Jbtronics\SettingsBundle\Settings\SettingsParameter; +use Jbtronics\SettingsBundle\Settings\SettingsTrait; +use Symfony\Component\Form\Extension\Core\Type\NumberType; +use Symfony\Component\Translation\TranslatableMessage as TM; +use Symfony\Component\Validator\Constraints as Assert; + +#[Settings(label: new TM("settings.ips.pololu"), description: new TM("settings.ips.pololu.help"))] +#[SettingsIcon("fa-plug")] +class PololuSettings +{ + use SettingsTrait; + + #[SettingsParameter(label: new TM("settings.ips.lcsc.enabled"), + envVar: "bool:PROVIDER_POLOLU_ENABLED", envVarMode: EnvVarMode::OVERWRITE)] + public bool $enabled = false; + + /** @var int The minimum time in seconds between two page requests to pololu.com */ + #[SettingsParameter(label: new TM("settings.ips.pololu.requestDelay"), description: new TM("settings.ips.pololu.requestDelay.help"), + formType: NumberType::class, formOptions: ["attr" => ["min" => 0, "max" => 30]], + envVar: "int:PROVIDER_POLOLU_REQUEST_DELAY", envVarMode: EnvVarMode::OVERWRITE)] + #[Assert\Range(min: 0, max: 30)] + public int $requestDelay = 1; +} diff --git a/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php b/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php new file mode 100644 index 000000000..151a451a6 --- /dev/null +++ b/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php @@ -0,0 +1,367 @@ +. + */ + +declare(strict_types=1); + +namespace App\Tests\Services\InfoProviderSystem\Providers; + +use App\Entity\Parts\ManufacturingStatus; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Services\InfoProviderSystem\Providers\PololuProvider; +use App\Services\InfoProviderSystem\Providers\ProviderCapabilities; +use App\Settings\InfoProviderSystem\PololuSettings; +use App\Tests\SettingsTestHelper; +use PHPUnit\Framework\TestCase; +use Symfony\Component\Cache\Adapter\ArrayAdapter; +use Symfony\Component\HttpClient\MockHttpClient; +use Symfony\Component\HttpClient\Response\MockResponse; + +final class PololuProviderTest extends TestCase +{ + private PololuSettings $settings; + private PololuProvider $provider; + private MockHttpClient $httpClient; + + private const SEARCH_PAGE = <<<'HTML' +
+ +
+ HTML; + + private const PRODUCT_PAGE = <<<'HTML' + + +
+ +

DRV8833 Dual Motor Driver Carrier

+
+ + +
+ + + + + +
Pololu item #: 2130
Brand: Pololu
Status: Active and Preferred 
+ + + + + +
Price breakUnit price (US$)
+ 1 + + 10.95 +
510.07
1008.53
+
+
+

This tiny breakout board for TI’s DRV8833 dual motor driver can deliver 1.2 A per channel.

+
+ %TABS%
Description
+

Overview

+

Bottom view with dimensions.

+

We also carry a DRV8835 dual motor driver carrier.

+
+
+ HTML; + + private const TABS = <<<'HTML' + Specs (5) + Resources (4) + HTML; + + private const EMPTY_TABS = <<<'HTML' + Specs (0) + Resources (0) + HTML; + + private const SPECS_PAGE = <<<'HTML' +

Dimensions

+ + + +
+ Size: + + 0.5″ × 0.8″1 +
Weight:1.0 g1
+

General specifications

+ + + + + +
Motor driver:DRV8833
Minimum operating voltage:2.7 V
Reverse voltage protection?:Y
Other PCB markings:0J3762
+

Notes:

1
Without included hardware.
+
+ HTML; + + private const RESOURCES_PAGE = <<<'HTML' +

File downloads

+ +

Recommended links

+ +
+ HTML; + + protected function setUp(): void + { + $this->httpClient = new MockHttpClient(); + $this->settings = SettingsTestHelper::createSettingsDummy(PololuSettings::class); + $this->settings->enabled = true; + //Do not slow down the tests + $this->settings->requestDelay = 0; + $this->provider = new PololuProvider($this->httpClient, $this->settings, new ArrayAdapter()); + } + + private function productPage(string $tabs = self::TABS): string + { + return str_replace('%TABS%', $tabs, self::PRODUCT_PAGE); + } + + public function testGetProviderInfo(): void + { + $info = $this->provider->getProviderInfo(); + + $this->assertSame('pololu', $info->key); + $this->assertSame('Pololu', $info->name); + $this->assertSame(PololuSettings::class, $info->settingsClass); + $this->assertContains(ProviderCapabilities::BASIC, $info->capabilities); + $this->assertContains(ProviderCapabilities::PICTURE, $info->capabilities); + $this->assertContains(ProviderCapabilities::DATASHEET, $info->capabilities); + $this->assertContains(ProviderCapabilities::PRICE, $info->capabilities); + $this->assertContains(ProviderCapabilities::PARAMETERS, $info->capabilities); + } + + public function testIsActive(): void + { + $this->assertTrue($this->provider->isActive()); + + $this->settings->enabled = false; + $this->assertFalse($this->provider->isActive()); + } + + public function testSearchByKeyword(): void + { + $requestedUrl = null; + $this->httpClient->setResponseFactory(function (string $method, string $url) use (&$requestedUrl) { + $requestedUrl = $url; + return new MockResponse(self::SEARCH_PAGE); + }); + + $results = $this->provider->searchByKeyword('DRV8833'); + + $this->assertStringStartsWith('https://www.pololu.com/search?', $requestedUrl); + $this->assertStringContainsString('query=DRV8833', $requestedUrl); + $this->assertStringContainsString('search_type]=products', $requestedUrl); + + //The category result must be skipped + $this->assertCount(2, $results); + $this->assertContainsOnlyInstancesOf(SearchResultDTO::class, $results); + + $this->assertSame('pololu', $results[0]->provider_key); + $this->assertSame('2130', $results[0]->provider_id); + $this->assertSame('DRV8833 Dual Motor Driver Carrier', $results[0]->name); + $this->assertSame('https://www.pololu.com/product/2130', $results[0]->provider_url); + $this->assertSame('https://a.pololu-files.com/picture/0J3864.100.jpg?abc', $results[0]->preview_image_url); + $this->assertStringContainsString('carrier for the DRV8833', $results[0]->description); + + $this->assertSame('2196', $results[1]->provider_id); + $this->assertSame('Tamiya 70203 Low-Current Motor Gearbox (3-Speed)', $results[1]->name); + } + + public function testSearchByKeywordWithoutResults(): void + { + $this->httpClient->setResponseFactory([new MockResponse('')]); + + $this->assertSame([], $this->provider->searchByKeyword('doesnotexist')); + } + + public function testGetDetails(): void + { + $requestedUrls = []; + $responses = [$this->productPage(), self::SPECS_PAGE, self::RESOURCES_PAGE]; + $this->httpClient->setResponseFactory(function (string $method, string $url) use (&$requestedUrls, &$responses) { + $requestedUrls[] = $url; + return new MockResponse(array_shift($responses)); + }); + + $details = $this->provider->getDetails('2130'); + + $this->assertSame([ + 'https://www.pololu.com/product/2130', + 'https://www.pololu.com/product/2130/specs', + 'https://www.pololu.com/product/2130/resources', + ], $requestedUrls); + + $this->assertInstanceOf(PartDetailDTO::class, $details); + $this->assertSame('pololu', $details->provider_key); + $this->assertSame('2130', $details->provider_id); + $this->assertSame('DRV8833 Dual Motor Driver Carrier', $details->name); + $this->assertStringStartsWith('This tiny breakout board', $details->description); + $this->assertSame('Electronics -> Brushed DC Motor Drivers', $details->category); + $this->assertSame('Pololu', $details->manufacturer); + $this->assertSame('2130', $details->mpn); + $this->assertSame(ManufacturingStatus::ACTIVE, $details->manufacturing_status); + $this->assertSame('https://www.pololu.com/product/2130', $details->provider_url); + $this->assertSame('https://www.pololu.com/product/2130', $details->manufacturer_product_url); + $this->assertSame(1.0, $details->mass); + + //Images + $this->assertSame('https://a.pololu-files.com/picture/0J3864.1200.jpg?abc', $details->preview_image_url); + $this->assertCount(2, $details->images); + $this->assertSame('https://a.pololu-files.com/picture/0J3864.1200.jpg?abc', $details->images[0]->url); + $this->assertSame('DRV8833 dual motor driver carrier.', $details->images[0]->name); + $this->assertSame('https://a.pololu-files.com/picture/0J3872.1200.png?def', $details->images[1]->url); + + //Notes: pictures are removed and links are made absolute + $this->assertStringContainsString('

Overview

', $details->notes); + $this->assertStringContainsString('href="https://www.pololu.com/product/2135"', $details->notes); + $this->assertStringNotContainsString('picture_with_caption', $details->notes); + $this->assertStringNotContainsString('onclick', $details->notes); + + //Prices and stock + $this->assertCount(1, $details->vendor_infos); + $purchaseInfo = $details->vendor_infos[0]; + $this->assertSame('Pololu', $purchaseInfo->distributor_name); + $this->assertSame('2130', $purchaseInfo->order_number); + $this->assertSame('https://www.pololu.com/product/2130', $purchaseInfo->product_url); + $this->assertSame(148.0, $purchaseInfo->available_amount); + $this->assertFalse($purchaseInfo->prices_include_vat); + $this->assertCount(3, $purchaseInfo->prices); + $this->assertSame(1.0, $purchaseInfo->prices[0]->minimum_discount_amount); + $this->assertSame('10.95', $purchaseInfo->prices[0]->price); + $this->assertSame('USD', $purchaseInfo->prices[0]->currency_iso_code); + $this->assertSame(100.0, $purchaseInfo->prices[2]->minimum_discount_amount); + $this->assertSame('8.53', $purchaseInfo->prices[2]->price); + + //Files: all file downloads, but only the recommended links which point to a file + $this->assertCount(3, $details->datasheets); + $this->assertSame('https://www.pololu.com/file/0J1615/drv8833-dual-motor-driver-carrier-dimensions.pdf', $details->datasheets[0]->url); + $this->assertSame('Dimension diagram of the DRV8833 Dual Motor Driver Carrier', $details->datasheets[0]->name); + $this->assertSame('https://www.pololu.com/file/0J1616/drv8833-dual-motor-driver-carrier-step.step', $details->datasheets[1]->url); + $this->assertSame('https://www.ti.com/lit/ds/symlink/drv8833.pdf', $details->datasheets[2]->url); + + //Parameters + $this->assertCount(6, $details->parameters); + $parameters = []; + foreach ($details->parameters as $parameter) { + $parameters[$parameter->name] = $parameter; + } + + $this->assertSame('0.5″ × 0.8″', $parameters['Size']->value_text); + $this->assertSame('Dimensions', $parameters['Size']->group); + $this->assertSame(1.0, $parameters['Weight']->value_typ); + $this->assertSame('g', $parameters['Weight']->unit); + $this->assertSame('DRV8833', $parameters['Motor driver']->value_text); + $this->assertSame('General specifications', $parameters['Motor driver']->group); + $this->assertSame(2.7, $parameters['Minimum operating voltage']->value_typ); + $this->assertSame('V', $parameters['Minimum operating voltage']->unit); + $this->assertSame('Yes', $parameters['Reverse voltage protection?']->value_text); + //A marking must not be split into a number and a unit + $this->assertSame('0J3762', $parameters['Other PCB markings']->value_text); + $this->assertNull($parameters['Other PCB markings']->value_typ); + } + + public function testGetDetailsSkipsEmptyTabs(): void + { + $requests = 0; + $this->httpClient->setResponseFactory(function () use (&$requests) { + $requests++; + return new MockResponse($this->productPage(self::EMPTY_TABS)); + }); + + $details = $this->provider->getDetails('2130'); + + $this->assertSame(1, $requests); + $this->assertSame([], $details->parameters); + $this->assertSame([], $details->datasheets); + $this->assertNull($details->mass); + } + + public function testGetDetailsOfOtherBrand(): void + { + $page = str_replace('Pololu', 'Tamiya', $this->productPage(self::EMPTY_TABS)); + $this->httpClient->setResponseFactory([new MockResponse($page)]); + + $details = $this->provider->getDetails('2196'); + + $this->assertSame('Tamiya', $details->manufacturer); + //The Pololu item number is no manufacturer part number for products of other brands + $this->assertNull($details->mpn); + $this->assertNull($details->manufacturer_product_url); + $this->assertSame('2196', $details->vendor_infos[0]->order_number); + } + + public function testGetDetailsWithInvalidId(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->provider->getDetails('abc'); + } + + public function testNoFurtherRequestsAfterBeingBlocked(): void + { + $requests = 0; + $this->httpClient->setResponseFactory(function () use (&$requests) { + $requests++; + return new MockResponse('Too many requests', ['http_code' => 429]); + }); + + try { + $this->provider->searchByKeyword('DRV8833'); + $this->fail('An exception should have been thrown'); + } catch (\RuntimeException $exception) { + $this->assertStringContainsString('429', $exception->getMessage()); + } + + try { + $this->provider->getDetails('2130'); + $this->fail('An exception should have been thrown'); + } catch (\RuntimeException) { + //The second lookup must not have sent a request + $this->assertSame(1, $requests); + } + } + + public function testGetHandledDomains(): void + { + $this->assertSame(['pololu.com'], $this->provider->getHandledDomains()); + } + + public function testGetIDFromURL(): void + { + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130')); + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130/resources')); + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130?foo=bar')); + $this->assertNull($this->provider->getIDFromURL('https://www.pololu.com/category/11/brushed-dc-motor-drivers')); + } +} diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index c0a90ff5c..da886a07c 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -15518,5 +15518,29 @@ Buerklin-API Authentication server: Unknown + + + settings.ips.pololu + Pololu + + + + + settings.ips.pololu.help + Pololu.com offers no official API, so this info provider webscrapes the website to extract info. It could break at any time, use it at your own risk. + + + + + settings.ips.pololu.requestDelay + Delay between page requests + + + + + settings.ips.pololu.requestDelay.help + The minimum time in seconds between two page requests to pololu.com. Getting the details of a product needs up to three page requests. + +