feat: add price scraper with JSON-LD/meta/regex fallback

- Implement extractPrice() to extract price from HTML with three fallback methods:
  1. JSON-LD Product schema (json-ld)
  2. og:price:amount meta tag (meta)
  3. Euro symbol regex pattern (regex)
- Implement fetchAndExtractPrice() for fetching URLs and extracting prices
- Includes helper functions for parsing and normalizing price values
- Add comprehensive test suite with fixtures for each extraction method
- All tests passing (14/14)

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
This commit is contained in:
Sjoerd de Vries
2026-09-05 15:06:27 +02:00
parent 492833dda4
commit 5e7d247a16
6 changed files with 154 additions and 0 deletions
+94
View File
@@ -0,0 +1,94 @@
const cheerio = require('cheerio');
function roundMoney(n) {
return Math.round(n * 100) / 100;
}
function parsePriceValue(raw) {
if (raw == null) return null;
const cleaned = String(raw).trim().replace(',', '.');
const value = parseFloat(cleaned);
return Number.isFinite(value) ? roundMoney(value) : null;
}
function searchJsonLdNode(node) {
if (Array.isArray(node)) {
for (const item of node) {
const price = searchJsonLdNode(item);
if (price != null) return price;
}
return null;
}
if (node && typeof node === 'object') {
const type = node['@type'];
const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product'));
if (isProduct && node.offers) {
const offers = Array.isArray(node.offers) ? node.offers[0] : node.offers;
const price = parsePriceValue(offers && offers.price);
if (price != null) return price;
}
if (Array.isArray(node['@graph'])) {
const price = searchJsonLdNode(node['@graph']);
if (price != null) return price;
}
}
return null;
}
function findPriceInJsonLd($) {
const scripts = $('script[type="application/ld+json"]').toArray();
for (const el of scripts) {
let data;
try {
data = JSON.parse($(el).contents().text());
} catch {
continue;
}
const price = searchJsonLdNode(data);
if (price != null) return price;
}
return null;
}
function findPriceInMeta($) {
const selectors = ['meta[property="product:price:amount"]', 'meta[property="og:price:amount"]'];
for (const selector of selectors) {
const price = parsePriceValue($(selector).attr('content'));
if (price != null) return price;
}
return null;
}
function findPriceInText($) {
const text = $('body').text();
const match = text.match(/€\s?(\d+[.,]\d{2})/);
return match ? parsePriceValue(match[1]) : null;
}
function extractPrice(html) {
const $ = cheerio.load(html);
let price = findPriceInJsonLd($);
if (price != null) return { price, method: 'json-ld' };
price = findPriceInMeta($);
if (price != null) return { price, method: 'meta' };
price = findPriceInText($);
if (price != null) return { price, method: 'regex' };
return { price: null, method: null };
}
async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) {
let response;
try {
response = await fetchImpl(url);
} catch (err) {
return { price: null, method: null, error: err };
}
if (!response.ok) {
return { price: null, method: null, error: new Error(`HTTP ${response.status}`) };
}
const html = await response.text();
const { price, method } = extractPrice(html);
return { price, method, error: null };
}
module.exports = { extractPrice, fetchAndExtractPrice, parsePriceValue };