feat: add price scraper with JSON-LD/meta/regex fallback
- Implement extractPrice() to extract price from HTML with three fallback methods: 1. JSON-LD Product schema (json-ld) 2. og:price:amount meta tag (meta) 3. Euro symbol regex pattern (regex) - Implement fetchAndExtractPrice() for fetching URLs and extracting prices - Includes helper functions for parsing and normalizing price values - Add comprehensive test suite with fixtures for each extraction method - All tests passing (14/14) Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
function roundMoney(n) {
|
||||
return Math.round(n * 100) / 100;
|
||||
}
|
||||
|
||||
function parsePriceValue(raw) {
|
||||
if (raw == null) return null;
|
||||
const cleaned = String(raw).trim().replace(',', '.');
|
||||
const value = parseFloat(cleaned);
|
||||
return Number.isFinite(value) ? roundMoney(value) : null;
|
||||
}
|
||||
|
||||
function searchJsonLdNode(node) {
|
||||
if (Array.isArray(node)) {
|
||||
for (const item of node) {
|
||||
const price = searchJsonLdNode(item);
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (node && typeof node === 'object') {
|
||||
const type = node['@type'];
|
||||
const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product'));
|
||||
if (isProduct && node.offers) {
|
||||
const offers = Array.isArray(node.offers) ? node.offers[0] : node.offers;
|
||||
const price = parsePriceValue(offers && offers.price);
|
||||
if (price != null) return price;
|
||||
}
|
||||
if (Array.isArray(node['@graph'])) {
|
||||
const price = searchJsonLdNode(node['@graph']);
|
||||
if (price != null) return price;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInJsonLd($) {
|
||||
const scripts = $('script[type="application/ld+json"]').toArray();
|
||||
for (const el of scripts) {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse($(el).contents().text());
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const price = searchJsonLdNode(data);
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInMeta($) {
|
||||
const selectors = ['meta[property="product:price:amount"]', 'meta[property="og:price:amount"]'];
|
||||
for (const selector of selectors) {
|
||||
const price = parsePriceValue($(selector).attr('content'));
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInText($) {
|
||||
const text = $('body').text();
|
||||
const match = text.match(/€\s?(\d+[.,]\d{2})/);
|
||||
return match ? parsePriceValue(match[1]) : null;
|
||||
}
|
||||
|
||||
function extractPrice(html) {
|
||||
const $ = cheerio.load(html);
|
||||
let price = findPriceInJsonLd($);
|
||||
if (price != null) return { price, method: 'json-ld' };
|
||||
price = findPriceInMeta($);
|
||||
if (price != null) return { price, method: 'meta' };
|
||||
price = findPriceInText($);
|
||||
if (price != null) return { price, method: 'regex' };
|
||||
return { price: null, method: null };
|
||||
}
|
||||
|
||||
async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url);
|
||||
} catch (err) {
|
||||
return { price: null, method: null, error: err };
|
||||
}
|
||||
if (!response.ok) {
|
||||
return { price: null, method: null, error: new Error(`HTTP ${response.status}`) };
|
||||
}
|
||||
const html = await response.text();
|
||||
const { price, method } = extractPrice(html);
|
||||
return { price, method, error: null };
|
||||
}
|
||||
|
||||
module.exports = { extractPrice, fetchAndExtractPrice, parsePriceValue };
|
||||
Reference in New Issue
Block a user