feat: add price scraper with JSON-LD/meta/regex fallback
- Implement extractPrice() to extract price from HTML with three fallback methods: 1. JSON-LD Product schema (json-ld) 2. og:price:amount meta tag (meta) 3. Euro symbol regex pattern (regex) - Implement fetchAndExtractPrice() for fetching URLs and extracting prices - Includes helper functions for parsing and normalizing price values - Add comprehensive test suite with fixtures for each extraction method - All tests passing (14/14) Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
This commit is contained in:
Sjoerd de Vries
@@ -0,0 +1,94 @@
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
function roundMoney(n) {
|
||||
return Math.round(n * 100) / 100;
|
||||
}
|
||||
|
||||
function parsePriceValue(raw) {
|
||||
if (raw == null) return null;
|
||||
const cleaned = String(raw).trim().replace(',', '.');
|
||||
const value = parseFloat(cleaned);
|
||||
return Number.isFinite(value) ? roundMoney(value) : null;
|
||||
}
|
||||
|
||||
function searchJsonLdNode(node) {
|
||||
if (Array.isArray(node)) {
|
||||
for (const item of node) {
|
||||
const price = searchJsonLdNode(item);
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (node && typeof node === 'object') {
|
||||
const type = node['@type'];
|
||||
const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product'));
|
||||
if (isProduct && node.offers) {
|
||||
const offers = Array.isArray(node.offers) ? node.offers[0] : node.offers;
|
||||
const price = parsePriceValue(offers && offers.price);
|
||||
if (price != null) return price;
|
||||
}
|
||||
if (Array.isArray(node['@graph'])) {
|
||||
const price = searchJsonLdNode(node['@graph']);
|
||||
if (price != null) return price;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInJsonLd($) {
|
||||
const scripts = $('script[type="application/ld+json"]').toArray();
|
||||
for (const el of scripts) {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse($(el).contents().text());
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const price = searchJsonLdNode(data);
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInMeta($) {
|
||||
const selectors = ['meta[property="product:price:amount"]', 'meta[property="og:price:amount"]'];
|
||||
for (const selector of selectors) {
|
||||
const price = parsePriceValue($(selector).attr('content'));
|
||||
if (price != null) return price;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findPriceInText($) {
|
||||
const text = $('body').text();
|
||||
const match = text.match(/€\s?(\d+[.,]\d{2})/);
|
||||
return match ? parsePriceValue(match[1]) : null;
|
||||
}
|
||||
|
||||
function extractPrice(html) {
|
||||
const $ = cheerio.load(html);
|
||||
let price = findPriceInJsonLd($);
|
||||
if (price != null) return { price, method: 'json-ld' };
|
||||
price = findPriceInMeta($);
|
||||
if (price != null) return { price, method: 'meta' };
|
||||
price = findPriceInText($);
|
||||
if (price != null) return { price, method: 'regex' };
|
||||
return { price: null, method: null };
|
||||
}
|
||||
|
||||
async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(url);
|
||||
} catch (err) {
|
||||
return { price: null, method: null, error: err };
|
||||
}
|
||||
if (!response.ok) {
|
||||
return { price: null, method: null, error: new Error(`HTTP ${response.status}`) };
|
||||
}
|
||||
const html = await response.text();
|
||||
const { price, method } = extractPrice(html);
|
||||
return { price, method, error: null };
|
||||
}
|
||||
|
||||
module.exports = { extractPrice, fetchAndExtractPrice, parsePriceValue };
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
<html><head>
|
||||
<script type="application/ld+json">
|
||||
{"@context":"https://schema.org","@type":"Product","name":"Voorbeeld product","offers":{"@type":"Offer","price":"19.99","priceCurrency":"EUR"}}
|
||||
</script>
|
||||
</head><body><p>Voorbeeld product</p></body></html>
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
<html><head>
|
||||
<meta property="og:price:amount" content="24.50" />
|
||||
</head><body><p>Voorbeeld product</p></body></html>
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
<html><body><p>Geen prijs op deze pagina.</p></body></html>
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
<html><body><p>Onze prijs: €14,95 vandaag</p></body></html>
|
||||
@@ -0,0 +1,50 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { extractPrice, fetchAndExtractPrice } = require('../src/scraper');
|
||||
|
||||
function loadFixture(name) {
|
||||
return fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8');
|
||||
}
|
||||
|
||||
test('extractPrice reads price from JSON-LD Product offers', () => {
|
||||
const result = extractPrice(loadFixture('json-ld.html'));
|
||||
assert.deepEqual(result, { price: 19.99, method: 'json-ld' });
|
||||
});
|
||||
|
||||
test('extractPrice reads price from og:price:amount meta tag', () => {
|
||||
const result = extractPrice(loadFixture('meta-tag.html'));
|
||||
assert.deepEqual(result, { price: 24.50, method: 'meta' });
|
||||
});
|
||||
|
||||
test('extractPrice falls back to euro-amount regex in page text', () => {
|
||||
const result = extractPrice(loadFixture('regex-fallback.html'));
|
||||
assert.deepEqual(result, { price: 14.95, method: 'regex' });
|
||||
});
|
||||
|
||||
test('extractPrice returns null price when nothing matches', () => {
|
||||
const result = extractPrice(loadFixture('no-price.html'));
|
||||
assert.deepEqual(result, { price: null, method: null });
|
||||
});
|
||||
|
||||
test('fetchAndExtractPrice returns error when fetch rejects', async () => {
|
||||
const fetchImpl = async () => { throw new Error('network down'); };
|
||||
const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl });
|
||||
assert.equal(result.price, null);
|
||||
assert.ok(result.error instanceof Error);
|
||||
});
|
||||
|
||||
test('fetchAndExtractPrice returns error on non-OK response', async () => {
|
||||
const fetchImpl = async () => ({ ok: false, status: 404, text: async () => '' });
|
||||
const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl });
|
||||
assert.equal(result.price, null);
|
||||
assert.ok(result.error instanceof Error);
|
||||
});
|
||||
|
||||
test('fetchAndExtractPrice extracts price from fetched HTML', async () => {
|
||||
const html = loadFixture('meta-tag.html');
|
||||
const fetchImpl = async () => ({ ok: true, status: 200, text: async () => html });
|
||||
const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl });
|
||||
assert.deepEqual(result, { price: 24.50, method: 'meta', error: null });
|
||||
});
|
||||
Reference in New Issue
Block a user