diff --git a/src/scraper.js b/src/scraper.js new file mode 100644 index 0000000..0439a85 --- /dev/null +++ b/src/scraper.js @@ -0,0 +1,94 @@ +const cheerio = require('cheerio'); + +function roundMoney(n) { + return Math.round(n * 100) / 100; +} + +function parsePriceValue(raw) { + if (raw == null) return null; + const cleaned = String(raw).trim().replace(',', '.'); + const value = parseFloat(cleaned); + return Number.isFinite(value) ? roundMoney(value) : null; +} + +function searchJsonLdNode(node) { + if (Array.isArray(node)) { + for (const item of node) { + const price = searchJsonLdNode(item); + if (price != null) return price; + } + return null; + } + if (node && typeof node === 'object') { + const type = node['@type']; + const isProduct = type === 'Product' || (Array.isArray(type) && type.includes('Product')); + if (isProduct && node.offers) { + const offers = Array.isArray(node.offers) ? node.offers[0] : node.offers; + const price = parsePriceValue(offers && offers.price); + if (price != null) return price; + } + if (Array.isArray(node['@graph'])) { + const price = searchJsonLdNode(node['@graph']); + if (price != null) return price; + } + } + return null; +} + +function findPriceInJsonLd($) { + const scripts = $('script[type="application/ld+json"]').toArray(); + for (const el of scripts) { + let data; + try { + data = JSON.parse($(el).contents().text()); + } catch { + continue; + } + const price = searchJsonLdNode(data); + if (price != null) return price; + } + return null; +} + +function findPriceInMeta($) { + const selectors = ['meta[property="product:price:amount"]', 'meta[property="og:price:amount"]']; + for (const selector of selectors) { + const price = parsePriceValue($(selector).attr('content')); + if (price != null) return price; + } + return null; +} + +function findPriceInText($) { + const text = $('body').text(); + const match = text.match(/€\s?(\d+[.,]\d{2})/); + return match ? parsePriceValue(match[1]) : null; +} + +function extractPrice(html) { + const $ = cheerio.load(html); + let price = findPriceInJsonLd($); + if (price != null) return { price, method: 'json-ld' }; + price = findPriceInMeta($); + if (price != null) return { price, method: 'meta' }; + price = findPriceInText($); + if (price != null) return { price, method: 'regex' }; + return { price: null, method: null }; +} + +async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) { + let response; + try { + response = await fetchImpl(url); + } catch (err) { + return { price: null, method: null, error: err }; + } + if (!response.ok) { + return { price: null, method: null, error: new Error(`HTTP ${response.status}`) }; + } + const html = await response.text(); + const { price, method } = extractPrice(html); + return { price, method, error: null }; +} + +module.exports = { extractPrice, fetchAndExtractPrice, parsePriceValue }; diff --git a/test/fixtures/json-ld.html b/test/fixtures/json-ld.html new file mode 100644 index 0000000..6c3352a --- /dev/null +++ b/test/fixtures/json-ld.html @@ -0,0 +1,5 @@ + + +

Voorbeeld product

\ No newline at end of file diff --git a/test/fixtures/meta-tag.html b/test/fixtures/meta-tag.html new file mode 100644 index 0000000..ac186f8 --- /dev/null +++ b/test/fixtures/meta-tag.html @@ -0,0 +1,3 @@ + + +

Voorbeeld product

\ No newline at end of file diff --git a/test/fixtures/no-price.html b/test/fixtures/no-price.html new file mode 100644 index 0000000..3c0b5ac --- /dev/null +++ b/test/fixtures/no-price.html @@ -0,0 +1 @@ +

Geen prijs op deze pagina.

\ No newline at end of file diff --git a/test/fixtures/regex-fallback.html b/test/fixtures/regex-fallback.html new file mode 100644 index 0000000..49b6b40 --- /dev/null +++ b/test/fixtures/regex-fallback.html @@ -0,0 +1 @@ +

Onze prijs: €14,95 vandaag

\ No newline at end of file diff --git a/test/scraper.test.js b/test/scraper.test.js new file mode 100644 index 0000000..0c217b3 --- /dev/null +++ b/test/scraper.test.js @@ -0,0 +1,50 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { extractPrice, fetchAndExtractPrice } = require('../src/scraper'); + +function loadFixture(name) { + return fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8'); +} + +test('extractPrice reads price from JSON-LD Product offers', () => { + const result = extractPrice(loadFixture('json-ld.html')); + assert.deepEqual(result, { price: 19.99, method: 'json-ld' }); +}); + +test('extractPrice reads price from og:price:amount meta tag', () => { + const result = extractPrice(loadFixture('meta-tag.html')); + assert.deepEqual(result, { price: 24.50, method: 'meta' }); +}); + +test('extractPrice falls back to euro-amount regex in page text', () => { + const result = extractPrice(loadFixture('regex-fallback.html')); + assert.deepEqual(result, { price: 14.95, method: 'regex' }); +}); + +test('extractPrice returns null price when nothing matches', () => { + const result = extractPrice(loadFixture('no-price.html')); + assert.deepEqual(result, { price: null, method: null }); +}); + +test('fetchAndExtractPrice returns error when fetch rejects', async () => { + const fetchImpl = async () => { throw new Error('network down'); }; + const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl }); + assert.equal(result.price, null); + assert.ok(result.error instanceof Error); +}); + +test('fetchAndExtractPrice returns error on non-OK response', async () => { + const fetchImpl = async () => ({ ok: false, status: 404, text: async () => '' }); + const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl }); + assert.equal(result.price, null); + assert.ok(result.error instanceof Error); +}); + +test('fetchAndExtractPrice extracts price from fetched HTML', async () => { + const html = loadFixture('meta-tag.html'); + const fetchImpl = async () => ({ ok: true, status: 200, text: async () => html }); + const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl }); + assert.deepEqual(result, { price: 24.50, method: 'meta', error: null }); +});