diff --git a/src/scraper.js b/src/scraper.js index 0439a85..2e1f9a2 100644 --- a/src/scraper.js +++ b/src/scraper.js @@ -4,10 +4,32 @@ function roundMoney(n) { return Math.round(n * 100) / 100; } -function parsePriceValue(raw) { +// Turns a human-written amount into a plain JS-parsable number string. +// When both separators occur, the last one is the decimal separator +// ("1.299,00" and "1,299.00" both mean 1299.00); a lone comma is decimal +// only when exactly two digits follow it, otherwise it groups thousands. +function normalizeAmount(raw) { if (raw == null) return null; - const cleaned = String(raw).trim().replace(',', '.'); - const value = parseFloat(cleaned); + const match = String(raw).trim().match(/-?\d[\d.,]*/); + if (!match) return null; + const str = match[0].replace(/[.,]+$/, ''); + const lastDot = str.lastIndexOf('.'); + const lastComma = str.lastIndexOf(','); + if (lastDot !== -1 && lastComma !== -1) { + return lastComma > lastDot + ? str.replace(/\./g, '').replace(',', '.') + : str.replace(/,/g, ''); + } + if (lastComma !== -1) { + return /,\d{2}$/.test(str) ? str.replace(',', '.') : str.replace(/,/g, ''); + } + return str; +} + +function parsePriceValue(raw) { + const normalized = normalizeAmount(raw); + if (normalized == null) return null; + const value = parseFloat(normalized); return Number.isFinite(value) ? roundMoney(value) : null; } @@ -59,9 +81,14 @@ function findPriceInMeta($) { return null; } +// Thousands-grouped amounts first, then a plain amount, so "€1.299,00" +// is not truncated to "€1.29". +const TEXT_PRICE_RE = /€\s?(\d{1,3}(?:[.,]\d{3})+[.,]\d{2}|\d+[.,]\d{2})/; + function findPriceInText($) { - const text = $('body').text(); - const match = text.match(/€\s?(\d+[.,]\d{2})/); + const text = $('body').clone().find('script, style, noscript').remove().end() + .text(); + const match = text.match(TEXT_PRICE_RE); return match ? parsePriceValue(match[1]) : null; } @@ -76,7 +103,18 @@ function extractPrice(html) { return { price: null, method: null }; } -async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) { +const FETCH_TIMEOUT_MS = 15000; +const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'; + +function defaultFetch(url) { + return fetch(url, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + headers: { 'User-Agent': USER_AGENT }, + }); +} + +async function fetchAndExtractPrice(url, { fetchImpl = defaultFetch } = {}) { let response; try { response = await fetchImpl(url); diff --git a/test/fixtures/script-decoy.html b/test/fixtures/script-decoy.html new file mode 100644 index 0000000..5ea5f08 --- /dev/null +++ b/test/fixtures/script-decoy.html @@ -0,0 +1,5 @@ + + + +

Onze prijs: €49,95 vandaag

+ \ No newline at end of file diff --git a/test/fixtures/thousands-meta.html b/test/fixtures/thousands-meta.html new file mode 100644 index 0000000..2a16caf --- /dev/null +++ b/test/fixtures/thousands-meta.html @@ -0,0 +1,3 @@ + + +

Duur product

\ No newline at end of file diff --git a/test/fixtures/thousands-regex.html b/test/fixtures/thousands-regex.html new file mode 100644 index 0000000..dbbb965 --- /dev/null +++ b/test/fixtures/thousands-regex.html @@ -0,0 +1 @@ +

Nu voor €1.299,00 incl. btw

\ No newline at end of file diff --git a/test/scraper.test.js b/test/scraper.test.js index 0c217b3..a3ebfbc 100644 --- a/test/scraper.test.js +++ b/test/scraper.test.js @@ -2,7 +2,7 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); -const { extractPrice, fetchAndExtractPrice } = require('../src/scraper'); +const { extractPrice, fetchAndExtractPrice, parsePriceValue } = require('../src/scraper'); function loadFixture(name) { return fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8'); @@ -28,6 +28,31 @@ test('extractPrice returns null price when nothing matches', () => { assert.deepEqual(result, { price: null, method: null }); }); +test('parsePriceValue handles both thousands-separator conventions', () => { + assert.equal(parsePriceValue('1.299,00'), 1299.00); + assert.equal(parsePriceValue('1,299.00'), 1299.00); + assert.equal(parsePriceValue('19,99'), 19.99); + assert.equal(parsePriceValue('19.99'), 19.99); + assert.equal(parsePriceValue('1.234.567,89'), 1234567.89); + assert.equal(parsePriceValue(null), null); + assert.equal(parsePriceValue('geen prijs'), null); +}); + +test('extractPrice reads a thousands-grouped price from a meta tag', () => { + const result = extractPrice(loadFixture('thousands-meta.html')); + assert.deepEqual(result, { price: 1299.00, method: 'meta' }); +}); + +test('extractPrice reads a thousands-grouped price from page text', () => { + const result = extractPrice(loadFixture('thousands-regex.html')); + assert.deepEqual(result, { price: 1299.00, method: 'regex' }); +}); + +test('extractPrice ignores euro amounts inside script and style tags', () => { + const result = extractPrice(loadFixture('script-decoy.html')); + assert.deepEqual(result, { price: 49.95, method: 'regex' }); +}); + test('fetchAndExtractPrice returns error when fetch rejects', async () => { const fetchImpl = async () => { throw new Error('network down'); }; const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl }); @@ -42,6 +67,19 @@ test('fetchAndExtractPrice returns error on non-OK response', async () => { assert.ok(result.error instanceof Error); }); +test('fetchAndExtractPrice returns error when the request times out', async () => { + const fetchImpl = async () => { + const err = new Error('The operation was aborted due to timeout'); + err.name = 'TimeoutError'; + throw err; + }; + const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl }); + assert.deepEqual(result.price, null); + assert.equal(result.method, null); + assert.ok(result.error instanceof Error); + assert.equal(result.error.name, 'TimeoutError'); +}); + test('fetchAndExtractPrice extracts price from fetched HTML', async () => { const html = loadFixture('meta-tag.html'); const fetchImpl = async () => ({ ok: true, status: 200, text: async () => html });