From 52c99472671594f9dfaf6843f5fd4926585c872b Mon Sep 17 00:00:00 2001
From: Sjoerd de Vries <openai@sjoerddev.com>
Date: Sat, 5 Sep 2026 15:49:56 +0200
Subject: [PATCH] fix: harden price scraping (findings 1, 2, 6)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Finding 1: parsePriceValue mishandled thousands separators — "1.299,00"
and "1,299.00" both parsed to 1.3. Both the meta-tag path and the
text-regex path now funnel through a single normalizeAmount() helper
that picks the last separator as the decimal one, and the text regex
also matches thousands-grouped amounts.

Finding 2: the text fallback read <script>/<style> contents via
$('body').text(); those elements are now stripped from a clone first.

Finding 6: the default fetch now uses a 15s AbortSignal.timeout and a
browser-like User-Agent, so one unresponsive host cannot stall the
whole sequential run. The fetchImpl injection point is unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
---
 src/scraper.js                     | 50 ++++++++++++++++++++++++++----
 test/fixtures/script-decoy.html    |  5 +++
 test/fixtures/thousands-meta.html  |  3 ++
 test/fixtures/thousands-regex.html |  1 +
 test/scraper.test.js               | 40 +++++++++++++++++++++++-
 5 files changed, 92 insertions(+), 7 deletions(-)
 create mode 100644 test/fixtures/script-decoy.html
 create mode 100644 test/fixtures/thousands-meta.html
 create mode 100644 test/fixtures/thousands-regex.html

diff --git a/src/scraper.js b/src/scraper.js
index 0439a85..2e1f9a2 100644
--- a/src/scraper.js
+++ b/src/scraper.js
@@ -4,10 +4,32 @@ function roundMoney(n) {
   return Math.round(n * 100) / 100;
 }
 
-function parsePriceValue(raw) {
+// Turns a human-written amount into a plain JS-parsable number string.
+// When both separators occur, the last one is the decimal separator
+// ("1.299,00" and "1,299.00" both mean 1299.00); a lone comma is decimal
+// only when exactly two digits follow it, otherwise it groups thousands.
+function normalizeAmount(raw) {
   if (raw == null) return null;
-  const cleaned = String(raw).trim().replace(',', '.');
-  const value = parseFloat(cleaned);
+  const match = String(raw).trim().match(/-?\d[\d.,]*/);
+  if (!match) return null;
+  const str = match[0].replace(/[.,]+$/, '');
+  const lastDot = str.lastIndexOf('.');
+  const lastComma = str.lastIndexOf(',');
+  if (lastDot !== -1 && lastComma !== -1) {
+    return lastComma > lastDot
+      ? str.replace(/\./g, '').replace(',', '.')
+      : str.replace(/,/g, '');
+  }
+  if (lastComma !== -1) {
+    return /,\d{2}$/.test(str) ? str.replace(',', '.') : str.replace(/,/g, '');
+  }
+  return str;
+}
+
+function parsePriceValue(raw) {
+  const normalized = normalizeAmount(raw);
+  if (normalized == null) return null;
+  const value = parseFloat(normalized);
   return Number.isFinite(value) ? roundMoney(value) : null;
 }
 
@@ -59,9 +81,14 @@ function findPriceInMeta($) {
   return null;
 }
 
+// Thousands-grouped amounts first, then a plain amount, so "€1.299,00"
+// is not truncated to "€1.29".
+const TEXT_PRICE_RE = /€\s?(\d{1,3}(?:[.,]\d{3})+[.,]\d{2}|\d+[.,]\d{2})/;
+
 function findPriceInText($) {
-  const text = $('body').text();
-  const match = text.match(/€\s?(\d+[.,]\d{2})/);
+  const text = $('body').clone().find('script, style, noscript').remove().end()
+    .text();
+  const match = text.match(TEXT_PRICE_RE);
   return match ? parsePriceValue(match[1]) : null;
 }
 
@@ -76,7 +103,18 @@ function extractPrice(html) {
   return { price: null, method: null };
 }
 
-async function fetchAndExtractPrice(url, { fetchImpl = fetch } = {}) {
+const FETCH_TIMEOUT_MS = 15000;
+const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+  + '(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36';
+
+function defaultFetch(url) {
+  return fetch(url, {
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+    headers: { 'User-Agent': USER_AGENT },
+  });
+}
+
+async function fetchAndExtractPrice(url, { fetchImpl = defaultFetch } = {}) {
   let response;
   try {
     response = await fetchImpl(url);
diff --git a/test/fixtures/script-decoy.html b/test/fixtures/script-decoy.html
new file mode 100644
index 0000000..5ea5f08
--- /dev/null
+++ b/test/fixtures/script-decoy.html
@@ -0,0 +1,5 @@
+<html><body>
+<script>var config = {"fallbackPrice": "€9,99"};</script>
+<style>.price:after { content: "€1,11"; }</style>
+<p>Onze prijs: &euro;49,95 vandaag</p>
+</body></html>
\ No newline at end of file
diff --git a/test/fixtures/thousands-meta.html b/test/fixtures/thousands-meta.html
new file mode 100644
index 0000000..2a16caf
--- /dev/null
+++ b/test/fixtures/thousands-meta.html
@@ -0,0 +1,3 @@
+<html><head>
+<meta property="product:price:amount" content="1.299,00" />
+</head><body><p>Duur product</p></body></html>
\ No newline at end of file
diff --git a/test/fixtures/thousands-regex.html b/test/fixtures/thousands-regex.html
new file mode 100644
index 0000000..dbbb965
--- /dev/null
+++ b/test/fixtures/thousands-regex.html
@@ -0,0 +1 @@
+<html><body><p>Nu voor &euro;1.299,00 incl. btw</p></body></html>
\ No newline at end of file
diff --git a/test/scraper.test.js b/test/scraper.test.js
index 0c217b3..a3ebfbc 100644
--- a/test/scraper.test.js
+++ b/test/scraper.test.js
@@ -2,7 +2,7 @@ const test = require('node:test');
 const assert = require('node:assert/strict');
 const fs = require('node:fs');
 const path = require('node:path');
-const { extractPrice, fetchAndExtractPrice } = require('../src/scraper');
+const { extractPrice, fetchAndExtractPrice, parsePriceValue } = require('../src/scraper');
 
 function loadFixture(name) {
   return fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8');
@@ -28,6 +28,31 @@ test('extractPrice returns null price when nothing matches', () => {
   assert.deepEqual(result, { price: null, method: null });
 });
 
+test('parsePriceValue handles both thousands-separator conventions', () => {
+  assert.equal(parsePriceValue('1.299,00'), 1299.00);
+  assert.equal(parsePriceValue('1,299.00'), 1299.00);
+  assert.equal(parsePriceValue('19,99'), 19.99);
+  assert.equal(parsePriceValue('19.99'), 19.99);
+  assert.equal(parsePriceValue('1.234.567,89'), 1234567.89);
+  assert.equal(parsePriceValue(null), null);
+  assert.equal(parsePriceValue('geen prijs'), null);
+});
+
+test('extractPrice reads a thousands-grouped price from a meta tag', () => {
+  const result = extractPrice(loadFixture('thousands-meta.html'));
+  assert.deepEqual(result, { price: 1299.00, method: 'meta' });
+});
+
+test('extractPrice reads a thousands-grouped price from page text', () => {
+  const result = extractPrice(loadFixture('thousands-regex.html'));
+  assert.deepEqual(result, { price: 1299.00, method: 'regex' });
+});
+
+test('extractPrice ignores euro amounts inside script and style tags', () => {
+  const result = extractPrice(loadFixture('script-decoy.html'));
+  assert.deepEqual(result, { price: 49.95, method: 'regex' });
+});
+
 test('fetchAndExtractPrice returns error when fetch rejects', async () => {
   const fetchImpl = async () => { throw new Error('network down'); };
   const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl });
@@ -42,6 +67,19 @@ test('fetchAndExtractPrice returns error on non-OK response', async () => {
   assert.ok(result.error instanceof Error);
 });
 
+test('fetchAndExtractPrice returns error when the request times out', async () => {
+  const fetchImpl = async () => {
+    const err = new Error('The operation was aborted due to timeout');
+    err.name = 'TimeoutError';
+    throw err;
+  };
+  const result = await fetchAndExtractPrice('https://example.invalid', { fetchImpl });
+  assert.deepEqual(result.price, null);
+  assert.equal(result.method, null);
+  assert.ok(result.error instanceof Error);
+  assert.equal(result.error.name, 'TimeoutError');
+});
+
 test('fetchAndExtractPrice extracts price from fetched HTML', async () => {
   const html = loadFixture('meta-tag.html');
   const fetchImpl = async () => ({ ok: true, status: 200, text: async () => html });