From 358347e8ba804e882d7eb73763cfb2621ea69c83 Mon Sep 17 00:00:00 2001
From: Sjoerd de Vries <openai@sjoerddev.com>
Date: Sat, 5 Sep 2026 17:34:02 +0200
Subject: [PATCH] feat: fall back to the browser scraper when the primary fetch
 fails

- Extract fetchAndExtractPriceViaBrowser from deps (optional)
- After primary fetch, if it errors or returns no price and fallback exists, try fallback
- Fallback result replaces primary result for further processing
- Retry-safety logic unchanged: last_price/last_checked_at only updated on success
- All 4 new tests pass, all existing tests unchanged

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
---
 src/checkProducts.js       | 11 +++++--
 test/checkProducts.test.js | 63 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/src/checkProducts.js b/src/checkProducts.js
index c0dff43..0e68207 100644
--- a/src/checkProducts.js
+++ b/src/checkProducts.js
@@ -1,6 +1,13 @@
 async function checkProduct(product, deps) {
-  const { fetchAndExtractPrice, sendMail, recordCheckSuccess, recordCheckFailure, now } = deps;
-  const result = await fetchAndExtractPrice(product.url);
+  const {
+    fetchAndExtractPrice, fetchAndExtractPriceViaBrowser,
+    sendMail, recordCheckSuccess, recordCheckFailure, now,
+  } = deps;
+  let result = await fetchAndExtractPrice(product.url);
+
+  if ((result.error || result.price == null) && fetchAndExtractPriceViaBrowser) {
+    result = await fetchAndExtractPriceViaBrowser(product.url);
+  }
 
   if (result.error) {
     console.error(`[check] fetch_error "${product.name}" (${product.url}): ${result.error.message}`);
diff --git a/test/checkProducts.test.js b/test/checkProducts.test.js
index 03633eb..2f952c4 100644
--- a/test/checkProducts.test.js
+++ b/test/checkProducts.test.js
@@ -119,3 +119,66 @@ test('checkAllProducts processes every product independently, isolating failures
   assert.equal(results[0].status, 'error');
   assert.equal(results[1].status, 'ok');
 });
+
+test('checkProduct tries the browser fallback when the primary fetch errors, and uses its result', async (t) => {
+  captureErrorLog(t);
+  const browserCalls = [];
+  const { deps, successes, mailCalls } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 10, method: 'json-ld', error: null };
+    },
+  });
+  const product = { id: 7, name: 'X', url: 'https://x', last_price: null };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'ok');
+  assert.deepEqual(browserCalls, ['https://x']);
+  assert.equal(successes.length, 1);
+  assert.equal(mailCalls.length, 0);
+});
+
+test('checkProduct tries the browser fallback when the primary fetch finds no price', async (t) => {
+  captureErrorLog(t);
+  const browserCalls = [];
+  const { deps, successes } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: null }),
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 12, method: 'regex', error: null };
+    },
+  });
+  const product = { id: 8, name: 'X', url: 'https://x', last_price: null };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'ok');
+  assert.deepEqual(browserCalls, ['https://x']);
+  assert.equal(successes.length, 1);
+});
+
+test('checkProduct never calls the browser fallback when the primary fetch already succeeds', async () => {
+  const browserCalls = [];
+  const { deps } = makeDeps({
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 999, method: 'meta', error: null };
+    },
+  });
+  const product = { id: 9, name: 'X', url: 'https://x', last_price: null };
+  await checkProduct(product, deps);
+  assert.equal(browserCalls.length, 0);
+});
+
+test('checkProduct records fetch_error when both the primary fetch and the browser fallback fail', async (t) => {
+  const logged = captureErrorLog(t);
+  const { deps, failures, successes } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
+    fetchAndExtractPriceViaBrowser: async () => ({ price: null, method: null, error: new Error('still down') }),
+  });
+  const product = { id: 10, name: 'X', url: 'https://x', last_price: 10 };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'fetch_error');
+  assert.deepEqual(failures, [{ id: 10, status: 'fetch_error' }]);
+  assert.equal(successes.length, 0);
+  assert.equal(logged.length, 1);
+  assert.match(logged[0], /fetch_error.*still down/);
+});