feat: fall back to the browser scraper when the primary fetch fails

- Extract fetchAndExtractPriceViaBrowser from deps (optional)
- After primary fetch, if it errors or returns no price and fallback exists, try fallback
- Fallback result replaces primary result for further processing
- Retry-safety logic unchanged: last_price/last_checked_at only updated on success
- All 4 new tests pass, all existing tests unchanged

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
This commit is contained in:
Sjoerd de Vries
2026-09-05 17:34:02 +02:00
parent df3d3250e5
commit 358347e8ba
2 changed files with 72 additions and 2 deletions
+9 -2
View File
@@ -1,6 +1,13 @@
async function checkProduct(product, deps) { async function checkProduct(product, deps) {
const { fetchAndExtractPrice, sendMail, recordCheckSuccess, recordCheckFailure, now } = deps; const {
const result = await fetchAndExtractPrice(product.url); fetchAndExtractPrice, fetchAndExtractPriceViaBrowser,
sendMail, recordCheckSuccess, recordCheckFailure, now,
} = deps;
let result = await fetchAndExtractPrice(product.url);
if ((result.error || result.price == null) && fetchAndExtractPriceViaBrowser) {
result = await fetchAndExtractPriceViaBrowser(product.url);
}
if (result.error) { if (result.error) {
console.error(`[check] fetch_error "${product.name}" (${product.url}): ${result.error.message}`); console.error(`[check] fetch_error "${product.name}" (${product.url}): ${result.error.message}`);
+63
View File
@@ -119,3 +119,66 @@ test('checkAllProducts processes every product independently, isolating failures
assert.equal(results[0].status, 'error'); assert.equal(results[0].status, 'error');
assert.equal(results[1].status, 'ok'); assert.equal(results[1].status, 'ok');
}); });
test('checkProduct tries the browser fallback when the primary fetch errors, and uses its result', async (t) => {
captureErrorLog(t);
const browserCalls = [];
const { deps, successes, mailCalls } = makeDeps({
fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
fetchAndExtractPriceViaBrowser: async (url) => {
browserCalls.push(url);
return { price: 10, method: 'json-ld', error: null };
},
});
const product = { id: 7, name: 'X', url: 'https://x', last_price: null };
const result = await checkProduct(product, deps);
assert.equal(result.status, 'ok');
assert.deepEqual(browserCalls, ['https://x']);
assert.equal(successes.length, 1);
assert.equal(mailCalls.length, 0);
});
test('checkProduct tries the browser fallback when the primary fetch finds no price', async (t) => {
captureErrorLog(t);
const browserCalls = [];
const { deps, successes } = makeDeps({
fetchAndExtractPrice: async () => ({ price: null, method: null, error: null }),
fetchAndExtractPriceViaBrowser: async (url) => {
browserCalls.push(url);
return { price: 12, method: 'regex', error: null };
},
});
const product = { id: 8, name: 'X', url: 'https://x', last_price: null };
const result = await checkProduct(product, deps);
assert.equal(result.status, 'ok');
assert.deepEqual(browserCalls, ['https://x']);
assert.equal(successes.length, 1);
});
test('checkProduct never calls the browser fallback when the primary fetch already succeeds', async () => {
const browserCalls = [];
const { deps } = makeDeps({
fetchAndExtractPriceViaBrowser: async (url) => {
browserCalls.push(url);
return { price: 999, method: 'meta', error: null };
},
});
const product = { id: 9, name: 'X', url: 'https://x', last_price: null };
await checkProduct(product, deps);
assert.equal(browserCalls.length, 0);
});
test('checkProduct records fetch_error when both the primary fetch and the browser fallback fail', async (t) => {
const logged = captureErrorLog(t);
const { deps, failures, successes } = makeDeps({
fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
fetchAndExtractPriceViaBrowser: async () => ({ price: null, method: null, error: new Error('still down') }),
});
const product = { id: 10, name: 'X', url: 'https://x', last_price: 10 };
const result = await checkProduct(product, deps);
assert.equal(result.status, 'fetch_error');
assert.deepEqual(failures, [{ id: 10, status: 'fetch_error' }]);
assert.equal(successes.length, 0);
assert.equal(logged.length, 1);
assert.match(logged[0], /fetch_error.*still down/);
});