Fix final-review findings for Playwright fallback

- server.js: catch browser-launch failures inside the injected
  fetchAndExtractPriceViaBrowser wrapper so they route through
  checkProduct's normal error classification (recordCheckFailure)
  instead of escaping as a thrown error that bypasses the DB write.
- server.js: add an in-flight guard around runCheck so overlapping
  callers (cron + check-now) share one run instead of each launching
  its own Chromium.
- Dockerfile: copy only package.json/package-lock.json/node_modules
  before installing Chromium so the expensive install layer is keyed
  to dependency changes, not every commit; copy the rest of the app
  afterward.
- Dockerfile: clean up apt package lists after the Playwright install
  step, matching the build stage's existing cleanup.
- README.md: note that exercising the fallback locally (not just
  running the test suite) needs `npx playwright install chromium`.
- README.md: correct the fallback description — domcontentloaded
  navigation reads the page before most client-side JS finishes, so
  the fallback mainly helps with HTTP-client blocking, not JS-rendered
  prices; reworded to drop the inaccurate claim.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz

Dockerfile

@@ -8,6 +8,10 @@ COPY . .
 
 FROM node:22-slim
 WORKDIR /app
+COPY --from=build /app/package.json /app/package-lock.json ./
+COPY --from=build /app/node_modules ./node_modules
+RUN npx --yes playwright install --with-deps chromium \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=build /app .
 ENV NODE_ENV=production
 # De SQLite-database hoort op het persistente volume te staan; zonder deze

README.md

@@ -19,6 +19,11 @@ app bij het opstarten met een melding.
 De UI staat achter basic auth (`BASIC_AUTH_USER` / `BASIC_AUTH_PASS`) op
 `http://localhost:3000`.
 
+Wil je de headless-browser-fallback ook echt lokaal uitproberen (de
+testsuite gebruikt fakes en heeft hier geen echte browser voor nodig), run
+dan eerst `npx playwright install chromium` om de Chromium-binary te
+downloaden.
+
 ## Environment variables
 
 | Variabele | Omschrijving |
@@ -39,3 +44,10 @@ check. Een mislukte poging (pagina onbereikbaar, geen prijs gevonden,
 of de mail kon niet verstuurd worden) verandert de opgeslagen prijs
 niet, zodat de volgende dag opnieuw wordt geprobeerd. Via de "Nu
 controleren"-knop in de UI kan dit ook handmatig getriggerd worden.
+
+Als het ophalen van de prijs op de gewone manier mislukt (netwerkfout, of
+geen prijs gevonden in de HTML), probeert de app het één keer opnieuw met
+een echte headless-browser (Playwright/Chromium) — nuttig voor pagina's
+die eenvoudige HTTP-clients blokkeren. Geen garantie tegen elke vorm van
+bot-detectie, maar dekt een reëel deel van de gevallen die de gewone
+aanpak mist.

package-lock.json

@@ -14,7 +14,8 @@
         "express": "^4.19.2",
         "express-basic-auth": "^1.2.1",
         "node-cron": "^3.0.3",
-        "nodemailer": "^6.9.14"
+        "nodemailer": "^6.9.14",
+        "playwright": "^1.63.0"
       },
       "devDependencies": {
         "supertest": "^7.0.0"
@@ -1285,6 +1286,33 @@
       "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
       "license": "MIT"
     },
+    "node_modules/playwright": {
+      "version": "1.63.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz",
+      "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.63.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.63.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz",
+      "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
     "node_modules/prebuild-install": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",

package.json

@@ -15,7 +15,8 @@
     "express": "^4.19.2",
     "express-basic-auth": "^1.2.1",
     "node-cron": "^3.0.3",
-    "nodemailer": "^6.9.14"
+    "nodemailer": "^6.9.14",
+    "playwright": "^1.63.0"
   },
   "devDependencies": {
     "supertest": "^7.0.0"

server.js

@@ -8,6 +8,7 @@ const {
   recordCheckSuccess, recordCheckFailure,
 } = require('./src/db');
 const { fetchAndExtractPrice } = require('./src/scraper');
+const { launchBrowser, fetchAndExtractPriceViaBrowser } = require('./src/browserScraper');
 const { createTransport, sendPriceChangeEmail } = require('./src/mailer');
 const { checkAllProducts } = require('./src/checkProducts');
 const { start: startScheduler } = require('./src/scheduler');
@@ -36,23 +37,59 @@ function summarize(results) {
   return parts.length > 0 ? parts.join(' ') : 'geen producten';
 }
 
+let inFlightCheck = null;
 function runCheck() {
+  if (inFlightCheck) {
+    return inFlightCheck;
+  }
+  inFlightCheck = runCheckOnce().finally(() => {
+    inFlightCheck = null;
+  });
+  return inFlightCheck;
+}
+
+async function runCheckOnce() {
   const products = listProducts(db);
-  const deps = {
-    fetchAndExtractPrice,
-    sendMail: (payload) => sendPriceChangeEmail(transport, {
-      from: process.env.SMTP_FROM,
-      to: process.env.NOTIFY_EMAIL,
-      ...payload,
-    }),
-    recordCheckFailure: (id, status) => recordCheckFailure(db, id, status),
-    recordCheckSuccess: (id, data) => recordCheckSuccess(db, id, data),
-    now: () => new Date().toISOString(),
-  };
-  return checkAllProducts(products, deps).then((results) => {
+  let browserPromise = null;
+  function getBrowser() {
+    if (!browserPromise) {
+      browserPromise = launchBrowser();
+    }
+    return browserPromise;
+  }
+  try {
+    const deps = {
+      fetchAndExtractPrice,
+      fetchAndExtractPriceViaBrowser: async (url) => {
+        try {
+          const browser = await getBrowser();
+          return await fetchAndExtractPriceViaBrowser(browser, url);
+        } catch (err) {
+          return { price: null, method: null, error: err };
+        }
+      },
+      sendMail: (payload) => sendPriceChangeEmail(transport, {
+        from: process.env.SMTP_FROM,
+        to: process.env.NOTIFY_EMAIL,
+        ...payload,
+      }),
+      recordCheckFailure: (id, status) => recordCheckFailure(db, id, status),
+      recordCheckSuccess: (id, data) => recordCheckSuccess(db, id, data),
+      now: () => new Date().toISOString(),
+    };
+    const results = await checkAllProducts(products, deps);
     console.log(`[check] ${products.length} product(en) gecontroleerd: ${summarize(results)}`);
     return results;
-  });
+  } finally {
+    if (browserPromise) {
+      const browser = await browserPromise.catch(() => null);
+      if (browser) {
+        await browser.close().catch((err) => {
+          console.error(`[check] kon de browser niet netjes sluiten: ${err && err.message}`);
+        });
+      }
+    }
+  }
 }
 
 startScheduler(runCheck);

src/browserScraper.js

@@ -0,0 +1,25 @@
+const { chromium } = require('playwright');
+const { extractPrice } = require('./scraper');
+
+async function launchBrowser() {
+  return chromium.launch({ headless: true });
+}
+
+async function fetchAndExtractPriceViaBrowser(browser, url, { timeoutMs = 20000 } = {}) {
+  let page;
+  try {
+    page = await browser.newPage();
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+    const html = await page.content();
+    const { price, method } = extractPrice(html);
+    return { price, method, error: null };
+  } catch (err) {
+    return { price: null, method: null, error: err };
+  } finally {
+    if (page) {
+      await page.close().catch(() => {});
+    }
+  }
+}
+
+module.exports = { launchBrowser, fetchAndExtractPriceViaBrowser };

src/checkProducts.js

@@ -1,6 +1,13 @@
 async function checkProduct(product, deps) {
-  const { fetchAndExtractPrice, sendMail, recordCheckSuccess, recordCheckFailure, now } = deps;
-  const result = await fetchAndExtractPrice(product.url);
+  const {
+    fetchAndExtractPrice, fetchAndExtractPriceViaBrowser,
+    sendMail, recordCheckSuccess, recordCheckFailure, now,
+  } = deps;
+  let result = await fetchAndExtractPrice(product.url);
+
+  if ((result.error || result.price == null) && fetchAndExtractPriceViaBrowser) {
+    result = await fetchAndExtractPriceViaBrowser(product.url);
+  }
 
   if (result.error) {
     console.error(`[check] fetch_error "${product.name}" (${product.url}): ${result.error.message}`);

test/browserScraper.test.js

@@ -0,0 +1,70 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { fetchAndExtractPriceViaBrowser } = require('../src/browserScraper');
+
+test('fetchAndExtractPriceViaBrowser extracts a price from the rendered page', async () => {
+  let closed = false;
+  const html = '<html><head><meta property="og:price:amount" content="42.00" /></head><body></body></html>';
+  const page = {
+    goto: async () => {},
+    content: async () => html,
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.deepEqual(result, { price: 42, method: 'meta', error: null });
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns an error when navigation fails, and still closes the page', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => { throw new Error('Timeout 20000ms exceeded'); },
+    content: async () => '',
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.equal(result.price, null);
+  assert.equal(result.method, null);
+  assert.ok(result.error instanceof Error);
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns null price when nothing matches, and still closes the page', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => {},
+    content: async () => '<html><body><p>Geen prijs</p></body></html>',
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.deepEqual(result, { price: null, method: null, error: null });
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser closes the page even if reading its content throws', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => {},
+    content: async () => { throw new Error('content failed'); },
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.ok(result.error instanceof Error);
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser passes the timeout option through to page.goto', async () => {
+  let receivedOptions;
+  const page = {
+    goto: async (url, options) => { receivedOptions = options; },
+    content: async () => '<html><body></body></html>',
+    close: async () => {},
+  };
+  const browser = { newPage: async () => page };
+  await fetchAndExtractPriceViaBrowser(browser, 'https://example.com', { timeoutMs: 5000 });
+  assert.equal(receivedOptions.timeout, 5000);
+});

test/checkProducts.test.js

@@ -119,3 +119,66 @@ test('checkAllProducts processes every product independently, isolating failures
   assert.equal(results[0].status, 'error');
   assert.equal(results[1].status, 'ok');
 });
+
+test('checkProduct tries the browser fallback when the primary fetch errors, and uses its result', async (t) => {
+  captureErrorLog(t);
+  const browserCalls = [];
+  const { deps, successes, mailCalls } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 10, method: 'json-ld', error: null };
+    },
+  });
+  const product = { id: 7, name: 'X', url: 'https://x', last_price: null };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'ok');
+  assert.deepEqual(browserCalls, ['https://x']);
+  assert.equal(successes.length, 1);
+  assert.equal(mailCalls.length, 0);
+});
+
+test('checkProduct tries the browser fallback when the primary fetch finds no price', async (t) => {
+  captureErrorLog(t);
+  const browserCalls = [];
+  const { deps, successes } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: null }),
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 12, method: 'regex', error: null };
+    },
+  });
+  const product = { id: 8, name: 'X', url: 'https://x', last_price: null };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'ok');
+  assert.deepEqual(browserCalls, ['https://x']);
+  assert.equal(successes.length, 1);
+});
+
+test('checkProduct never calls the browser fallback when the primary fetch already succeeds', async () => {
+  const browserCalls = [];
+  const { deps } = makeDeps({
+    fetchAndExtractPriceViaBrowser: async (url) => {
+      browserCalls.push(url);
+      return { price: 999, method: 'meta', error: null };
+    },
+  });
+  const product = { id: 9, name: 'X', url: 'https://x', last_price: null };
+  await checkProduct(product, deps);
+  assert.equal(browserCalls.length, 0);
+});
+
+test('checkProduct records fetch_error when both the primary fetch and the browser fallback fail', async (t) => {
+  const logged = captureErrorLog(t);
+  const { deps, failures, successes } = makeDeps({
+    fetchAndExtractPrice: async () => ({ price: null, method: null, error: new Error('down') }),
+    fetchAndExtractPriceViaBrowser: async () => ({ price: null, method: null, error: new Error('still down') }),
+  });
+  const product = { id: 10, name: 'X', url: 'https://x', last_price: 10 };
+  const result = await checkProduct(product, deps);
+  assert.equal(result.status, 'fetch_error');
+  assert.deepEqual(failures, [{ id: 10, status: 'fetch_error' }]);
+  assert.equal(successes.length, 0);
+  assert.equal(logged.length, 1);
+  assert.match(logged[0], /fetch_error.*still down/);
+});