feat: add Playwright-based fallback scraper

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
2026-09-05 17:30:18 +02:00
parent dac7893cd2
commit df3d3250e5
4 changed files with 126 additions and 2 deletions
@@ -14,7 +14,8 @@
        "express": "^4.19.2",
        "express-basic-auth": "^1.2.1",
        "node-cron": "^3.0.3",
-        "nodemailer": "^6.9.14"
+        "nodemailer": "^6.9.14",
+        "playwright": "^1.63.0"
      },
      "devDependencies": {
        "supertest": "^7.0.0"
@@ -1285,6 +1286,33 @@
      "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
      "license": "MIT"
    },
+    "node_modules/playwright": {
+      "version": "1.63.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz",
+      "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.63.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.63.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz",
+      "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
    "node_modules/prebuild-install": {
      "version": "7.1.3",
      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@@ -15,7 +15,8 @@
    "express": "^4.19.2",
    "express-basic-auth": "^1.2.1",
    "node-cron": "^3.0.3",
-    "nodemailer": "^6.9.14"
+    "nodemailer": "^6.9.14",
+    "playwright": "^1.63.0"
  },
  "devDependencies": {
    "supertest": "^7.0.0"
@@ -0,0 +1,25 @@
+const { chromium } = require('playwright');
+const { extractPrice } = require('./scraper');
+
+async function launchBrowser() {
+  return chromium.launch({ headless: true });
+}
+
+async function fetchAndExtractPriceViaBrowser(browser, url, { timeoutMs = 20000 } = {}) {
+  let page;
+  try {
+    page = await browser.newPage();
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+    const html = await page.content();
+    const { price, method } = extractPrice(html);
+    return { price, method, error: null };
+  } catch (err) {
+    return { price: null, method: null, error: err };
+  } finally {
+    if (page) {
+      await page.close().catch(() => {});
+    }
+  }
+}
+
+module.exports = { launchBrowser, fetchAndExtractPriceViaBrowser };
@@ -0,0 +1,70 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { fetchAndExtractPriceViaBrowser } = require('../src/browserScraper');
+
+test('fetchAndExtractPriceViaBrowser extracts a price from the rendered page', async () => {
+  let closed = false;
+  const html = '<html><head><meta property="og:price:amount" content="42.00" /></head><body></body></html>';
+  const page = {
+    goto: async () => {},
+    content: async () => html,
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.deepEqual(result, { price: 42, method: 'meta', error: null });
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns an error when navigation fails, and still closes the page', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => { throw new Error('Timeout 20000ms exceeded'); },
+    content: async () => '',
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.equal(result.price, null);
+  assert.equal(result.method, null);
+  assert.ok(result.error instanceof Error);
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns null price when nothing matches, and still closes the page', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => {},
+    content: async () => '<html><body><p>Geen prijs</p></body></html>',
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.deepEqual(result, { price: null, method: null, error: null });
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser closes the page even if reading its content throws', async () => {
+  let closed = false;
+  const page = {
+    goto: async () => {},
+    content: async () => { throw new Error('content failed'); },
+    close: async () => { closed = true; },
+  };
+  const browser = { newPage: async () => page };
+  const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+  assert.ok(result.error instanceof Error);
+  assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser passes the timeout option through to page.goto', async () => {
+  let receivedOptions;
+  const page = {
+    goto: async (url, options) => { receivedOptions = options; },
+    content: async () => '<html><body></body></html>',
+    close: async () => {},
+  };
+  const browser = { newPage: async () => page };
+  await fetchAndExtractPriceViaBrowser(browser, 'https://example.com', { timeoutMs: 5000 });
+  assert.equal(receivedOptions.timeout, 5000);
+});