diff --git a/package-lock.json b/package-lock.json
index 0881b36..fce5014 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,7 +14,8 @@
"express": "^4.19.2",
"express-basic-auth": "^1.2.1",
"node-cron": "^3.0.3",
- "nodemailer": "^6.9.14"
+ "nodemailer": "^6.9.14",
+ "playwright": "^1.63.0"
},
"devDependencies": {
"supertest": "^7.0.0"
@@ -1285,6 +1286,33 @@
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"license": "MIT"
},
+ "node_modules/playwright": {
+ "version": "1.63.0",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz",
+ "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright-core": "1.63.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
+ "node_modules/playwright-core": {
+ "version": "1.63.0",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz",
+ "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==",
+ "license": "Apache-2.0",
+ "bin": {
+ "playwright-core": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
"node_modules/prebuild-install": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
diff --git a/package.json b/package.json
index 98d94e1..32bfe3b 100644
--- a/package.json
+++ b/package.json
@@ -15,7 +15,8 @@
"express": "^4.19.2",
"express-basic-auth": "^1.2.1",
"node-cron": "^3.0.3",
- "nodemailer": "^6.9.14"
+ "nodemailer": "^6.9.14",
+ "playwright": "^1.63.0"
},
"devDependencies": {
"supertest": "^7.0.0"
diff --git a/src/browserScraper.js b/src/browserScraper.js
new file mode 100644
index 0000000..76dc22b
--- /dev/null
+++ b/src/browserScraper.js
@@ -0,0 +1,25 @@
+const { chromium } = require('playwright');
+const { extractPrice } = require('./scraper');
+
+async function launchBrowser() {
+ return chromium.launch({ headless: true });
+}
+
+async function fetchAndExtractPriceViaBrowser(browser, url, { timeoutMs = 20000 } = {}) {
+ let page;
+ try {
+ page = await browser.newPage();
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+ const html = await page.content();
+ const { price, method } = extractPrice(html);
+ return { price, method, error: null };
+ } catch (err) {
+ return { price: null, method: null, error: err };
+ } finally {
+ if (page) {
+ await page.close().catch(() => {});
+ }
+ }
+}
+
+module.exports = { launchBrowser, fetchAndExtractPriceViaBrowser };
diff --git a/test/browserScraper.test.js b/test/browserScraper.test.js
new file mode 100644
index 0000000..4f632eb
--- /dev/null
+++ b/test/browserScraper.test.js
@@ -0,0 +1,70 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { fetchAndExtractPriceViaBrowser } = require('../src/browserScraper');
+
+test('fetchAndExtractPriceViaBrowser extracts a price from the rendered page', async () => {
+ let closed = false;
+ const html = '';
+ const page = {
+ goto: async () => {},
+ content: async () => html,
+ close: async () => { closed = true; },
+ };
+ const browser = { newPage: async () => page };
+ const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+ assert.deepEqual(result, { price: 42, method: 'meta', error: null });
+ assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns an error when navigation fails, and still closes the page', async () => {
+ let closed = false;
+ const page = {
+ goto: async () => { throw new Error('Timeout 20000ms exceeded'); },
+ content: async () => '',
+ close: async () => { closed = true; },
+ };
+ const browser = { newPage: async () => page };
+ const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+ assert.equal(result.price, null);
+ assert.equal(result.method, null);
+ assert.ok(result.error instanceof Error);
+ assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser returns null price when nothing matches, and still closes the page', async () => {
+ let closed = false;
+ const page = {
+ goto: async () => {},
+ content: async () => 'Geen prijs
',
+ close: async () => { closed = true; },
+ };
+ const browser = { newPage: async () => page };
+ const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+ assert.deepEqual(result, { price: null, method: null, error: null });
+ assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser closes the page even if reading its content throws', async () => {
+ let closed = false;
+ const page = {
+ goto: async () => {},
+ content: async () => { throw new Error('content failed'); },
+ close: async () => { closed = true; },
+ };
+ const browser = { newPage: async () => page };
+ const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
+ assert.ok(result.error instanceof Error);
+ assert.equal(closed, true);
+});
+
+test('fetchAndExtractPriceViaBrowser passes the timeout option through to page.goto', async () => {
+ let receivedOptions;
+ const page = {
+ goto: async (url, options) => { receivedOptions = options; },
+ content: async () => '',
+ close: async () => {},
+ };
+ const browser = { newPage: async () => page };
+ await fetchAndExtractPriceViaBrowser(browser, 'https://example.com', { timeoutMs: 5000 });
+ assert.equal(receivedOptions.timeout, 5000);
+});