sjoerd/korting
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Playwright-based fallback scraper

Browse Source 
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013gwc3aCiyQxmCWQke8RvEz
This commit is contained in:
Sjoerd de Vries
2026-09-05 17:30:18 +02:00
parent dac7893cd2
commit df3d3250e5
4 changed files with 126 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+29 -1
View File
@@ -14,7 +14,8 @@
"express": "^4.19.2", "express": "^4.19.2",
"express-basic-auth": "^1.2.1", "express-basic-auth": "^1.2.1",
"node-cron": "^3.0.3", "node-cron": "^3.0.3",
"nodemailer": "^6.9.14" "nodemailer": "^6.9.14",
"playwright": "^1.63.0"
}, },
"devDependencies": { "devDependencies": {
"supertest": "^7.0.0" "supertest": "^7.0.0"
@@ -1285,6 +1286,33 @@
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/playwright": {
"version": "1.63.0",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz",
"integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==",
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.63.0"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/playwright-core": {
"version": "1.63.0",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz",
"integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==",
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/prebuild-install": { "node_modules/prebuild-install": {
"version": "7.1.3", "version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
package.json
+2 -1
View File
@@ -15,7 +15,8 @@
"express": "^4.19.2", "express": "^4.19.2",
"express-basic-auth": "^1.2.1", "express-basic-auth": "^1.2.1",
"node-cron": "^3.0.3", "node-cron": "^3.0.3",
"nodemailer": "^6.9.14" "nodemailer": "^6.9.14",
"playwright": "^1.63.0"
}, },
"devDependencies": { "devDependencies": {
"supertest": "^7.0.0" "supertest": "^7.0.0"
src/browserScraper.js
+25
View File
@@ -0,0 +1,25 @@
const { chromium } = require('playwright');
const { extractPrice } = require('./scraper');
async function launchBrowser() {
return chromium.launch({ headless: true });
}
async function fetchAndExtractPriceViaBrowser(browser, url, { timeoutMs = 20000 } = {}) {
let page;
try {
page = await browser.newPage();
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
const html = await page.content();
const { price, method } = extractPrice(html);
return { price, method, error: null };
} catch (err) {
return { price: null, method: null, error: err };
} finally {
if (page) {
await page.close().catch(() => {});
}
}
}
module.exports = { launchBrowser, fetchAndExtractPriceViaBrowser };
test/browserScraper.test.js
+70
View File
@@ -0,0 +1,70 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { fetchAndExtractPriceViaBrowser } = require('../src/browserScraper');
test('fetchAndExtractPriceViaBrowser extracts a price from the rendered page', async () => {
let closed = false;
const html = '<html><head><meta property="og:price:amount" content="42.00" /></head><body></body></html>';
const page = {
goto: async () => {},
content: async () => html,
close: async () => { closed = true; },
};
const browser = { newPage: async () => page };
const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
assert.deepEqual(result, { price: 42, method: 'meta', error: null });
assert.equal(closed, true);
});
test('fetchAndExtractPriceViaBrowser returns an error when navigation fails, and still closes the page', async () => {
let closed = false;
const page = {
goto: async () => { throw new Error('Timeout 20000ms exceeded'); },
content: async () => '',
close: async () => { closed = true; },
};
const browser = { newPage: async () => page };
const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
assert.equal(result.price, null);
assert.equal(result.method, null);
assert.ok(result.error instanceof Error);
assert.equal(closed, true);
});
test('fetchAndExtractPriceViaBrowser returns null price when nothing matches, and still closes the page', async () => {
let closed = false;
const page = {
goto: async () => {},
content: async () => '<html><body><p>Geen prijs</p></body></html>',
close: async () => { closed = true; },
};
const browser = { newPage: async () => page };
const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
assert.deepEqual(result, { price: null, method: null, error: null });
assert.equal(closed, true);
});
test('fetchAndExtractPriceViaBrowser closes the page even if reading its content throws', async () => {
let closed = false;
const page = {
goto: async () => {},
content: async () => { throw new Error('content failed'); },
close: async () => { closed = true; },
};
const browser = { newPage: async () => page };
const result = await fetchAndExtractPriceViaBrowser(browser, 'https://example.com');
assert.ok(result.error instanceof Error);
assert.equal(closed, true);
});
test('fetchAndExtractPriceViaBrowser passes the timeout option through to page.goto', async () => {
let receivedOptions;
const page = {
goto: async (url, options) => { receivedOptions = options; },
content: async () => '<html><body></body></html>',
close: async () => {},
};
const browser = { newPage: async () => page };
await fetchAndExtractPriceViaBrowser(browser, 'https://example.com', { timeoutMs: 5000 });
assert.equal(receivedOptions.timeout, 5000);
});