452 lines
18 KiB
JavaScript
452 lines
18 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import test from 'node:test';
|
|
|
|
import {
|
|
crawlWiki,
|
|
extractPage,
|
|
normalizeWikiUrl,
|
|
routeToCapturePath,
|
|
} from '../scripts/lib/live-wiki.mjs';
|
|
import { runCapture } from '../scripts/capture-live-wiki.mjs';
|
|
|
|
test('normalizes only same-origin content routes', () => {
|
|
assert.equal(
|
|
normalizeWikiUrl('/useful-guides/description?draft=1#details'),
|
|
'https://wiki.msvincognito.nl/useful-guides/description',
|
|
);
|
|
assert.equal(
|
|
normalizeWikiUrl('https://wiki.msvincognito.nl/useful-guides/description/'),
|
|
'https://wiki.msvincognito.nl/useful-guides/description',
|
|
);
|
|
assert.equal(normalizeWikiUrl('https://example.com/page'), null);
|
|
assert.equal(normalizeWikiUrl('/assets/diagram.png'), null);
|
|
assert.equal(normalizeWikiUrl('mailto:board@msvincognito.nl'), null);
|
|
});
|
|
|
|
test('maps routes to readable Markdown files', () => {
|
|
assert.equal(
|
|
routeToCapturePath('https://wiki.msvincognito.nl/useful-guides/description'),
|
|
'useful-guides/description.md',
|
|
);
|
|
assert.equal(routeToCapturePath('https://wiki.msvincognito.nl/'), 'index.md');
|
|
assert.throws(
|
|
() => routeToCapturePath('https://wiki.msvincognito.nl/foo%20bar'),
|
|
/Unsafe live-wiki route segment: foo%20bar/,
|
|
);
|
|
});
|
|
|
|
test('extracts the main article and internal links', () => {
|
|
const html = `
|
|
<html>
|
|
<body>
|
|
<nav><a href="/ignored">Repeated navigation</a></nav>
|
|
<main>
|
|
<h1>Guide</h1>
|
|
<p>Text<br>Second line</p>
|
|
<a href="/next?from=guide#part">Next</a>
|
|
<a href="https://example.com/external">External</a>
|
|
<form><input name="secret"><button>Submit</button></form>
|
|
<script>throw new Error('not content')</script>
|
|
</main>
|
|
<footer>Repeated footer</footer>
|
|
</body>
|
|
</html>
|
|
`;
|
|
|
|
const page = extractPage(html, 'https://wiki.msvincognito.nl/guide');
|
|
|
|
assert.equal(page.title, 'Guide');
|
|
assert.match(page.markdown, /^# Guide/m);
|
|
assert.match(page.markdown, /Text/);
|
|
assert.doesNotMatch(page.markdown, /[ \t]+$/m);
|
|
assert.doesNotMatch(page.markdown, /Submit|not content|Repeated footer/);
|
|
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/next']);
|
|
});
|
|
|
|
test('falls back from main to article and then body', () => {
|
|
const articlePage = extractPage(
|
|
'<article><h2>Article title</h2><p>Article text</p></article>',
|
|
'https://wiki.msvincognito.nl/article',
|
|
);
|
|
const bodyPage = extractPage(
|
|
'<html><head><title>Body title</title></head><body><header class="sticky top-0 z-40">Site chrome</header><p>Body text</p></body></html>',
|
|
'https://wiki.msvincognito.nl/body',
|
|
);
|
|
|
|
assert.equal(articlePage.title, 'Article title');
|
|
assert.match(articlePage.markdown, /Article text/);
|
|
assert.equal(bodyPage.title, 'Body title');
|
|
assert.match(bodyPage.markdown, /Body text/);
|
|
assert.doesNotMatch(bodyPage.markdown, /Site chrome/);
|
|
});
|
|
|
|
test('removes the live wiki previous-next panel and repeated page footer', () => {
|
|
const page = extractPage(`
|
|
<main>
|
|
<div>
|
|
<h1>Course</h1>
|
|
<p>Useful article content.</p>
|
|
<div class="mt-16"><div class="border-t pt-6 lg:flex"><a href="/previous">Previous</a></div></div>
|
|
</div>
|
|
<div class="hidden text-sm lg:block">
|
|
<a href="https://github.com/example">Star on GitHub</a>
|
|
</div>
|
|
<footer class="text-muted-foreground py-6 md:px-8 md:py-0">
|
|
<p>Copyright © 2025 MSV Incognito</p>
|
|
</footer>
|
|
</main>
|
|
`, 'https://wiki.msvincognito.nl/course');
|
|
|
|
assert.match(page.markdown, /Useful article content/);
|
|
assert.doesNotMatch(page.markdown, /Previous|Star on GitHub|Copyright/);
|
|
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/previous']);
|
|
});
|
|
|
|
test('preserves legitimate article headers, asides, footers, and spaced sections', () => {
|
|
const page = extractPage(`
|
|
<main>
|
|
<article>
|
|
<header><h1>Research note</h1><p>By the board</p></header>
|
|
<aside>Important contextual aside.</aside>
|
|
<div class="mt-16">A deliberately spaced substantive section.</div>
|
|
<footer>Article footnote.</footer>
|
|
</article>
|
|
</main>
|
|
`, 'https://wiki.msvincognito.nl/research-note');
|
|
|
|
assert.match(page.markdown, /# Research note/);
|
|
assert.match(page.markdown, /By the board/);
|
|
assert.match(page.markdown, /Important contextual aside/);
|
|
assert.match(page.markdown, /deliberately spaced substantive section/);
|
|
assert.match(page.markdown, /Article footnote/);
|
|
});
|
|
|
|
test('crawls in sorted order, writes successful pages, and records failures', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-crawl-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const manifestPath = path.join(captureRoot, 'manifest.json');
|
|
const requests = [];
|
|
const waits = [];
|
|
const responses = new Map([
|
|
[
|
|
'https://wiki.msvincognito.nl/',
|
|
response('<main><h1>Home</h1><a href="/z-last">Z</a><a href="/a-first">A</a></main>'),
|
|
],
|
|
[
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
response('<main><h1>First</h1><p>Captured text.</p></main>'),
|
|
],
|
|
['https://wiki.msvincognito.nl/z-last', response('Unavailable', 503)],
|
|
]);
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath,
|
|
outputRoot,
|
|
fetchImpl: async (url) => {
|
|
requests.push(url);
|
|
return responses.get(url);
|
|
},
|
|
waitImpl: async (milliseconds) => waits.push(milliseconds),
|
|
});
|
|
|
|
assert.deepEqual(requests, [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
'https://wiki.msvincognito.nl/z-last',
|
|
]);
|
|
assert.deepEqual(waits, [150, 150]);
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
]);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/z-last',
|
|
error: 'HTTP 503 Service Unavailable',
|
|
}]);
|
|
|
|
const captured = await readFile(path.join(outputRoot, 'a-first.md'), 'utf8');
|
|
assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/a-first\n> Captured: 2026-08-02\n\n/);
|
|
assert.match(captured, /# First\n\nCaptured text\./);
|
|
assert.equal(
|
|
manifest.pages[1].contentHash,
|
|
createHash('sha256').update(captured).digest('hex'),
|
|
);
|
|
|
|
const savedManifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
assert.deepEqual(savedManifest, manifest);
|
|
});
|
|
|
|
test('follows same-origin redirects manually, paces every hop, and captures the final URL', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const requests = [];
|
|
const waits = [];
|
|
const responses = new Map([
|
|
['https://wiki.msvincognito.nl/', redirectResponse('/guide')],
|
|
['https://wiki.msvincognito.nl/guide', response('<main><h1>Guide</h1><p>Final content.</p></main>')],
|
|
]);
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot,
|
|
fetchImpl: async (url, options) => {
|
|
requests.push({ url, redirect: options.redirect });
|
|
return responses.get(url);
|
|
},
|
|
waitImpl: async (milliseconds) => waits.push(milliseconds),
|
|
});
|
|
|
|
assert.deepEqual(requests, [
|
|
{ url: 'https://wiki.msvincognito.nl/', redirect: 'manual' },
|
|
{ url: 'https://wiki.msvincognito.nl/guide', redirect: 'manual' },
|
|
]);
|
|
assert.deepEqual(waits, [150]);
|
|
assert.deepEqual(manifest.pages.map(({ url, path: capturePath }) => ({ url, path: capturePath })), [{
|
|
url: 'https://wiki.msvincognito.nl/guide',
|
|
path: 'live-wiki/2026-08-02/guide.md',
|
|
}]);
|
|
const captured = await readFile(path.join(outputRoot, 'guide.md'), 'utf8');
|
|
assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/guide$/m);
|
|
});
|
|
|
|
test('rejects cross-origin redirects without requesting the target', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-cross-origin-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const requests = [];
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
|
|
fetchImpl: async (url) => {
|
|
requests.push(url);
|
|
return redirectResponse('https://example.com/private');
|
|
},
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(requests, ['https://wiki.msvincognito.nl/']);
|
|
assert.deepEqual(manifest.pages, []);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/',
|
|
error: 'Redirect target is not an authorized content URL: https://example.com/private',
|
|
}]);
|
|
});
|
|
|
|
test('stops a same-origin redirect chain at the configured hop limit', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-limit-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const requests = [];
|
|
const waits = [];
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
maxRedirects: 1,
|
|
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
|
|
fetchImpl: async (url) => {
|
|
requests.push(url);
|
|
return url.endsWith('/one')
|
|
? redirectResponse('/two')
|
|
: redirectResponse('/one');
|
|
},
|
|
waitImpl: async (milliseconds) => waits.push(milliseconds),
|
|
});
|
|
|
|
assert.deepEqual(requests, [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/one',
|
|
]);
|
|
assert.deepEqual(waits, [150]);
|
|
assert.deepEqual(manifest.pages, []);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/',
|
|
error: 'Redirect limit of 1 exceeded at https://wiki.msvincognito.nl/two',
|
|
}]);
|
|
});
|
|
|
|
test('captures a redirected final URL only once when its canonical route is also queued', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-canonical-redirect-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const requests = [];
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
|
|
fetchImpl: async (url) => {
|
|
requests.push(url);
|
|
if (url.endsWith('/alias')) return redirectResponse('/target');
|
|
if (url.endsWith('/target')) return response('<main><h1>Target</h1></main>');
|
|
return response('<main><h1>Home</h1><a href="/alias">Alias</a><a href="/target">Target</a></main>');
|
|
},
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(requests, [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/alias',
|
|
'https://wiki.msvincognito.nl/target',
|
|
]);
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/target',
|
|
]);
|
|
});
|
|
|
|
test('records an unsafe route as a per-URL failure and preserves other pages', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-unsafe-path-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot,
|
|
fetchImpl: async (url) => url.endsWith('/foo%20bar')
|
|
? response('<main><h1>Unsafe</h1></main>')
|
|
: response('<main><h1>Home</h1><a href="/foo%20bar">Unsafe route</a></main>'),
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/foo%20bar',
|
|
error: 'Unsafe live-wiki route segment: foo%20bar',
|
|
}]);
|
|
assert.match(await readFile(path.join(outputRoot, 'index.md'), 'utf8'), /# Home/);
|
|
});
|
|
|
|
test('records destination collisions before a later page can overwrite a capture', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-collision-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot,
|
|
fetchImpl: async (url) => url.endsWith('/index')
|
|
? response('<main><h1>Colliding page</h1></main>')
|
|
: response('<main><h1>Home</h1><a href="/index">Collision</a></main>'),
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/index',
|
|
error: 'Capture destination collision: https://wiki.msvincognito.nl/index and https://wiki.msvincognito.nl/ both map to index.md',
|
|
}]);
|
|
const captured = await readFile(path.join(outputRoot, 'index.md'), 'utf8');
|
|
assert.match(captured, /# Home/);
|
|
assert.doesNotMatch(captured, /Colliding page/);
|
|
});
|
|
|
|
test('rejects discovery beyond the configured page ceiling', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-limit-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
|
|
await assert.rejects(
|
|
crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
|
|
maxPages: 2,
|
|
fetchImpl: async () => response(
|
|
'<main><h1>Home</h1><a href="/one">One</a><a href="/two">Two</a></main>',
|
|
),
|
|
waitImpl: async () => {},
|
|
}),
|
|
/more than 2 content pages/i,
|
|
);
|
|
});
|
|
|
|
test('excludes soft 404 navigation placeholders and removes stale captures', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-soft-404-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const stalePath = path.join(outputRoot, 'missing.md');
|
|
const unrelatedStalePath = path.join(outputRoot, 'obsolete.md');
|
|
await mkdir(outputRoot, { recursive: true });
|
|
await writeFile(stalePath, 'stale capture', 'utf8');
|
|
await writeFile(unrelatedStalePath, 'obsolete capture', 'utf8');
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot,
|
|
fetchImpl: async (url) => url.endsWith('/missing')
|
|
? response('<main><h3>404</h3><p>This page could not be found.</p></main>')
|
|
: response('<main><h1>Home</h1><a href="/missing">Missing category</a></main>'),
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']);
|
|
assert.deepEqual(manifest.failures, []);
|
|
await assert.rejects(readFile(stalePath, 'utf8'), { code: 'ENOENT' });
|
|
await assert.rejects(readFile(unrelatedStalePath, 'utf8'), { code: 'ENOENT' });
|
|
});
|
|
|
|
test('leaves the previous snapshot and manifest untouched after a fatal crawl error', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-atomic-failure-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const manifestPath = path.join(captureRoot, 'manifest.json');
|
|
await mkdir(outputRoot, { recursive: true });
|
|
await writeFile(path.join(outputRoot, 'previous.md'), 'previous snapshot', 'utf8');
|
|
await writeFile(manifestPath, '{"snapshot":"previous"}\n', 'utf8');
|
|
|
|
await assert.rejects(crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath,
|
|
maxPages: 2,
|
|
outputRoot,
|
|
fetchImpl: async (url) => url.endsWith('/one')
|
|
? response('<main><h1>One</h1><a href="/two">Two</a></main>')
|
|
: response('<main><h1>Home</h1><a href="/one">One</a></main>'),
|
|
waitImpl: async () => {},
|
|
}), /more than 2 content pages/i);
|
|
|
|
assert.deepEqual(await readdir(outputRoot), ['previous.md']);
|
|
assert.equal(await readFile(path.join(outputRoot, 'previous.md'), 'utf8'), 'previous snapshot');
|
|
assert.equal(await readFile(manifestPath, 'utf8'), '{"snapshot":"previous"}\n');
|
|
});
|
|
|
|
test('capture command reports failure after the crawler preserves its manifest', async () => {
|
|
const messages = [];
|
|
const exitCode = await runCapture({
|
|
crawlImpl: async () => ({
|
|
pages: [{ url: 'https://wiki.msvincognito.nl/' }],
|
|
failures: [{ url: 'https://wiki.msvincognito.nl/broken', error: 'HTTP 503' }],
|
|
}),
|
|
logError: (message) => messages.push(message),
|
|
logInfo: () => {},
|
|
});
|
|
|
|
assert.equal(exitCode, 1);
|
|
assert.deepEqual(messages, ['Captured 1 page with 1 failure; inspect to-be-studied/manifest.json.']);
|
|
});
|
|
|
|
function response(body, status = 200) {
|
|
return new Response(body, {
|
|
status,
|
|
statusText: status === 200 ? 'OK' : status === 503 ? 'Service Unavailable' : 'Error',
|
|
headers: { 'content-type': 'text/html; charset=utf-8' },
|
|
});
|
|
}
|
|
|
|
function redirectResponse(location, status = 302) {
|
|
return new Response('', {
|
|
status,
|
|
statusText: status === 302 ? 'Found' : 'Redirect',
|
|
headers: { location },
|
|
});
|
|
}
|