import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; import { crawlWiki, extractPage, normalizeWikiUrl, routeToCapturePath, } from '../scripts/lib/live-wiki.mjs'; import { runCapture } from '../scripts/capture-live-wiki.mjs'; test('normalizes only same-origin content routes', () => { assert.equal( normalizeWikiUrl('/useful-guides/description?draft=1#details'), 'https://wiki.msvincognito.nl/useful-guides/description', ); assert.equal( normalizeWikiUrl('https://wiki.msvincognito.nl/useful-guides/description/'), 'https://wiki.msvincognito.nl/useful-guides/description', ); assert.equal(normalizeWikiUrl('https://example.com/page'), null); assert.equal(normalizeWikiUrl('/assets/diagram.png'), null); assert.equal(normalizeWikiUrl('mailto:board@msvincognito.nl'), null); }); test('maps routes to readable Markdown files', () => { assert.equal( routeToCapturePath('https://wiki.msvincognito.nl/useful-guides/description'), 'useful-guides/description.md', ); assert.equal(routeToCapturePath('https://wiki.msvincognito.nl/'), 'index.md'); assert.throws( () => routeToCapturePath('https://wiki.msvincognito.nl/foo%20bar'), /Unsafe live-wiki route segment: foo%20bar/, ); }); test('extracts the main article and internal links', () => { const html = `

Guide

Text
Second line

Next External
`; const page = extractPage(html, 'https://wiki.msvincognito.nl/guide'); assert.equal(page.title, 'Guide'); assert.match(page.markdown, /^# Guide/m); assert.match(page.markdown, /Text/); assert.doesNotMatch(page.markdown, /[ \t]+$/m); assert.doesNotMatch(page.markdown, /Submit|not content|Repeated footer/); assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/next']); }); test('falls back from main to article and then body', () => { const articlePage = extractPage( '

Article title

Article text

', 'https://wiki.msvincognito.nl/article', ); const bodyPage = extractPage( 'Body title
Site chrome

Body text

', 'https://wiki.msvincognito.nl/body', ); assert.equal(articlePage.title, 'Article title'); assert.match(articlePage.markdown, /Article text/); assert.equal(bodyPage.title, 'Body title'); assert.match(bodyPage.markdown, /Body text/); assert.doesNotMatch(bodyPage.markdown, /Site chrome/); }); test('removes the live wiki previous-next panel and repeated page footer', () => { const page = extractPage(`

Course

Useful article content.

Previous
`, 'https://wiki.msvincognito.nl/course'); assert.match(page.markdown, /Useful article content/); assert.doesNotMatch(page.markdown, /Previous|Star on GitHub|Copyright/); assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/previous']); }); test('preserves legitimate article headers, asides, footers, and spaced sections', () => { const page = extractPage(`

Research note

By the board

A deliberately spaced substantive section.
`, 'https://wiki.msvincognito.nl/research-note'); assert.match(page.markdown, /# Research note/); assert.match(page.markdown, /By the board/); assert.match(page.markdown, /Important contextual aside/); assert.match(page.markdown, /deliberately spaced substantive section/); assert.match(page.markdown, /Article footnote/); }); test('crawls in sorted order, writes successful pages, and records failures', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-crawl-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const manifestPath = path.join(captureRoot, 'manifest.json'); const requests = []; const waits = []; const responses = new Map([ [ 'https://wiki.msvincognito.nl/', response('

Home

ZA
'), ], [ 'https://wiki.msvincognito.nl/a-first', response('

First

Captured text.

'), ], ['https://wiki.msvincognito.nl/z-last', response('Unavailable', 503)], ]); const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath, outputRoot, fetchImpl: async (url) => { requests.push(url); return responses.get(url); }, waitImpl: async (milliseconds) => waits.push(milliseconds), }); assert.deepEqual(requests, [ 'https://wiki.msvincognito.nl/', 'https://wiki.msvincognito.nl/a-first', 'https://wiki.msvincognito.nl/z-last', ]); assert.deepEqual(waits, [150, 150]); assert.deepEqual(manifest.pages.map(({ url }) => url), [ 'https://wiki.msvincognito.nl/', 'https://wiki.msvincognito.nl/a-first', ]); assert.deepEqual(manifest.failures, [{ url: 'https://wiki.msvincognito.nl/z-last', error: 'HTTP 503 Service Unavailable', }]); const captured = await readFile(path.join(outputRoot, 'a-first.md'), 'utf8'); assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/a-first\n> Captured: 2026-08-02\n\n/); assert.match(captured, /# First\n\nCaptured text\./); assert.equal( manifest.pages[1].contentHash, createHash('sha256').update(captured).digest('hex'), ); const savedManifest = JSON.parse(await readFile(manifestPath, 'utf8')); assert.deepEqual(savedManifest, manifest); }); test('follows same-origin redirects manually, paces every hop, and captures the final URL', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const requests = []; const waits = []; const responses = new Map([ ['https://wiki.msvincognito.nl/', redirectResponse('/guide')], ['https://wiki.msvincognito.nl/guide', response('

Guide

Final content.

')], ]); const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot, fetchImpl: async (url, options) => { requests.push({ url, redirect: options.redirect }); return responses.get(url); }, waitImpl: async (milliseconds) => waits.push(milliseconds), }); assert.deepEqual(requests, [ { url: 'https://wiki.msvincognito.nl/', redirect: 'manual' }, { url: 'https://wiki.msvincognito.nl/guide', redirect: 'manual' }, ]); assert.deepEqual(waits, [150]); assert.deepEqual(manifest.pages.map(({ url, path: capturePath }) => ({ url, path: capturePath })), [{ url: 'https://wiki.msvincognito.nl/guide', path: 'live-wiki/2026-08-02/guide.md', }]); const captured = await readFile(path.join(outputRoot, 'guide.md'), 'utf8'); assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/guide$/m); }); test('rejects cross-origin redirects without requesting the target', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-cross-origin-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const requests = []; const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), fetchImpl: async (url) => { requests.push(url); return redirectResponse('https://example.com/private'); }, waitImpl: async () => {}, }); assert.deepEqual(requests, ['https://wiki.msvincognito.nl/']); assert.deepEqual(manifest.pages, []); assert.deepEqual(manifest.failures, [{ url: 'https://wiki.msvincognito.nl/', error: 'Redirect target is not an authorized content URL: https://example.com/private', }]); }); test('stops a same-origin redirect chain at the configured hop limit', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-limit-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const requests = []; const waits = []; const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), maxRedirects: 1, outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), fetchImpl: async (url) => { requests.push(url); return url.endsWith('/one') ? redirectResponse('/two') : redirectResponse('/one'); }, waitImpl: async (milliseconds) => waits.push(milliseconds), }); assert.deepEqual(requests, [ 'https://wiki.msvincognito.nl/', 'https://wiki.msvincognito.nl/one', ]); assert.deepEqual(waits, [150]); assert.deepEqual(manifest.pages, []); assert.deepEqual(manifest.failures, [{ url: 'https://wiki.msvincognito.nl/', error: 'Redirect limit of 1 exceeded at https://wiki.msvincognito.nl/two', }]); }); test('captures a redirected final URL only once when its canonical route is also queued', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-canonical-redirect-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const requests = []; const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), fetchImpl: async (url) => { requests.push(url); if (url.endsWith('/alias')) return redirectResponse('/target'); if (url.endsWith('/target')) return response('

Target

'); return response('

Home

AliasTarget
'); }, waitImpl: async () => {}, }); assert.deepEqual(requests, [ 'https://wiki.msvincognito.nl/', 'https://wiki.msvincognito.nl/alias', 'https://wiki.msvincognito.nl/target', ]); assert.deepEqual(manifest.pages.map(({ url }) => url), [ 'https://wiki.msvincognito.nl/', 'https://wiki.msvincognito.nl/target', ]); }); test('records an unsafe route as a per-URL failure and preserves other pages', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-unsafe-path-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot, fetchImpl: async (url) => url.endsWith('/foo%20bar') ? response('

Unsafe

') : response('

Home

Unsafe route
'), waitImpl: async () => {}, }); assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); assert.deepEqual(manifest.failures, [{ url: 'https://wiki.msvincognito.nl/foo%20bar', error: 'Unsafe live-wiki route segment: foo%20bar', }]); assert.match(await readFile(path.join(outputRoot, 'index.md'), 'utf8'), /# Home/); }); test('records destination collisions before a later page can overwrite a capture', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-collision-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot, fetchImpl: async (url) => url.endsWith('/index') ? response('

Colliding page

') : response('

Home

Collision
'), waitImpl: async () => {}, }); assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); assert.deepEqual(manifest.failures, [{ url: 'https://wiki.msvincognito.nl/index', error: 'Capture destination collision: https://wiki.msvincognito.nl/index and https://wiki.msvincognito.nl/ both map to index.md', }]); const captured = await readFile(path.join(outputRoot, 'index.md'), 'utf8'); assert.match(captured, /# Home/); assert.doesNotMatch(captured, /Colliding page/); }); test('rejects discovery beyond the configured page ceiling', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-limit-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); await assert.rejects( crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), maxPages: 2, fetchImpl: async () => response( '

Home

OneTwo
', ), waitImpl: async () => {}, }), /more than 2 content pages/i, ); }); test('excludes soft 404 navigation placeholders and removes stale captures', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-soft-404-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const stalePath = path.join(outputRoot, 'missing.md'); const unrelatedStalePath = path.join(outputRoot, 'obsolete.md'); await mkdir(outputRoot, { recursive: true }); await writeFile(stalePath, 'stale capture', 'utf8'); await writeFile(unrelatedStalePath, 'obsolete capture', 'utf8'); const manifest = await crawlWiki({ capturedAt: '2026-08-02', manifestPath: path.join(captureRoot, 'manifest.json'), outputRoot, fetchImpl: async (url) => url.endsWith('/missing') ? response('

404

This page could not be found.

') : response('

Home

Missing category
'), waitImpl: async () => {}, }); assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); assert.deepEqual(manifest.failures, []); await assert.rejects(readFile(stalePath, 'utf8'), { code: 'ENOENT' }); await assert.rejects(readFile(unrelatedStalePath, 'utf8'), { code: 'ENOENT' }); }); test('leaves the previous snapshot and manifest untouched after a fatal crawl error', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-atomic-failure-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const manifestPath = path.join(captureRoot, 'manifest.json'); await mkdir(outputRoot, { recursive: true }); await writeFile(path.join(outputRoot, 'previous.md'), 'previous snapshot', 'utf8'); await writeFile(manifestPath, '{"snapshot":"previous"}\n', 'utf8'); await assert.rejects(crawlWiki({ capturedAt: '2026-08-02', manifestPath, maxPages: 2, outputRoot, fetchImpl: async (url) => url.endsWith('/one') ? response('

One

Two
') : response('

Home

One
'), waitImpl: async () => {}, }), /more than 2 content pages/i); assert.deepEqual(await readdir(outputRoot), ['previous.md']); assert.equal(await readFile(path.join(outputRoot, 'previous.md'), 'utf8'), 'previous snapshot'); assert.equal(await readFile(manifestPath, 'utf8'), '{"snapshot":"previous"}\n'); }); test('capture command reports failure after the crawler preserves its manifest', async () => { const messages = []; const exitCode = await runCapture({ crawlImpl: async () => ({ pages: [{ url: 'https://wiki.msvincognito.nl/' }], failures: [{ url: 'https://wiki.msvincognito.nl/broken', error: 'HTTP 503' }], }), logError: (message) => messages.push(message), logInfo: () => {}, }); assert.equal(exitCode, 1); assert.deepEqual(messages, ['Captured 1 page with 1 failure; inspect to-be-studied/manifest.json.']); }); function response(body, status = 200) { return new Response(body, { status, statusText: status === 200 ? 'OK' : status === 503 ? 'Service Unavailable' : 'Error', headers: { 'content-type': 'text/html; charset=utf-8' }, }); } function redirectResponse(location, status = 302) { return new Response('', { status, statusText: status === 302 ? 'Found' : 'Redirect', headers: { location }, }); }