import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
crawlWiki,
extractPage,
normalizeWikiUrl,
routeToCapturePath,
} from '../scripts/lib/live-wiki.mjs';
import { runCapture } from '../scripts/capture-live-wiki.mjs';
test('normalizes only same-origin content routes', () => {
assert.equal(
normalizeWikiUrl('/useful-guides/description?draft=1#details'),
'https://wiki.msvincognito.nl/useful-guides/description',
);
assert.equal(
normalizeWikiUrl('https://wiki.msvincognito.nl/useful-guides/description/'),
'https://wiki.msvincognito.nl/useful-guides/description',
);
assert.equal(normalizeWikiUrl('https://example.com/page'), null);
assert.equal(normalizeWikiUrl('/assets/diagram.png'), null);
assert.equal(normalizeWikiUrl('mailto:board@msvincognito.nl'), null);
});
test('maps routes to readable Markdown files', () => {
assert.equal(
routeToCapturePath('https://wiki.msvincognito.nl/useful-guides/description'),
'useful-guides/description.md',
);
assert.equal(routeToCapturePath('https://wiki.msvincognito.nl/'), 'index.md');
});
test('extracts the main article and internal links', () => {
const html = `
Guide
Text
Second line
Next
External
`;
const page = extractPage(html, 'https://wiki.msvincognito.nl/guide');
assert.equal(page.title, 'Guide');
assert.match(page.markdown, /^# Guide/m);
assert.match(page.markdown, /Text/);
assert.doesNotMatch(page.markdown, /[ \t]+$/m);
assert.doesNotMatch(page.markdown, /Submit|not content|Repeated footer/);
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/next']);
});
test('falls back from main to article and then body', () => {
const articlePage = extractPage(
'Article title
Article text
',
'https://wiki.msvincognito.nl/article',
);
const bodyPage = extractPage(
'Body titleBody text
',
'https://wiki.msvincognito.nl/body',
);
assert.equal(articlePage.title, 'Article title');
assert.match(articlePage.markdown, /Article text/);
assert.equal(bodyPage.title, 'Body title');
assert.match(bodyPage.markdown, /Body text/);
assert.doesNotMatch(bodyPage.markdown, /Site chrome/);
});
test('removes the live wiki previous-next panel and repeated page footer', () => {
const page = extractPage(`
Course
Useful article content.
`, 'https://wiki.msvincognito.nl/course');
assert.match(page.markdown, /Useful article content/);
assert.doesNotMatch(page.markdown, /Previous|Star on GitHub/);
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/previous']);
});
test('crawls in sorted order, writes successful pages, and records failures', async (context) => {
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-crawl-'));
context.after(() => rm(captureRoot, { recursive: true, force: true }));
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
const manifestPath = path.join(captureRoot, 'manifest.json');
const requests = [];
const waits = [];
const responses = new Map([
[
'https://wiki.msvincognito.nl/',
response('Home
ZA'),
],
[
'https://wiki.msvincognito.nl/a-first',
response('First
Captured text.
'),
],
['https://wiki.msvincognito.nl/z-last', response('Unavailable', 503)],
]);
const manifest = await crawlWiki({
capturedAt: '2026-08-02',
manifestPath,
outputRoot,
fetchImpl: async (url) => {
requests.push(url);
return responses.get(url);
},
waitImpl: async (milliseconds) => waits.push(milliseconds),
});
assert.deepEqual(requests, [
'https://wiki.msvincognito.nl/',
'https://wiki.msvincognito.nl/a-first',
'https://wiki.msvincognito.nl/z-last',
]);
assert.deepEqual(waits, [150, 150]);
assert.deepEqual(manifest.pages.map(({ url }) => url), [
'https://wiki.msvincognito.nl/',
'https://wiki.msvincognito.nl/a-first',
]);
assert.deepEqual(manifest.failures, [{
url: 'https://wiki.msvincognito.nl/z-last',
error: 'HTTP 503 Service Unavailable',
}]);
const captured = await readFile(path.join(outputRoot, 'a-first.md'), 'utf8');
assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/a-first\n> Captured: 2026-08-02\n\n/);
assert.match(captured, /# First\n\nCaptured text\./);
assert.equal(
manifest.pages[1].contentHash,
createHash('sha256').update(captured).digest('hex'),
);
const savedManifest = JSON.parse(await readFile(manifestPath, 'utf8'));
assert.deepEqual(savedManifest, manifest);
});
test('rejects discovery beyond the configured page ceiling', async (context) => {
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-limit-'));
context.after(() => rm(captureRoot, { recursive: true, force: true }));
await assert.rejects(
crawlWiki({
capturedAt: '2026-08-02',
manifestPath: path.join(captureRoot, 'manifest.json'),
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
maxPages: 2,
fetchImpl: async () => response(
'Home
OneTwo',
),
waitImpl: async () => {},
}),
/more than 2 content pages/i,
);
});
test('excludes soft 404 navigation placeholders and removes stale captures', async (context) => {
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-soft-404-'));
context.after(() => rm(captureRoot, { recursive: true, force: true }));
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
const stalePath = path.join(outputRoot, 'missing.md');
await mkdir(outputRoot, { recursive: true });
await writeFile(stalePath, 'stale capture', 'utf8');
const manifest = await crawlWiki({
capturedAt: '2026-08-02',
manifestPath: path.join(captureRoot, 'manifest.json'),
outputRoot,
fetchImpl: async (url) => url.endsWith('/missing')
? response('404
This page could not be found.
')
: response('Home
Missing category'),
waitImpl: async () => {},
});
assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']);
assert.deepEqual(manifest.failures, []);
await assert.rejects(readFile(stalePath, 'utf8'), { code: 'ENOENT' });
});
test('capture command reports failure after the crawler preserves its manifest', async () => {
const messages = [];
const exitCode = await runCapture({
crawlImpl: async () => ({
pages: [{ url: 'https://wiki.msvincognito.nl/' }],
failures: [{ url: 'https://wiki.msvincognito.nl/broken', error: 'HTTP 503' }],
}),
logError: (message) => messages.push(message),
logInfo: () => {},
});
assert.equal(exitCode, 1);
assert.deepEqual(messages, ['Captured 1 page with 1 failure; inspect to-be-studied/manifest.json.']);
});
function response(body, status = 200) {
return new Response(body, {
status,
statusText: status === 200 ? 'OK' : status === 503 ? 'Service Unavailable' : 'Error',
headers: { 'content-type': 'text/html; charset=utf-8' },
});
}