222 lines
8 KiB
JavaScript
222 lines
8 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import test from 'node:test';
|
|
|
|
import {
|
|
crawlWiki,
|
|
extractPage,
|
|
normalizeWikiUrl,
|
|
routeToCapturePath,
|
|
} from '../scripts/lib/live-wiki.mjs';
|
|
import { runCapture } from '../scripts/capture-live-wiki.mjs';
|
|
|
|
test('normalizes only same-origin content routes', () => {
|
|
assert.equal(
|
|
normalizeWikiUrl('/useful-guides/description?draft=1#details'),
|
|
'https://wiki.msvincognito.nl/useful-guides/description',
|
|
);
|
|
assert.equal(
|
|
normalizeWikiUrl('https://wiki.msvincognito.nl/useful-guides/description/'),
|
|
'https://wiki.msvincognito.nl/useful-guides/description',
|
|
);
|
|
assert.equal(normalizeWikiUrl('https://example.com/page'), null);
|
|
assert.equal(normalizeWikiUrl('/assets/diagram.png'), null);
|
|
assert.equal(normalizeWikiUrl('mailto:board@msvincognito.nl'), null);
|
|
});
|
|
|
|
test('maps routes to readable Markdown files', () => {
|
|
assert.equal(
|
|
routeToCapturePath('https://wiki.msvincognito.nl/useful-guides/description'),
|
|
'useful-guides/description.md',
|
|
);
|
|
assert.equal(routeToCapturePath('https://wiki.msvincognito.nl/'), 'index.md');
|
|
});
|
|
|
|
test('extracts the main article and internal links', () => {
|
|
const html = `
|
|
<html>
|
|
<body>
|
|
<nav><a href="/ignored">Repeated navigation</a></nav>
|
|
<main>
|
|
<h1>Guide</h1>
|
|
<p>Text<br>Second line</p>
|
|
<a href="/next?from=guide#part">Next</a>
|
|
<a href="https://example.com/external">External</a>
|
|
<form><input name="secret"><button>Submit</button></form>
|
|
<script>throw new Error('not content')</script>
|
|
</main>
|
|
<footer>Repeated footer</footer>
|
|
</body>
|
|
</html>
|
|
`;
|
|
|
|
const page = extractPage(html, 'https://wiki.msvincognito.nl/guide');
|
|
|
|
assert.equal(page.title, 'Guide');
|
|
assert.match(page.markdown, /^# Guide/m);
|
|
assert.match(page.markdown, /Text/);
|
|
assert.doesNotMatch(page.markdown, /[ \t]+$/m);
|
|
assert.doesNotMatch(page.markdown, /Submit|not content|Repeated footer/);
|
|
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/next']);
|
|
});
|
|
|
|
test('falls back from main to article and then body', () => {
|
|
const articlePage = extractPage(
|
|
'<article><h2>Article title</h2><p>Article text</p></article>',
|
|
'https://wiki.msvincognito.nl/article',
|
|
);
|
|
const bodyPage = extractPage(
|
|
'<html><head><title>Body title</title></head><body><header>Site chrome</header><p>Body text</p></body></html>',
|
|
'https://wiki.msvincognito.nl/body',
|
|
);
|
|
|
|
assert.equal(articlePage.title, 'Article title');
|
|
assert.match(articlePage.markdown, /Article text/);
|
|
assert.equal(bodyPage.title, 'Body title');
|
|
assert.match(bodyPage.markdown, /Body text/);
|
|
assert.doesNotMatch(bodyPage.markdown, /Site chrome/);
|
|
});
|
|
|
|
test('removes the live wiki previous-next panel and repeated page footer', () => {
|
|
const page = extractPage(`
|
|
<main>
|
|
<div>
|
|
<h1>Course</h1>
|
|
<p>Useful article content.</p>
|
|
<div class="mt-16"><a href="/previous">Previous</a></div>
|
|
</div>
|
|
<div class="hidden text-sm lg:block">
|
|
<a href="https://github.com/example">Star on GitHub</a>
|
|
</div>
|
|
</main>
|
|
`, 'https://wiki.msvincognito.nl/course');
|
|
|
|
assert.match(page.markdown, /Useful article content/);
|
|
assert.doesNotMatch(page.markdown, /Previous|Star on GitHub/);
|
|
assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/previous']);
|
|
});
|
|
|
|
test('crawls in sorted order, writes successful pages, and records failures', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-crawl-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const manifestPath = path.join(captureRoot, 'manifest.json');
|
|
const requests = [];
|
|
const waits = [];
|
|
const responses = new Map([
|
|
[
|
|
'https://wiki.msvincognito.nl/',
|
|
response('<main><h1>Home</h1><a href="/z-last">Z</a><a href="/a-first">A</a></main>'),
|
|
],
|
|
[
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
response('<main><h1>First</h1><p>Captured text.</p></main>'),
|
|
],
|
|
['https://wiki.msvincognito.nl/z-last', response('Unavailable', 503)],
|
|
]);
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath,
|
|
outputRoot,
|
|
fetchImpl: async (url) => {
|
|
requests.push(url);
|
|
return responses.get(url);
|
|
},
|
|
waitImpl: async (milliseconds) => waits.push(milliseconds),
|
|
});
|
|
|
|
assert.deepEqual(requests, [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
'https://wiki.msvincognito.nl/z-last',
|
|
]);
|
|
assert.deepEqual(waits, [150, 150]);
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), [
|
|
'https://wiki.msvincognito.nl/',
|
|
'https://wiki.msvincognito.nl/a-first',
|
|
]);
|
|
assert.deepEqual(manifest.failures, [{
|
|
url: 'https://wiki.msvincognito.nl/z-last',
|
|
error: 'HTTP 503 Service Unavailable',
|
|
}]);
|
|
|
|
const captured = await readFile(path.join(outputRoot, 'a-first.md'), 'utf8');
|
|
assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/a-first\n> Captured: 2026-08-02\n\n/);
|
|
assert.match(captured, /# First\n\nCaptured text\./);
|
|
assert.equal(
|
|
manifest.pages[1].contentHash,
|
|
createHash('sha256').update(captured).digest('hex'),
|
|
);
|
|
|
|
const savedManifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
assert.deepEqual(savedManifest, manifest);
|
|
});
|
|
|
|
test('rejects discovery beyond the configured page ceiling', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-limit-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
|
|
await assert.rejects(
|
|
crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'),
|
|
maxPages: 2,
|
|
fetchImpl: async () => response(
|
|
'<main><h1>Home</h1><a href="/one">One</a><a href="/two">Two</a></main>',
|
|
),
|
|
waitImpl: async () => {},
|
|
}),
|
|
/more than 2 content pages/i,
|
|
);
|
|
});
|
|
|
|
test('excludes soft 404 navigation placeholders and removes stale captures', async (context) => {
|
|
const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-soft-404-'));
|
|
context.after(() => rm(captureRoot, { recursive: true, force: true }));
|
|
const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02');
|
|
const stalePath = path.join(outputRoot, 'missing.md');
|
|
await mkdir(outputRoot, { recursive: true });
|
|
await writeFile(stalePath, 'stale capture', 'utf8');
|
|
|
|
const manifest = await crawlWiki({
|
|
capturedAt: '2026-08-02',
|
|
manifestPath: path.join(captureRoot, 'manifest.json'),
|
|
outputRoot,
|
|
fetchImpl: async (url) => url.endsWith('/missing')
|
|
? response('<main><h3>404</h3><p>This page could not be found.</p></main>')
|
|
: response('<main><h1>Home</h1><a href="/missing">Missing category</a></main>'),
|
|
waitImpl: async () => {},
|
|
});
|
|
|
|
assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']);
|
|
assert.deepEqual(manifest.failures, []);
|
|
await assert.rejects(readFile(stalePath, 'utf8'), { code: 'ENOENT' });
|
|
});
|
|
|
|
test('capture command reports failure after the crawler preserves its manifest', async () => {
|
|
const messages = [];
|
|
const exitCode = await runCapture({
|
|
crawlImpl: async () => ({
|
|
pages: [{ url: 'https://wiki.msvincognito.nl/' }],
|
|
failures: [{ url: 'https://wiki.msvincognito.nl/broken', error: 'HTTP 503' }],
|
|
}),
|
|
logError: (message) => messages.push(message),
|
|
logInfo: () => {},
|
|
});
|
|
|
|
assert.equal(exitCode, 1);
|
|
assert.deepEqual(messages, ['Captured 1 page with 1 failure; inspect to-be-studied/manifest.json.']);
|
|
});
|
|
|
|
function response(body, status = 200) {
|
|
return new Response(body, {
|
|
status,
|
|
statusText: status === 200 ? 'OK' : status === 503 ? 'Service Unavailable' : 'Error',
|
|
headers: { 'content-type': 'text/html; charset=utf-8' },
|
|
});
|
|
}
|