From ebaa5eff9262894d8ca3707d448b1b6c338bc879 Mon Sep 17 00:00:00 2001 From: msa46 Date: Sun, 2 Aug 2026 17:48:35 +0200 Subject: [PATCH] fix: harden live wiki capture boundaries --- scripts/lib/live-wiki.mjs | 340 +++++++++++++----- tests/live-wiki.test.mjs | 238 +++++++++++- to-be-studied/README.md | 2 +- to-be-studied/comparison.md | 14 +- .../2026-08-02/artificial-intelligence.md | 8 - .../live-wiki/2026-08-02/computer-science.md | 8 - ...ata-science-and-artificial-intelligence.md | 8 - .../data-science-for-decision-making.md | 8 - .../live-wiki/2026-08-02/useful-guides.md | 50 --- to-be-studied/manifest.json | 30 -- 10 files changed, 490 insertions(+), 216 deletions(-) delete mode 100644 to-be-studied/live-wiki/2026-08-02/artificial-intelligence.md delete mode 100644 to-be-studied/live-wiki/2026-08-02/computer-science.md delete mode 100644 to-be-studied/live-wiki/2026-08-02/data-science-and-artificial-intelligence.md delete mode 100644 to-be-studied/live-wiki/2026-08-02/data-science-for-decision-making.md delete mode 100644 to-be-studied/live-wiki/2026-08-02/useful-guides.md diff --git a/scripts/lib/live-wiki.mjs b/scripts/lib/live-wiki.mjs index cf1e1c4..46543ee 100644 --- a/scripts/lib/live-wiki.mjs +++ b/scripts/lib/live-wiki.mjs @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, mkdtemp, rename, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { parseHTML } from 'linkedom'; @@ -11,12 +11,9 @@ const ASSET_EXTENSION = /\.(?:avif|bmp|css|csv|docx?|eot|gif|ico|jpe?g|js|json|m const NON_CONTENT_SELECTOR = [ 'script', 'style', - 'header', 'nav', - 'aside', 'form', 'button', - 'footer', 'noscript', 'iframe', 'dialog', @@ -70,7 +67,7 @@ export function routeToCapturePath(input) { const route = pathname .split('/') .filter(Boolean) - .map((segment) => sanitizePathSegment(segment)) + .map((segment) => validatePathSegment(segment)) .join('/'); return `${route}.md`; @@ -131,6 +128,7 @@ export async function crawlWiki({ fetchImpl = globalThis.fetch, manifestPath, maxPages = 500, + maxRedirects = 5, outputRoot, startUrls = [LIVE_WIKI_BASE_URL], waitImpl = wait, @@ -144,6 +142,9 @@ export async function crawlWiki({ if (!Number.isInteger(maxPages) || maxPages < 1) { throw new TypeError('maxPages must be a positive integer.'); } + if (!Number.isInteger(maxRedirects) || maxRedirects < 0) { + throw new TypeError('maxRedirects must be a non-negative integer.'); + } const queue = []; const discovered = new Set(); @@ -153,87 +154,239 @@ export async function crawlWiki({ const pages = []; const failures = []; + const capturedUrls = new Set(); + const captureDestinations = new Map(); let requestCount = 0; - await mkdir(outputRoot, { recursive: true }); + await mkdir(path.dirname(outputRoot), { recursive: true }); + const stagingRoot = await mkdtemp(path.join( + path.dirname(outputRoot), + `.${path.basename(outputRoot)}-staging-`, + )); + const stagedOutputRoot = path.join(stagingRoot, 'snapshot'); + const stagedManifestPath = path.join(stagingRoot, 'manifest.json'); + await mkdir(stagedOutputRoot, { recursive: true }); - while (queue.length > 0) { - queue.sort(); - const url = queue.shift(); - - if (requestCount > 0) { - await waitImpl(delayMs); - } - requestCount += 1; - const capturePath = routeToCapturePath(url); - const destination = path.join(outputRoot, ...capturePath.split('/')); - - try { - const response = await fetchImpl(url, { - headers: { - accept: 'text/html,application/xhtml+xml', - 'user-agent': 'Incognito-Wiki research capture (+https://msvincognito.nl/)', - }, - redirect: 'follow', - }); - - if (!response) { - throw new Error('No response'); - } - if (!response.ok) { - throw new Error(`HTTP ${response.status} ${response.statusText}`.trim()); - } - - const contentType = response.headers.get('content-type') ?? ''; - if (contentType && !/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) { - throw new Error(`Unexpected content type: ${contentType}`); - } - - const page = extractPage(await response.text(), url); - if (page.title === '404' && /This page could not be found\./i.test(page.markdown)) { - throw new SoftNotFoundError('Soft 404: This page could not be found.'); - } - for (const link of page.links) { - discover(link, url, discovered, queue, maxPages); - } - - const content = `> Source: ${url}\n> Captured: ${capturedAt}\n\n${page.markdown}`; - - await mkdir(path.dirname(destination), { recursive: true }); - await writeFile(destination, content, 'utf8'); - - pages.push({ - url, - title: page.title, - path: toPosixPath(path.relative(path.dirname(manifestPath), destination)), - contentHash: createHash('sha256').update(content).digest('hex'), - }); - } catch (error) { - if (error instanceof PageLimitError) { - throw error; - } - await rm(destination, { force: true }); - if (error instanceof SoftNotFoundError) { + try { + while (queue.length > 0) { + queue.sort(); + const url = queue.shift(); + if (capturedUrls.has(url)) { continue; } - failures.push({ - url, - error: error instanceof Error ? error.message : String(error), - }); + let destination; + let destinationKey; + let destinationOwner; + + try { + const fetched = await fetchWithAuthorizedRedirects(url, { + fetchImpl, + maxRedirects, + pace: async () => { + if (requestCount > 0) { + await waitImpl(delayMs); + } + requestCount += 1; + }, + }); + const { response, url: finalUrl } = fetched; + + if (!response) { + throw new Error('No response'); + } + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}`.trim()); + } + if (capturedUrls.has(finalUrl)) { + continue; + } + + const contentType = response.headers.get('content-type') ?? ''; + if (contentType && !/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) { + throw new Error(`Unexpected content type: ${contentType}`); + } + + const page = extractPage(await response.text(), finalUrl); + if (page.title === '404' && /This page could not be found\./i.test(page.markdown)) { + throw new SoftNotFoundError('Soft 404: This page could not be found.'); + } + for (const link of page.links) { + discover(link, finalUrl, discovered, queue, maxPages); + } + + const capturePath = routeToCapturePath(finalUrl); + destinationKey = capturePath.normalize('NFC').toLowerCase(); + const existingUrl = captureDestinations.get(destinationKey); + if (existingUrl && existingUrl !== finalUrl) { + throw new Error(`Capture destination collision: ${finalUrl} and ${existingUrl} both map to ${capturePath}`); + } + captureDestinations.set(destinationKey, finalUrl); + destinationOwner = finalUrl; + destination = path.join(stagedOutputRoot, ...capturePath.split('/')); + const finalDestination = path.join(outputRoot, ...capturePath.split('/')); + const content = `> Source: ${finalUrl}\n> Captured: ${capturedAt}\n\n${page.markdown}`; + + await mkdir(path.dirname(destination), { recursive: true }); + await writeFile(destination, content, 'utf8'); + + pages.push({ + url: finalUrl, + title: page.title, + path: toPosixPath(path.relative(path.dirname(manifestPath), finalDestination)), + contentHash: createHash('sha256').update(content).digest('hex'), + }); + capturedUrls.add(finalUrl); + } catch (error) { + if (error instanceof PageLimitError) { + throw error; + } + if (destination) { + await rm(destination, { force: true }); + } + if (destinationKey && captureDestinations.get(destinationKey) === destinationOwner) { + captureDestinations.delete(destinationKey); + } + if (error instanceof SoftNotFoundError) { + continue; + } + failures.push({ + url, + error: error instanceof Error ? error.message : String(error), + }); + } } + + const manifest = { + capturedAt, + baseUrl: LIVE_WIKI_BASE_URL, + pages: pages.sort(compareByUrl), + failures: failures.sort(compareByUrl), + }; + + await writeFile(stagedManifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); + await publishCapture({ + manifestPath, + outputRoot, + stagedManifestPath, + stagedOutputRoot, + stagingRoot, + }); + return manifest; + } finally { + await rm(stagingRoot, { recursive: true, force: true }); } +} - const manifest = { - capturedAt, - baseUrl: LIVE_WIKI_BASE_URL, - pages: pages.sort(compareByUrl), - failures: failures.sort(compareByUrl), - }; +async function publishCapture({ + manifestPath, + outputRoot, + stagedManifestPath, + stagedOutputRoot, + stagingRoot, +}) { + const previousOutput = path.join(stagingRoot, 'previous-snapshot'); + const previousManifest = path.join(stagingRoot, 'previous-manifest.json'); + let movedPreviousOutput = false; + let movedPreviousManifest = false; + let publishedOutput = false; + let publishedManifest = false; + await mkdir(path.dirname(outputRoot), { recursive: true }); await mkdir(path.dirname(manifestPath), { recursive: true }); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); - return manifest; + try { + if (await pathExists(outputRoot)) { + await rename(outputRoot, previousOutput); + movedPreviousOutput = true; + } + if (await pathExists(manifestPath)) { + await rename(manifestPath, previousManifest); + movedPreviousManifest = true; + } + + await rename(stagedOutputRoot, outputRoot); + publishedOutput = true; + await rename(stagedManifestPath, manifestPath); + publishedManifest = true; + } catch (error) { + if (publishedManifest) { + await rm(manifestPath, { force: true }); + } + if (publishedOutput) { + await rm(outputRoot, { recursive: true, force: true }); + } + if (movedPreviousOutput) { + await rename(previousOutput, outputRoot); + } + if (movedPreviousManifest) { + await rename(previousManifest, manifestPath); + } + throw error; + } +} + +async function fetchWithAuthorizedRedirects(startUrl, { + fetchImpl, + maxRedirects, + pace, +}) { + let currentUrl = startUrl; + const visited = new Set([currentUrl]); + + for (let redirects = 0; ; redirects += 1) { + await pace(); + const response = await fetchImpl(currentUrl, { + headers: { + accept: 'text/html,application/xhtml+xml', + 'user-agent': 'Incognito-Wiki research capture (+https://msvincognito.nl/)', + }, + redirect: 'manual', + }); + + if (!response) { + return { response, url: currentUrl }; + } + + if (![301, 302, 303, 307, 308].includes(response.status)) { + const responseUrl = response.url + ? normalizeWikiUrl(response.url, currentUrl) + : currentUrl; + if (!responseUrl) { + throw new Error(`Final response URL is not authorized: ${response.url}`); + } + return { response, url: responseUrl }; + } + + const location = response.headers.get('location'); + if (!location) { + throw new Error(`HTTP ${response.status} redirect is missing a Location header`); + } + const redirectUrl = normalizeWikiUrl(location, currentUrl); + if (!redirectUrl) { + throw new Error(`Redirect target is not an authorized content URL: ${location}`); + } + if (redirects >= maxRedirects) { + throw new Error(`Redirect limit of ${maxRedirects} exceeded at ${redirectUrl}`); + } + if (visited.has(redirectUrl)) { + throw new Error(`Redirect loop detected at ${redirectUrl}`); + } + + visited.add(redirectUrl); + currentUrl = redirectUrl; + } +} + +async function pathExists(candidate) { + try { + await access(candidate); + return true; + } catch (error) { + if (error?.code === 'ENOENT') { + return false; + } + throw error; + } } function cleanText(value = '') { @@ -241,40 +394,43 @@ function cleanText(value = '') { } function removeLiveWikiChrome(content) { + for (const header of content.querySelectorAll('header[class]')) { + const classes = new Set(header.classList); + if (classes.has('sticky') && classes.has('top-0') && classes.has('z-40')) { + header.remove(); + } + } + for (const element of content.querySelectorAll('[class]')) { const classes = new Set(element.classList); const text = cleanText(element.textContent); - const isPreviousNextNavigation = classes.has('mt-16'); + const previousNextPanel = element.querySelector('.border-t.pt-6'); + const isPreviousNextNavigation = classes.has('mt-16') + && previousNextPanel?.querySelector('a[href]'); const isRepeatedPageFooter = classes.has('hidden') && classes.has('text-sm') && classes.has('lg:block') && /Star on GitHub|Create Issues/.test(text); + const isGlobalSiteFooter = element.localName === 'footer' + && classes.has('text-muted-foreground') + && classes.has('py-6') + && classes.has('md:px-8') + && classes.has('md:py-0') + && /Copyright © \d{4} MSV Incognito/.test(text); - if (isPreviousNextNavigation || isRepeatedPageFooter) { + if (isPreviousNextNavigation || isRepeatedPageFooter || isGlobalSiteFooter) { element.remove(); } } } -function sanitizePathSegment(segment) { - let decoded; - try { - decoded = decodeURIComponent(segment); - } catch { - decoded = segment; - } - - const safe = decoded - .normalize('NFKC') - .replace(/[^\p{Letter}\p{Number}._-]+/gu, '-') - .replace(/^-+|-+$/g, ''); - - if (!safe || safe === '.' || safe === '..') { +function validatePathSegment(segment) { + if (!/^[A-Za-z0-9._-]+$/.test(segment) || segment === '.' || segment === '..') { throw new Error(`Unsafe live-wiki route segment: ${segment}`); } - return safe; + return segment; } function titleFromUrl(url) { diff --git a/tests/live-wiki.test.mjs b/tests/live-wiki.test.mjs index 5956d19..ea0091c 100644 --- a/tests/live-wiki.test.mjs +++ b/tests/live-wiki.test.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -33,6 +33,10 @@ test('maps routes to readable Markdown files', () => { 'useful-guides/description.md', ); assert.equal(routeToCapturePath('https://wiki.msvincognito.nl/'), 'index.md'); + assert.throws( + () => routeToCapturePath('https://wiki.msvincognito.nl/foo%20bar'), + /Unsafe live-wiki route segment: foo%20bar/, + ); }); test('extracts the main article and internal links', () => { @@ -69,7 +73,7 @@ test('falls back from main to article and then body', () => { 'https://wiki.msvincognito.nl/article', ); const bodyPage = extractPage( - 'Body title
Site chrome

Body text

', + 'Body title
Site chrome

Body text

', 'https://wiki.msvincognito.nl/body', ); @@ -86,19 +90,41 @@ test('removes the live wiki previous-next panel and repeated page footer', () =>

Course

Useful article content.

-
Previous
+
Previous
+ `, 'https://wiki.msvincognito.nl/course'); assert.match(page.markdown, /Useful article content/); - assert.doesNotMatch(page.markdown, /Previous|Star on GitHub/); + assert.doesNotMatch(page.markdown, /Previous|Star on GitHub|Copyright/); assert.deepEqual(page.links, ['https://wiki.msvincognito.nl/previous']); }); +test('preserves legitimate article headers, asides, footers, and spaced sections', () => { + const page = extractPage(` +
+
+

Research note

By the board

+ +
A deliberately spaced substantive section.
+
Article footnote.
+
+
+ `, 'https://wiki.msvincognito.nl/research-note'); + + assert.match(page.markdown, /# Research note/); + assert.match(page.markdown, /By the board/); + assert.match(page.markdown, /Important contextual aside/); + assert.match(page.markdown, /deliberately spaced substantive section/); + assert.match(page.markdown, /Article footnote/); +}); + test('crawls in sorted order, writes successful pages, and records failures', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-crawl-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); @@ -156,6 +182,174 @@ test('crawls in sorted order, writes successful pages, and records failures', as assert.deepEqual(savedManifest, manifest); }); +test('follows same-origin redirects manually, paces every hop, and captures the final URL', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); + const requests = []; + const waits = []; + const responses = new Map([ + ['https://wiki.msvincognito.nl/', redirectResponse('/guide')], + ['https://wiki.msvincognito.nl/guide', response('

Guide

Final content.

')], + ]); + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + outputRoot, + fetchImpl: async (url, options) => { + requests.push({ url, redirect: options.redirect }); + return responses.get(url); + }, + waitImpl: async (milliseconds) => waits.push(milliseconds), + }); + + assert.deepEqual(requests, [ + { url: 'https://wiki.msvincognito.nl/', redirect: 'manual' }, + { url: 'https://wiki.msvincognito.nl/guide', redirect: 'manual' }, + ]); + assert.deepEqual(waits, [150]); + assert.deepEqual(manifest.pages.map(({ url, path: capturePath }) => ({ url, path: capturePath })), [{ + url: 'https://wiki.msvincognito.nl/guide', + path: 'live-wiki/2026-08-02/guide.md', + }]); + const captured = await readFile(path.join(outputRoot, 'guide.md'), 'utf8'); + assert.match(captured, /^> Source: https:\/\/wiki\.msvincognito\.nl\/guide$/m); +}); + +test('rejects cross-origin redirects without requesting the target', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-cross-origin-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const requests = []; + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), + fetchImpl: async (url) => { + requests.push(url); + return redirectResponse('https://example.com/private'); + }, + waitImpl: async () => {}, + }); + + assert.deepEqual(requests, ['https://wiki.msvincognito.nl/']); + assert.deepEqual(manifest.pages, []); + assert.deepEqual(manifest.failures, [{ + url: 'https://wiki.msvincognito.nl/', + error: 'Redirect target is not an authorized content URL: https://example.com/private', + }]); +}); + +test('stops a same-origin redirect chain at the configured hop limit', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-redirect-limit-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const requests = []; + const waits = []; + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + maxRedirects: 1, + outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), + fetchImpl: async (url) => { + requests.push(url); + return url.endsWith('/one') + ? redirectResponse('/two') + : redirectResponse('/one'); + }, + waitImpl: async (milliseconds) => waits.push(milliseconds), + }); + + assert.deepEqual(requests, [ + 'https://wiki.msvincognito.nl/', + 'https://wiki.msvincognito.nl/one', + ]); + assert.deepEqual(waits, [150]); + assert.deepEqual(manifest.pages, []); + assert.deepEqual(manifest.failures, [{ + url: 'https://wiki.msvincognito.nl/', + error: 'Redirect limit of 1 exceeded at https://wiki.msvincognito.nl/two', + }]); +}); + +test('captures a redirected final URL only once when its canonical route is also queued', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-canonical-redirect-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const requests = []; + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + outputRoot: path.join(captureRoot, 'live-wiki', '2026-08-02'), + fetchImpl: async (url) => { + requests.push(url); + if (url.endsWith('/alias')) return redirectResponse('/target'); + if (url.endsWith('/target')) return response('

Target

'); + return response('

Home

AliasTarget
'); + }, + waitImpl: async () => {}, + }); + + assert.deepEqual(requests, [ + 'https://wiki.msvincognito.nl/', + 'https://wiki.msvincognito.nl/alias', + 'https://wiki.msvincognito.nl/target', + ]); + assert.deepEqual(manifest.pages.map(({ url }) => url), [ + 'https://wiki.msvincognito.nl/', + 'https://wiki.msvincognito.nl/target', + ]); +}); + +test('records an unsafe route as a per-URL failure and preserves other pages', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-unsafe-path-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + outputRoot, + fetchImpl: async (url) => url.endsWith('/foo%20bar') + ? response('

Unsafe

') + : response('

Home

Unsafe route
'), + waitImpl: async () => {}, + }); + + assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); + assert.deepEqual(manifest.failures, [{ + url: 'https://wiki.msvincognito.nl/foo%20bar', + error: 'Unsafe live-wiki route segment: foo%20bar', + }]); + assert.match(await readFile(path.join(outputRoot, 'index.md'), 'utf8'), /# Home/); +}); + +test('records destination collisions before a later page can overwrite a capture', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-collision-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); + + const manifest = await crawlWiki({ + capturedAt: '2026-08-02', + manifestPath: path.join(captureRoot, 'manifest.json'), + outputRoot, + fetchImpl: async (url) => url.endsWith('/index') + ? response('

Colliding page

') + : response('

Home

Collision
'), + waitImpl: async () => {}, + }); + + assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); + assert.deepEqual(manifest.failures, [{ + url: 'https://wiki.msvincognito.nl/index', + error: 'Capture destination collision: https://wiki.msvincognito.nl/index and https://wiki.msvincognito.nl/ both map to index.md', + }]); + const captured = await readFile(path.join(outputRoot, 'index.md'), 'utf8'); + assert.match(captured, /# Home/); + assert.doesNotMatch(captured, /Colliding page/); +}); + test('rejects discovery beyond the configured page ceiling', async (context) => { const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-limit-')); context.after(() => rm(captureRoot, { recursive: true, force: true })); @@ -180,8 +374,10 @@ test('excludes soft 404 navigation placeholders and removes stale captures', asy context.after(() => rm(captureRoot, { recursive: true, force: true })); const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); const stalePath = path.join(outputRoot, 'missing.md'); + const unrelatedStalePath = path.join(outputRoot, 'obsolete.md'); await mkdir(outputRoot, { recursive: true }); await writeFile(stalePath, 'stale capture', 'utf8'); + await writeFile(unrelatedStalePath, 'obsolete capture', 'utf8'); const manifest = await crawlWiki({ capturedAt: '2026-08-02', @@ -196,6 +392,32 @@ test('excludes soft 404 navigation placeholders and removes stale captures', asy assert.deepEqual(manifest.pages.map(({ url }) => url), ['https://wiki.msvincognito.nl/']); assert.deepEqual(manifest.failures, []); await assert.rejects(readFile(stalePath, 'utf8'), { code: 'ENOENT' }); + await assert.rejects(readFile(unrelatedStalePath, 'utf8'), { code: 'ENOENT' }); +}); + +test('leaves the previous snapshot and manifest untouched after a fatal crawl error', async (context) => { + const captureRoot = await mkdtemp(path.join(tmpdir(), 'live-wiki-atomic-failure-')); + context.after(() => rm(captureRoot, { recursive: true, force: true })); + const outputRoot = path.join(captureRoot, 'live-wiki', '2026-08-02'); + const manifestPath = path.join(captureRoot, 'manifest.json'); + await mkdir(outputRoot, { recursive: true }); + await writeFile(path.join(outputRoot, 'previous.md'), 'previous snapshot', 'utf8'); + await writeFile(manifestPath, '{"snapshot":"previous"}\n', 'utf8'); + + await assert.rejects(crawlWiki({ + capturedAt: '2026-08-02', + manifestPath, + maxPages: 2, + outputRoot, + fetchImpl: async (url) => url.endsWith('/one') + ? response('

One

Two
') + : response('

Home

One
'), + waitImpl: async () => {}, + }), /more than 2 content pages/i); + + assert.deepEqual(await readdir(outputRoot), ['previous.md']); + assert.equal(await readFile(path.join(outputRoot, 'previous.md'), 'utf8'), 'previous snapshot'); + assert.equal(await readFile(manifestPath, 'utf8'), '{"snapshot":"previous"}\n'); }); test('capture command reports failure after the crawler preserves its manifest', async () => { @@ -220,3 +442,11 @@ function response(body, status = 200) { headers: { 'content-type': 'text/html; charset=utf-8' }, }); } + +function redirectResponse(location, status = 302) { + return new Response('', { + status, + statusText: status === 302 ? 'Found' : 'Redirect', + headers: { location }, + }); +} diff --git a/to-be-studied/README.md b/to-be-studied/README.md index f827a94..2c6fa6b 100644 --- a/to-be-studied/README.md +++ b/to-be-studied/README.md @@ -2,7 +2,7 @@ This directory is a research snapshot of public, unauthenticated pages from `https://wiki.msvincognito.nl/`. It is not a source for the published migration and may contain incomplete, superseded, or outdated material. -The snapshot was captured on 2026-08-02 with `npm run capture:live-wiki`. The crawler stays on the live wiki's origin, never authenticates or submits forms, waits 150 ms between requests, refuses to crawl more than 500 content pages, and records request failures in `manifest.json`. It converts the first `
` element (falling back to `
` and then ``) to readable Markdown after removing site chrome. Public navigation enumerated the content; the site's `/sitemap.xml` and `/robots.txt` routes both render its soft-404 page, so neither supplied additional URLs. The capture excludes navigation category URLs that return the same deterministic soft-404 page instead of content. +The snapshot was captured on 2026-08-02 with `npm run capture:live-wiki`. The crawler stays on the live wiki's origin, never authenticates or submits forms, waits 150 ms between every request and redirect hop, refuses to crawl more than 500 content pages, and records request failures in `manifest.json`. It follows only bounded, same-origin redirects and records each final canonical URL once. It converts the first `
` element (falling back to `
` and then ``) to readable Markdown after removing site chrome. Public navigation enumerated the content; the site's `/sitemap.xml` and `/robots.txt` routes both render its soft-404 page, so neither supplied additional URLs. The capture excludes navigation category URLs that return the same deterministic soft-404 page instead of content. Files under `live-wiki/2026-08-02/` retain their public source URL and capture date. `manifest.json` records the sorted capture inventory and SHA-256 hash of every Markdown file. `comparison.md` compares that inventory with the 29-entry DokuWiki export manifest. diff --git a/to-be-studied/comparison.md b/to-be-studied/comparison.md index c76b401..19760e1 100644 --- a/to-be-studied/comparison.md +++ b/to-be-studied/comparison.md @@ -1,14 +1,14 @@ # Export and live-wiki comparison -This is a research inventory, not a merge decision. It compares the 98 successful pages in `manifest.json` with all 29 source entries in `docs/migration-manifest.json`. The comparison is page-level: a live course page can be new-only even when its surrounding year/programme taxonomy overlaps an exported index page. +This is a research inventory, not a merge decision. It compares the 93 successful canonical pages in `manifest.json` with all 29 source entries in `docs/migration-manifest.json`. The comparison is page-level: a live course page can be new-only even when its surrounding year/programme taxonomy overlaps an exported index page. Five public navigation aliases redirect to canonical description routes and are not duplicated in the capture. ## New live-wiki pages with no exported page equivalent -These 88 successful live pages have no one-to-one page in the export. Course titles are listed so every captured course page is concretely classified rather than represented only by a count. +These 87 successful live pages have no one-to-one page in the export. Course titles are listed so every captured course page is concretely classified rather than represented only by a count. | Live route or family | Count | Captured pages | Classification | | --- | ---: | --- | --- | -| `/computer-science` and `/computer-science/course-description` | 2 | Course Description (two routes with the same sparse content) | The export has no Computer Science programme page; its bachelor programme is Data Science and Artificial Intelligence. | +| `/computer-science/course-description` | 1 | Course Description | The export has no Computer Science programme page; its bachelor programme is Data Science and Artificial Intelligence. The `/computer-science` navigation alias redirects here. | | `/computer-science/year-1/**` | 12 | Discrete Math; Introduction to Computer Science; Procedural Programming; Calculus; Logic; Objects in Programming; Computer Architecture; Data Structures and Algorithms; Linear Algebra; Databases; Algorithmic Design; Statistics | New course-level hierarchy organized by year and period; the export has no corresponding course pages. | | `/computer-science/year-2/**` | 14 | Intelligent User Interfaces; Introduction to Artificial Intelligence; Operating Systems; Computer Networks; Software Engineering and Architectures; Computer Security; Embedded Programming; Parallel Programming; Numerical Methods; Principles of Programming Languages; M2-1: AI and Machine Learning; M2-1: Intelligent Interaction; M2-2: Cybersecurity; M2-2: High Performance Computing | New course/elective pages; no exported equivalents. | | `/computer-science/year-3/**` | 16 | Digital Society; Game Theory; Graph Theory; Introduction to Quantum Computing; Robotics and Embedded Systems; Ubiquitous Computing & Internet of Things; Block Chains; Cryptography; Immersive Technologies; Introduction to Bio-Informatics; Large Scale IT and Cloud Computing; Software and Systems Verification; Startup Engineering: Building Scalable Tech Ventures; The History and Philosophy of Computing; Operating Systems; Theory of Computing | New course/elective pages; no exported equivalents. | @@ -53,13 +53,13 @@ All seven exported project pages are called out explicitly. Year landing pages a | Export source path | Live URL | Substantive difference | | --- | --- | --- | | `pages/start.txt` + `pages/study.txt` | `https://wiki.msvincognito.nl/` | The export has association/registration/editing instructions, official links, Maastricht University context, and programme descriptions. The live home is a short DACS landing page linking four programme areas and Useful Guides. | -| `pages/study/bachelor.txt` | `https://wiki.msvincognito.nl/data-science-and-artificial-intelligence` and `/course-description` | Both identify the Data Science and Artificial Intelligence bachelor area. The export explains the programme index and contribution workflow; both live routes currently contain only a sparse “Course Description / #Hello” page. | +| `pages/study/bachelor.txt` | `https://wiki.msvincognito.nl/data-science-and-artificial-intelligence/course-description` | Both identify the Data Science and Artificial Intelligence bachelor area. The export explains the programme index and contribution workflow; the canonical live route currently contains only a sparse “Course Description / #Hello” page. The base navigation route redirects here. | | `pages/study/bachelor/year_1.txt` | `https://wiki.msvincognito.nl/data-science-and-artificial-intelligence/year-1/period-1/discrete-mathematics` (representative of 12 Year 1 leaf pages) | The export is a year index with a missing schedule image and page-creation controls. The live wiki has period/course leaf pages but its Year 1 and period category URLs render soft 404s. | | `pages/study/bachelor/year_2.txt` | `https://wiki.msvincognito.nl/data-science-and-artificial-intelligence/year-2/period-1/databases` (representative of 13 Year 2 leaf pages) | The export is a schedule/index shell plus upload instructions. The live wiki replaces that organization with period/elective course leaves and no valid Year 2 landing page. | | `pages/study/bachelor/year_3.txt` | `https://wiki.msvincognito.nl/data-science-and-artificial-intelligence/year-3/period-4/data-analysis` (representative of 16 Year 3 leaf pages) | The export discusses thesis timing and indexes thesis/project/study-abroad pages. The live branch lists course/elective leaves and contains no thesis or project page. | -| `pages/study/master_ai.txt` | `https://wiki.msvincognito.nl/artificial-intelligence` and `/course-description` | Both label the MSc Artificial Intelligence area. The export describes course summaries, exams, year navigation, and contribution flow; the live routes are currently a sparse “Course Description / #Hello” page with no year/course content. | -| `pages/study/master_dsdm.txt` | `https://wiki.msvincognito.nl/data-science-for-decision-making` and `/course-description` | Both label the MSc Data Science for Decision Making area. The export has year navigation and contribution context; the live routes contain only the sparse description placeholder. | -| `pages/study/useful_information.txt` | `https://wiki.msvincognito.nl/useful-guides` and `/description` | The export indexes locations, IT services, and laptop advice. The live index instead presents Housing, Laptop Buying Advice, Linux Tricks, and Surviving DACS, and claims the guides are updated “somewhat regularly.” | +| `pages/study/master_ai.txt` | `https://wiki.msvincognito.nl/artificial-intelligence/course-description` | Both label the MSc Artificial Intelligence area. The export describes course summaries, exams, year navigation, and contribution flow; the canonical live route is currently a sparse “Course Description / #Hello” page with no year/course content. The base navigation route redirects here. | +| `pages/study/master_dsdm.txt` | `https://wiki.msvincognito.nl/data-science-for-decision-making/course-description` | Both label the MSc Data Science for Decision Making area. The export has year navigation and contribution context; the canonical live route contains only the sparse description placeholder. The base navigation route redirects here. | +| `pages/study/useful_information.txt` | `https://wiki.msvincognito.nl/useful-guides/description` | The export indexes locations, IT services, and laptop advice. The canonical live index instead presents Housing, Laptop Buying Advice, Linux Tricks, and Surviving DACS, and claims the guides are updated “somewhat regularly.” The `/useful-guides` navigation alias redirects here. | | `pages/study/useful_information/pages/laptop_buy_advice.txt` | `https://wiki.msvincognito.nl/useful-guides/laptop-buying-advice` | The live page preserves and edits the exported tier guide, marks it as last updated 2021, and adds a current minimum of 16 GB RAM/512 GB SSD, Apple M-series compatibility, and an Nvidia/CUDA recommendation for data-science programmes. | ## Navigation placeholders excluded from the capture diff --git a/to-be-studied/live-wiki/2026-08-02/artificial-intelligence.md b/to-be-studied/live-wiki/2026-08-02/artificial-intelligence.md deleted file mode 100644 index c0d0493..0000000 --- a/to-be-studied/live-wiki/2026-08-02/artificial-intelligence.md +++ /dev/null @@ -1,8 +0,0 @@ -> Source: https://wiki.msvincognito.nl/artificial-intelligence -> Captured: 2026-08-02 - -# Course Description - -#Hello - -#Hello diff --git a/to-be-studied/live-wiki/2026-08-02/computer-science.md b/to-be-studied/live-wiki/2026-08-02/computer-science.md deleted file mode 100644 index 4b5afda..0000000 --- a/to-be-studied/live-wiki/2026-08-02/computer-science.md +++ /dev/null @@ -1,8 +0,0 @@ -> Source: https://wiki.msvincognito.nl/computer-science -> Captured: 2026-08-02 - -# Course Description - -#Hello - -#Hello diff --git a/to-be-studied/live-wiki/2026-08-02/data-science-and-artificial-intelligence.md b/to-be-studied/live-wiki/2026-08-02/data-science-and-artificial-intelligence.md deleted file mode 100644 index d4e7c3c..0000000 --- a/to-be-studied/live-wiki/2026-08-02/data-science-and-artificial-intelligence.md +++ /dev/null @@ -1,8 +0,0 @@ -> Source: https://wiki.msvincognito.nl/data-science-and-artificial-intelligence -> Captured: 2026-08-02 - -# Course Description - -#Hello - -#Hello diff --git a/to-be-studied/live-wiki/2026-08-02/data-science-for-decision-making.md b/to-be-studied/live-wiki/2026-08-02/data-science-for-decision-making.md deleted file mode 100644 index 4c28efa..0000000 --- a/to-be-studied/live-wiki/2026-08-02/data-science-for-decision-making.md +++ /dev/null @@ -1,8 +0,0 @@ -> Source: https://wiki.msvincognito.nl/data-science-for-decision-making -> Captured: 2026-08-02 - -# Course Description - -#Hello - -#Hello diff --git a/to-be-studied/live-wiki/2026-08-02/useful-guides.md b/to-be-studied/live-wiki/2026-08-02/useful-guides.md deleted file mode 100644 index 0c7fba7..0000000 --- a/to-be-studied/live-wiki/2026-08-02/useful-guides.md +++ /dev/null @@ -1,50 +0,0 @@ -> Source: https://wiki.msvincognito.nl/useful-guides -> Captured: 2026-08-02 - -# Useful Guides - -Essential resources and tips for your academic journey at Maastricht University - -Welcome to the **Useful Guides** section of the Incognito Wiki! This collection contains practical resources and tips to help you navigate your academic journey and student life at **Maastricht University**. - -[ - -### Housing Guide - -Essential information for finding accommodation in **Maastricht**. Navigate the housing market, understand rental agreements, and discover the best neighborhoods for students. - - - -](https://wiki.msvincognito.nl/useful-guides/housing-guide) - -[ - -### Laptop Buying Advice - -Recommendations for choosing the right laptop for your studies. Get insights on specifications, brands, and budget considerations tailored for **DACS students**. - - - -](https://wiki.msvincognito.nl/useful-guides/laptop-buying-advice) - -[ - -### Linux Tricks - -Helpful tips and commands for working with **Linux systems**. Master the command line, learn essential shortcuts, and boost your productivity. - - - -](https://wiki.msvincognito.nl/useful-guides/linux-tricks) - -[ - -### Surviving DACS - -Insider tips for making the most of your time in the **Department of Data Science & Knowledge Engineering**. Study strategies, course insights, and academic advice. - - - -](https://wiki.msvincognito.nl/useful-guides/survivingdacs) - -These guides are created by fellow students and updated somewhat regularly to ensure they remain relevant and helpful. Whether you're a first-year student just starting out or a senior looking for specific technical advice, you'll find valuable information here to support your academic success. diff --git a/to-be-studied/manifest.json b/to-be-studied/manifest.json index cca3e6f..ab0eb47 100644 --- a/to-be-studied/manifest.json +++ b/to-be-studied/manifest.json @@ -8,24 +8,12 @@ "path": "live-wiki/2026-08-02/index.md", "contentHash": "f1dae2fe65c445a89efae3a2b2b88e9e00c942894d64a158a9fc1ce69c71edb5" }, - { - "url": "https://wiki.msvincognito.nl/artificial-intelligence", - "title": "Course Description", - "path": "live-wiki/2026-08-02/artificial-intelligence.md", - "contentHash": "f2bb513f738e2dd940b4c6c5be2295d65fc9c0c7a59e96a93edef0efefafea36" - }, { "url": "https://wiki.msvincognito.nl/artificial-intelligence/course-description", "title": "Course Description", "path": "live-wiki/2026-08-02/artificial-intelligence/course-description.md", "contentHash": "9643a34b326331798ab1ac3d512f8b65107b6eb4029bb364a40cec7797fea982" }, - { - "url": "https://wiki.msvincognito.nl/computer-science", - "title": "Course Description", - "path": "live-wiki/2026-08-02/computer-science.md", - "contentHash": "e815decc8bc817d8338a3950451f729790feec25fc06e9663cad6a7ce0139d39" - }, { "url": "https://wiki.msvincognito.nl/computer-science/course-description", "title": "Course Description", @@ -284,12 +272,6 @@ "path": "live-wiki/2026-08-02/computer-science/year-3/period-4/theory-of-computing.md", "contentHash": "9e804bc6fad536feeebf5268ddfd6d597787f534b5de76c03d51f8b52283e71e" }, - { - "url": "https://wiki.msvincognito.nl/data-science-and-artificial-intelligence", - "title": "Course Description", - "path": "live-wiki/2026-08-02/data-science-and-artificial-intelligence.md", - "contentHash": "8d9f7850bc1124678c0ee0533472423bf0ea4906ef95471ea87de3ec1fe5ab10" - }, { "url": "https://wiki.msvincognito.nl/data-science-and-artificial-intelligence/course-description", "title": "Course Description", @@ -542,24 +524,12 @@ "path": "live-wiki/2026-08-02/data-science-and-artificial-intelligence/year-3/period-4/operations-research.md", "contentHash": "542864580dd0d5e839ce607581807bfdde2454e591980948e0d3828233070250" }, - { - "url": "https://wiki.msvincognito.nl/data-science-for-decision-making", - "title": "Course Description", - "path": "live-wiki/2026-08-02/data-science-for-decision-making.md", - "contentHash": "a9a3d434f808edcfd5760bc99433b568268de7548bd4a9a9da51677ea78899f1" - }, { "url": "https://wiki.msvincognito.nl/data-science-for-decision-making/course-description", "title": "Course Description", "path": "live-wiki/2026-08-02/data-science-for-decision-making/course-description.md", "contentHash": "d356ae2e36ef1fa1a575d9afe333597864a0da19e18e99ebecc575c733007858" }, - { - "url": "https://wiki.msvincognito.nl/useful-guides", - "title": "Useful Guides", - "path": "live-wiki/2026-08-02/useful-guides.md", - "contentHash": "b60fccd25e412b5f52149f89e4efa2f802c14f1a0c44500190ec099f5cd997f6" - }, { "url": "https://wiki.msvincognito.nl/useful-guides/description", "title": "Useful Guides",