467 lines
14 KiB
JavaScript
467 lines
14 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { access, mkdir, mkdtemp, rename, rm, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
import { parseHTML } from 'linkedom';
|
|
import TurndownService from 'turndown';
|
|
|
|
export const LIVE_WIKI_BASE_URL = 'https://wiki.msvincognito.nl/';
|
|
|
|
const ASSET_EXTENSION = /\.(?:avif|bmp|css|csv|docx?|eot|gif|ico|jpe?g|js|json|map|mp3|mp4|ogg|otf|pdf|png|pptx?|rar|rss|svg|tar|tiff?|tsv|txt|wav|webm|webmanifest|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
const NON_CONTENT_SELECTOR = [
|
|
'script',
|
|
'style',
|
|
'nav',
|
|
'form',
|
|
'button',
|
|
'noscript',
|
|
'iframe',
|
|
'dialog',
|
|
'[role="navigation"]',
|
|
'[role="contentinfo"]',
|
|
].join(',');
|
|
|
|
class PageLimitError extends Error {}
|
|
class SoftNotFoundError extends Error {}
|
|
|
|
export function normalizeWikiUrl(input, relativeTo = LIVE_WIKI_BASE_URL) {
|
|
let url;
|
|
|
|
try {
|
|
url = new URL(input, relativeTo);
|
|
} catch {
|
|
return null;
|
|
}
|
|
|
|
const baseUrl = new URL(LIVE_WIKI_BASE_URL);
|
|
if (url.origin !== baseUrl.origin || !['http:', 'https:'].includes(url.protocol)) {
|
|
return null;
|
|
}
|
|
|
|
url.search = '';
|
|
url.hash = '';
|
|
url.pathname = url.pathname.replace(/\/{2,}/g, '/');
|
|
|
|
if (url.pathname !== '/') {
|
|
url.pathname = url.pathname.replace(/\/+$/, '');
|
|
}
|
|
|
|
if (ASSET_EXTENSION.test(url.pathname)) {
|
|
return null;
|
|
}
|
|
|
|
return url.href;
|
|
}
|
|
|
|
export function routeToCapturePath(input) {
|
|
const normalizedUrl = normalizeWikiUrl(input);
|
|
if (!normalizedUrl) {
|
|
throw new TypeError(`Cannot map non-content URL to a capture path: ${input}`);
|
|
}
|
|
|
|
const { pathname } = new URL(normalizedUrl);
|
|
if (pathname === '/') {
|
|
return 'index.md';
|
|
}
|
|
|
|
const route = pathname
|
|
.split('/')
|
|
.filter(Boolean)
|
|
.map((segment) => validatePathSegment(segment))
|
|
.join('/');
|
|
|
|
return `${route}.md`;
|
|
}
|
|
|
|
export function extractPage(html, url) {
|
|
const { document } = parseHTML(html);
|
|
const content = document.querySelector('main')
|
|
?? document.querySelector('article')
|
|
?? document.body;
|
|
|
|
if (!content) {
|
|
throw new Error(`No readable content root found at ${url}`);
|
|
}
|
|
|
|
const links = new Set();
|
|
const currentUrl = normalizeWikiUrl(url);
|
|
for (const anchor of content.querySelectorAll('a[href]')) {
|
|
const normalizedLink = normalizeWikiUrl(anchor.getAttribute('href'), url);
|
|
if (normalizedLink && normalizedLink !== currentUrl) {
|
|
links.add(normalizedLink);
|
|
anchor.setAttribute('href', normalizedLink);
|
|
}
|
|
}
|
|
|
|
const heading = content.querySelector('h1, h2, h3');
|
|
const title = cleanText(heading?.textContent)
|
|
|| cleanText(document.querySelector('title')?.textContent)
|
|
|| titleFromUrl(url);
|
|
|
|
for (const element of content.querySelectorAll(NON_CONTENT_SELECTOR)) {
|
|
element.remove();
|
|
}
|
|
removeLiveWikiChrome(content);
|
|
|
|
const turndown = new TurndownService({
|
|
bulletListMarker: '-',
|
|
codeBlockStyle: 'fenced',
|
|
emDelimiter: '*',
|
|
headingStyle: 'atx',
|
|
});
|
|
turndown.addRule('lineBreakWithoutTrailingWhitespace', {
|
|
filter: 'br',
|
|
replacement: () => '<br>\n',
|
|
});
|
|
const markdown = `${turndown.turndown(content).trim().replace(/[ \t]+$/gm, '')}\n`;
|
|
|
|
return {
|
|
title,
|
|
markdown,
|
|
links: [...links].sort(),
|
|
};
|
|
}
|
|
|
|
export async function crawlWiki({
|
|
capturedAt = new Date().toISOString().slice(0, 10),
|
|
delayMs = 150,
|
|
fetchImpl = globalThis.fetch,
|
|
manifestPath,
|
|
maxPages = 500,
|
|
maxRedirects = 5,
|
|
outputRoot,
|
|
startUrls = [LIVE_WIKI_BASE_URL],
|
|
waitImpl = wait,
|
|
} = {}) {
|
|
if (!outputRoot || !manifestPath) {
|
|
throw new TypeError('crawlWiki requires outputRoot and manifestPath.');
|
|
}
|
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(capturedAt)) {
|
|
throw new TypeError('capturedAt must use YYYY-MM-DD format.');
|
|
}
|
|
if (!Number.isInteger(maxPages) || maxPages < 1) {
|
|
throw new TypeError('maxPages must be a positive integer.');
|
|
}
|
|
if (!Number.isInteger(maxRedirects) || maxRedirects < 0) {
|
|
throw new TypeError('maxRedirects must be a non-negative integer.');
|
|
}
|
|
|
|
const queue = [];
|
|
const discovered = new Set();
|
|
for (const startUrl of startUrls) {
|
|
discover(startUrl, LIVE_WIKI_BASE_URL, discovered, queue, maxPages);
|
|
}
|
|
|
|
const pages = [];
|
|
const failures = [];
|
|
const capturedUrls = new Set();
|
|
const captureDestinations = new Map();
|
|
let requestCount = 0;
|
|
|
|
await mkdir(path.dirname(outputRoot), { recursive: true });
|
|
const stagingRoot = await mkdtemp(path.join(
|
|
path.dirname(outputRoot),
|
|
`.${path.basename(outputRoot)}-staging-`,
|
|
));
|
|
const stagedOutputRoot = path.join(stagingRoot, 'snapshot');
|
|
const stagedManifestPath = path.join(stagingRoot, 'manifest.json');
|
|
await mkdir(stagedOutputRoot, { recursive: true });
|
|
|
|
try {
|
|
while (queue.length > 0) {
|
|
queue.sort();
|
|
const url = queue.shift();
|
|
if (capturedUrls.has(url)) {
|
|
continue;
|
|
}
|
|
let destination;
|
|
let destinationKey;
|
|
let destinationOwner;
|
|
|
|
try {
|
|
const fetched = await fetchWithAuthorizedRedirects(url, {
|
|
fetchImpl,
|
|
maxRedirects,
|
|
pace: async () => {
|
|
if (requestCount > 0) {
|
|
await waitImpl(delayMs);
|
|
}
|
|
requestCount += 1;
|
|
},
|
|
});
|
|
const { response, url: finalUrl } = fetched;
|
|
|
|
if (!response) {
|
|
throw new Error('No response');
|
|
}
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());
|
|
}
|
|
if (capturedUrls.has(finalUrl)) {
|
|
continue;
|
|
}
|
|
|
|
const contentType = response.headers.get('content-type') ?? '';
|
|
if (contentType && !/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
|
|
throw new Error(`Unexpected content type: ${contentType}`);
|
|
}
|
|
|
|
const page = extractPage(await response.text(), finalUrl);
|
|
if (page.title === '404' && /This page could not be found\./i.test(page.markdown)) {
|
|
throw new SoftNotFoundError('Soft 404: This page could not be found.');
|
|
}
|
|
for (const link of page.links) {
|
|
discover(link, finalUrl, discovered, queue, maxPages);
|
|
}
|
|
|
|
const capturePath = routeToCapturePath(finalUrl);
|
|
destinationKey = capturePath.normalize('NFC').toLowerCase();
|
|
const existingUrl = captureDestinations.get(destinationKey);
|
|
if (existingUrl && existingUrl !== finalUrl) {
|
|
throw new Error(`Capture destination collision: ${finalUrl} and ${existingUrl} both map to ${capturePath}`);
|
|
}
|
|
captureDestinations.set(destinationKey, finalUrl);
|
|
destinationOwner = finalUrl;
|
|
destination = path.join(stagedOutputRoot, ...capturePath.split('/'));
|
|
const finalDestination = path.join(outputRoot, ...capturePath.split('/'));
|
|
const content = `> Source: ${finalUrl}\n> Captured: ${capturedAt}\n\n${page.markdown}`;
|
|
|
|
await mkdir(path.dirname(destination), { recursive: true });
|
|
await writeFile(destination, content, 'utf8');
|
|
|
|
pages.push({
|
|
url: finalUrl,
|
|
title: page.title,
|
|
path: toPosixPath(path.relative(path.dirname(manifestPath), finalDestination)),
|
|
contentHash: createHash('sha256').update(content).digest('hex'),
|
|
});
|
|
capturedUrls.add(finalUrl);
|
|
} catch (error) {
|
|
if (error instanceof PageLimitError) {
|
|
throw error;
|
|
}
|
|
if (destination) {
|
|
await rm(destination, { force: true });
|
|
}
|
|
if (destinationKey && captureDestinations.get(destinationKey) === destinationOwner) {
|
|
captureDestinations.delete(destinationKey);
|
|
}
|
|
if (error instanceof SoftNotFoundError) {
|
|
continue;
|
|
}
|
|
failures.push({
|
|
url,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
const manifest = {
|
|
capturedAt,
|
|
baseUrl: LIVE_WIKI_BASE_URL,
|
|
pages: pages.sort(compareByUrl),
|
|
failures: failures.sort(compareByUrl),
|
|
};
|
|
|
|
await writeFile(stagedManifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
|
|
await publishCapture({
|
|
manifestPath,
|
|
outputRoot,
|
|
stagedManifestPath,
|
|
stagedOutputRoot,
|
|
stagingRoot,
|
|
});
|
|
return manifest;
|
|
} finally {
|
|
await rm(stagingRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function publishCapture({
|
|
manifestPath,
|
|
outputRoot,
|
|
stagedManifestPath,
|
|
stagedOutputRoot,
|
|
stagingRoot,
|
|
}) {
|
|
const previousOutput = path.join(stagingRoot, 'previous-snapshot');
|
|
const previousManifest = path.join(stagingRoot, 'previous-manifest.json');
|
|
let movedPreviousOutput = false;
|
|
let movedPreviousManifest = false;
|
|
let publishedOutput = false;
|
|
let publishedManifest = false;
|
|
|
|
await mkdir(path.dirname(outputRoot), { recursive: true });
|
|
await mkdir(path.dirname(manifestPath), { recursive: true });
|
|
|
|
try {
|
|
if (await pathExists(outputRoot)) {
|
|
await rename(outputRoot, previousOutput);
|
|
movedPreviousOutput = true;
|
|
}
|
|
if (await pathExists(manifestPath)) {
|
|
await rename(manifestPath, previousManifest);
|
|
movedPreviousManifest = true;
|
|
}
|
|
|
|
await rename(stagedOutputRoot, outputRoot);
|
|
publishedOutput = true;
|
|
await rename(stagedManifestPath, manifestPath);
|
|
publishedManifest = true;
|
|
} catch (error) {
|
|
if (publishedManifest) {
|
|
await rm(manifestPath, { force: true });
|
|
}
|
|
if (publishedOutput) {
|
|
await rm(outputRoot, { recursive: true, force: true });
|
|
}
|
|
if (movedPreviousOutput) {
|
|
await rename(previousOutput, outputRoot);
|
|
}
|
|
if (movedPreviousManifest) {
|
|
await rename(previousManifest, manifestPath);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function fetchWithAuthorizedRedirects(startUrl, {
|
|
fetchImpl,
|
|
maxRedirects,
|
|
pace,
|
|
}) {
|
|
let currentUrl = startUrl;
|
|
const visited = new Set([currentUrl]);
|
|
|
|
for (let redirects = 0; ; redirects += 1) {
|
|
await pace();
|
|
const response = await fetchImpl(currentUrl, {
|
|
headers: {
|
|
accept: 'text/html,application/xhtml+xml',
|
|
'user-agent': 'Incognito-Wiki research capture (+https://msvincognito.nl/)',
|
|
},
|
|
redirect: 'manual',
|
|
});
|
|
|
|
if (!response) {
|
|
return { response, url: currentUrl };
|
|
}
|
|
|
|
if (![301, 302, 303, 307, 308].includes(response.status)) {
|
|
const responseUrl = response.url
|
|
? normalizeWikiUrl(response.url, currentUrl)
|
|
: currentUrl;
|
|
if (!responseUrl) {
|
|
throw new Error(`Final response URL is not authorized: ${response.url}`);
|
|
}
|
|
return { response, url: responseUrl };
|
|
}
|
|
|
|
const location = response.headers.get('location');
|
|
if (!location) {
|
|
throw new Error(`HTTP ${response.status} redirect is missing a Location header`);
|
|
}
|
|
const redirectUrl = normalizeWikiUrl(location, currentUrl);
|
|
if (!redirectUrl) {
|
|
throw new Error(`Redirect target is not an authorized content URL: ${location}`);
|
|
}
|
|
if (redirects >= maxRedirects) {
|
|
throw new Error(`Redirect limit of ${maxRedirects} exceeded at ${redirectUrl}`);
|
|
}
|
|
if (visited.has(redirectUrl)) {
|
|
throw new Error(`Redirect loop detected at ${redirectUrl}`);
|
|
}
|
|
|
|
visited.add(redirectUrl);
|
|
currentUrl = redirectUrl;
|
|
}
|
|
}
|
|
|
|
async function pathExists(candidate) {
|
|
try {
|
|
await access(candidate);
|
|
return true;
|
|
} catch (error) {
|
|
if (error?.code === 'ENOENT') {
|
|
return false;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function cleanText(value = '') {
|
|
return value.replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function removeLiveWikiChrome(content) {
|
|
for (const header of content.querySelectorAll('header[class]')) {
|
|
const classes = new Set(header.classList);
|
|
if (classes.has('sticky') && classes.has('top-0') && classes.has('z-40')) {
|
|
header.remove();
|
|
}
|
|
}
|
|
|
|
for (const element of content.querySelectorAll('[class]')) {
|
|
const classes = new Set(element.classList);
|
|
const text = cleanText(element.textContent);
|
|
|
|
const previousNextPanel = element.querySelector('.border-t.pt-6');
|
|
const isPreviousNextNavigation = classes.has('mt-16')
|
|
&& previousNextPanel?.querySelector('a[href]');
|
|
const isRepeatedPageFooter = classes.has('hidden')
|
|
&& classes.has('text-sm')
|
|
&& classes.has('lg:block')
|
|
&& /Star on GitHub|Create Issues/.test(text);
|
|
const isGlobalSiteFooter = element.localName === 'footer'
|
|
&& classes.has('text-muted-foreground')
|
|
&& classes.has('py-6')
|
|
&& classes.has('md:px-8')
|
|
&& classes.has('md:py-0')
|
|
&& /Copyright © \d{4} MSV Incognito/.test(text);
|
|
|
|
if (isPreviousNextNavigation || isRepeatedPageFooter || isGlobalSiteFooter) {
|
|
element.remove();
|
|
}
|
|
}
|
|
}
|
|
|
|
function validatePathSegment(segment) {
|
|
if (!/^[A-Za-z0-9._-]+$/.test(segment) || segment === '.' || segment === '..') {
|
|
throw new Error(`Unsafe live-wiki route segment: ${segment}`);
|
|
}
|
|
|
|
return segment;
|
|
}
|
|
|
|
function titleFromUrl(url) {
|
|
const { pathname } = new URL(url);
|
|
const lastSegment = pathname.split('/').filter(Boolean).at(-1) ?? 'Home';
|
|
return decodeURIComponent(lastSegment)
|
|
.replace(/[-_]+/g, ' ')
|
|
.replace(/\b\p{Letter}/gu, (letter) => letter.toUpperCase());
|
|
}
|
|
|
|
function discover(input, relativeTo, discovered, queue, maxPages) {
|
|
const url = normalizeWikiUrl(input, relativeTo);
|
|
if (!url || discovered.has(url)) {
|
|
return;
|
|
}
|
|
|
|
discovered.add(url);
|
|
if (discovered.size > maxPages) {
|
|
throw new PageLimitError(`Discovered more than ${maxPages} content pages; refusing to truncate the crawl.`);
|
|
}
|
|
queue.push(url);
|
|
}
|
|
|
|
function compareByUrl(left, right) {
|
|
return left.url.localeCompare(right.url);
|
|
}
|
|
|
|
function toPosixPath(input) {
|
|
return input.split(path.sep).join('/');
|
|
}
|
|
|
|
function wait(milliseconds) {
|
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|