311 lines
8.3 KiB
JavaScript
311 lines
8.3 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
import { parseHTML } from 'linkedom';
|
|
import TurndownService from 'turndown';
|
|
|
|
export const LIVE_WIKI_BASE_URL = 'https://wiki.msvincognito.nl/';
|
|
|
|
const ASSET_EXTENSION = /\.(?:avif|bmp|css|csv|docx?|eot|gif|ico|jpe?g|js|json|map|mp3|mp4|ogg|otf|pdf|png|pptx?|rar|rss|svg|tar|tiff?|tsv|txt|wav|webm|webmanifest|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
const NON_CONTENT_SELECTOR = [
|
|
'script',
|
|
'style',
|
|
'header',
|
|
'nav',
|
|
'aside',
|
|
'form',
|
|
'button',
|
|
'footer',
|
|
'noscript',
|
|
'iframe',
|
|
'dialog',
|
|
'[role="navigation"]',
|
|
'[role="contentinfo"]',
|
|
].join(',');
|
|
|
|
class PageLimitError extends Error {}
|
|
class SoftNotFoundError extends Error {}
|
|
|
|
export function normalizeWikiUrl(input, relativeTo = LIVE_WIKI_BASE_URL) {
|
|
let url;
|
|
|
|
try {
|
|
url = new URL(input, relativeTo);
|
|
} catch {
|
|
return null;
|
|
}
|
|
|
|
const baseUrl = new URL(LIVE_WIKI_BASE_URL);
|
|
if (url.origin !== baseUrl.origin || !['http:', 'https:'].includes(url.protocol)) {
|
|
return null;
|
|
}
|
|
|
|
url.search = '';
|
|
url.hash = '';
|
|
url.pathname = url.pathname.replace(/\/{2,}/g, '/');
|
|
|
|
if (url.pathname !== '/') {
|
|
url.pathname = url.pathname.replace(/\/+$/, '');
|
|
}
|
|
|
|
if (ASSET_EXTENSION.test(url.pathname)) {
|
|
return null;
|
|
}
|
|
|
|
return url.href;
|
|
}
|
|
|
|
export function routeToCapturePath(input) {
|
|
const normalizedUrl = normalizeWikiUrl(input);
|
|
if (!normalizedUrl) {
|
|
throw new TypeError(`Cannot map non-content URL to a capture path: ${input}`);
|
|
}
|
|
|
|
const { pathname } = new URL(normalizedUrl);
|
|
if (pathname === '/') {
|
|
return 'index.md';
|
|
}
|
|
|
|
const route = pathname
|
|
.split('/')
|
|
.filter(Boolean)
|
|
.map((segment) => sanitizePathSegment(segment))
|
|
.join('/');
|
|
|
|
return `${route}.md`;
|
|
}
|
|
|
|
export function extractPage(html, url) {
|
|
const { document } = parseHTML(html);
|
|
const content = document.querySelector('main')
|
|
?? document.querySelector('article')
|
|
?? document.body;
|
|
|
|
if (!content) {
|
|
throw new Error(`No readable content root found at ${url}`);
|
|
}
|
|
|
|
const links = new Set();
|
|
const currentUrl = normalizeWikiUrl(url);
|
|
for (const anchor of content.querySelectorAll('a[href]')) {
|
|
const normalizedLink = normalizeWikiUrl(anchor.getAttribute('href'), url);
|
|
if (normalizedLink && normalizedLink !== currentUrl) {
|
|
links.add(normalizedLink);
|
|
anchor.setAttribute('href', normalizedLink);
|
|
}
|
|
}
|
|
|
|
const heading = content.querySelector('h1, h2, h3');
|
|
const title = cleanText(heading?.textContent)
|
|
|| cleanText(document.querySelector('title')?.textContent)
|
|
|| titleFromUrl(url);
|
|
|
|
for (const element of content.querySelectorAll(NON_CONTENT_SELECTOR)) {
|
|
element.remove();
|
|
}
|
|
removeLiveWikiChrome(content);
|
|
|
|
const turndown = new TurndownService({
|
|
bulletListMarker: '-',
|
|
codeBlockStyle: 'fenced',
|
|
emDelimiter: '*',
|
|
headingStyle: 'atx',
|
|
});
|
|
turndown.addRule('lineBreakWithoutTrailingWhitespace', {
|
|
filter: 'br',
|
|
replacement: () => '<br>\n',
|
|
});
|
|
const markdown = `${turndown.turndown(content).trim().replace(/[ \t]+$/gm, '')}\n`;
|
|
|
|
return {
|
|
title,
|
|
markdown,
|
|
links: [...links].sort(),
|
|
};
|
|
}
|
|
|
|
export async function crawlWiki({
|
|
capturedAt = new Date().toISOString().slice(0, 10),
|
|
delayMs = 150,
|
|
fetchImpl = globalThis.fetch,
|
|
manifestPath,
|
|
maxPages = 500,
|
|
outputRoot,
|
|
startUrls = [LIVE_WIKI_BASE_URL],
|
|
waitImpl = wait,
|
|
} = {}) {
|
|
if (!outputRoot || !manifestPath) {
|
|
throw new TypeError('crawlWiki requires outputRoot and manifestPath.');
|
|
}
|
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(capturedAt)) {
|
|
throw new TypeError('capturedAt must use YYYY-MM-DD format.');
|
|
}
|
|
if (!Number.isInteger(maxPages) || maxPages < 1) {
|
|
throw new TypeError('maxPages must be a positive integer.');
|
|
}
|
|
|
|
const queue = [];
|
|
const discovered = new Set();
|
|
for (const startUrl of startUrls) {
|
|
discover(startUrl, LIVE_WIKI_BASE_URL, discovered, queue, maxPages);
|
|
}
|
|
|
|
const pages = [];
|
|
const failures = [];
|
|
let requestCount = 0;
|
|
|
|
await mkdir(outputRoot, { recursive: true });
|
|
|
|
while (queue.length > 0) {
|
|
queue.sort();
|
|
const url = queue.shift();
|
|
|
|
if (requestCount > 0) {
|
|
await waitImpl(delayMs);
|
|
}
|
|
requestCount += 1;
|
|
const capturePath = routeToCapturePath(url);
|
|
const destination = path.join(outputRoot, ...capturePath.split('/'));
|
|
|
|
try {
|
|
const response = await fetchImpl(url, {
|
|
headers: {
|
|
accept: 'text/html,application/xhtml+xml',
|
|
'user-agent': 'Incognito-Wiki research capture (+https://msvincognito.nl/)',
|
|
},
|
|
redirect: 'follow',
|
|
});
|
|
|
|
if (!response) {
|
|
throw new Error('No response');
|
|
}
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());
|
|
}
|
|
|
|
const contentType = response.headers.get('content-type') ?? '';
|
|
if (contentType && !/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
|
|
throw new Error(`Unexpected content type: ${contentType}`);
|
|
}
|
|
|
|
const page = extractPage(await response.text(), url);
|
|
if (page.title === '404' && /This page could not be found\./i.test(page.markdown)) {
|
|
throw new SoftNotFoundError('Soft 404: This page could not be found.');
|
|
}
|
|
for (const link of page.links) {
|
|
discover(link, url, discovered, queue, maxPages);
|
|
}
|
|
|
|
const content = `> Source: ${url}\n> Captured: ${capturedAt}\n\n${page.markdown}`;
|
|
|
|
await mkdir(path.dirname(destination), { recursive: true });
|
|
await writeFile(destination, content, 'utf8');
|
|
|
|
pages.push({
|
|
url,
|
|
title: page.title,
|
|
path: toPosixPath(path.relative(path.dirname(manifestPath), destination)),
|
|
contentHash: createHash('sha256').update(content).digest('hex'),
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof PageLimitError) {
|
|
throw error;
|
|
}
|
|
await rm(destination, { force: true });
|
|
if (error instanceof SoftNotFoundError) {
|
|
continue;
|
|
}
|
|
failures.push({
|
|
url,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
const manifest = {
|
|
capturedAt,
|
|
baseUrl: LIVE_WIKI_BASE_URL,
|
|
pages: pages.sort(compareByUrl),
|
|
failures: failures.sort(compareByUrl),
|
|
};
|
|
|
|
await mkdir(path.dirname(manifestPath), { recursive: true });
|
|
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
|
|
|
|
return manifest;
|
|
}
|
|
|
|
function cleanText(value = '') {
|
|
return value.replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function removeLiveWikiChrome(content) {
|
|
for (const element of content.querySelectorAll('[class]')) {
|
|
const classes = new Set(element.classList);
|
|
const text = cleanText(element.textContent);
|
|
|
|
const isPreviousNextNavigation = classes.has('mt-16');
|
|
const isRepeatedPageFooter = classes.has('hidden')
|
|
&& classes.has('text-sm')
|
|
&& classes.has('lg:block')
|
|
&& /Star on GitHub|Create Issues/.test(text);
|
|
|
|
if (isPreviousNextNavigation || isRepeatedPageFooter) {
|
|
element.remove();
|
|
}
|
|
}
|
|
}
|
|
|
|
function sanitizePathSegment(segment) {
|
|
let decoded;
|
|
try {
|
|
decoded = decodeURIComponent(segment);
|
|
} catch {
|
|
decoded = segment;
|
|
}
|
|
|
|
const safe = decoded
|
|
.normalize('NFKC')
|
|
.replace(/[^\p{Letter}\p{Number}._-]+/gu, '-')
|
|
.replace(/^-+|-+$/g, '');
|
|
|
|
if (!safe || safe === '.' || safe === '..') {
|
|
throw new Error(`Unsafe live-wiki route segment: ${segment}`);
|
|
}
|
|
|
|
return safe;
|
|
}
|
|
|
|
function titleFromUrl(url) {
|
|
const { pathname } = new URL(url);
|
|
const lastSegment = pathname.split('/').filter(Boolean).at(-1) ?? 'Home';
|
|
return decodeURIComponent(lastSegment)
|
|
.replace(/[-_]+/g, ' ')
|
|
.replace(/\b\p{Letter}/gu, (letter) => letter.toUpperCase());
|
|
}
|
|
|
|
function discover(input, relativeTo, discovered, queue, maxPages) {
|
|
const url = normalizeWikiUrl(input, relativeTo);
|
|
if (!url || discovered.has(url)) {
|
|
return;
|
|
}
|
|
|
|
discovered.add(url);
|
|
if (discovered.size > maxPages) {
|
|
throw new PageLimitError(`Discovered more than ${maxPages} content pages; refusing to truncate the crawl.`);
|
|
}
|
|
queue.push(url);
|
|
}
|
|
|
|
function compareByUrl(left, right) {
|
|
return left.url.localeCompare(right.url);
|
|
}
|
|
|
|
function toPosixPath(input) {
|
|
return input.split(path.sep).join('/');
|
|
}
|
|
|
|
function wait(milliseconds) {
|
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|