41 lines
1.3 KiB
JavaScript
41 lines
1.3 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { execFile } from 'node:child_process';
|
|
import { resolve } from 'node:path';
|
|
import { promisify } from 'node:util';
|
|
|
|
import { crawlPreviousWiki } from './lib/previous-wiki.mjs';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
async function fetchOrigin(path) {
|
|
const { stdout } = await execFileAsync('curl', [
|
|
'--resolve', 'msvincognito.nl:443:185.104.29.94',
|
|
'-k',
|
|
'-L',
|
|
'--fail-with-body',
|
|
'--max-time', '30',
|
|
'-sS',
|
|
`https://msvincognito.nl${path}`,
|
|
], { maxBuffer: 16 * 1024 * 1024 });
|
|
return stdout;
|
|
}
|
|
|
|
const outputRoot = resolve('to-be-studied/previous-wiki/2026-08-03');
|
|
let lastReported = '';
|
|
const manifest = await crawlPreviousWiki({
|
|
capturedAt: '2026-08-03',
|
|
outputRoot,
|
|
fetchText: fetchOrigin,
|
|
onProgress({ phase, completed, discovered }) {
|
|
const shouldReport = completed === discovered || completed % 25 === 0;
|
|
const message = `${phase}: ${completed}/${discovered}`;
|
|
if (shouldReport && message !== lastReported) {
|
|
console.log(message);
|
|
lastReported = message;
|
|
}
|
|
},
|
|
});
|
|
|
|
console.log(`Previous wiki captured: ${manifest.pages.length} pages, ${manifest.namespaces.length} sitemap namespaces, ${manifest.failures.length} failures.`);
|
|
if (manifest.failures.length > 0) process.exitCode = 1;
|