Incognito-Wiki/scripts/lib/previous-wiki.mjs

import { parseHTML } from 'linkedom';
import { createHash } from 'node:crypto';
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';

const PROGRAMMES = new Map([
  ['bachelor', 'bachelor'],
  ['master_ai', 'master-ai'],
  ['master_dsdm', 'master-dsdm'],
]);

function slug(value) {
  return value.toLowerCase().replaceAll('_', '-').replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '');
}

export function extractSitemapEntries(html) {
  const { document } = parseHTML(html);
  const tree = document.querySelector('#index__tree');
  if (!tree) throw new Error('DokuWiki sitemap response is missing #index__tree');
  const namespaces = new Set();
  const pageIds = new Set();
  for (const anchor of tree.querySelectorAll('a.idx_dir')) {
    const href = anchor.getAttribute('href');
    if (!href) continue;
    const url = new URL(href, 'https://msvincognito.nl');
    const namespace = url.searchParams.get('idx');
    if (namespace) namespaces.add(namespace);
  }
  for (const anchor of tree.querySelectorAll('a[data-wiki-id]')) {
    const id = anchor.getAttribute('data-wiki-id')?.trim();
    if (id) pageIds.add(id);
  }
  return {
    namespaces: [...namespaces].sort(),
    pageIds: [...pageIds].sort(),
  };
}

export function destinationForPageId(pageId) {
  const parts = pageId.split(':');
  if (parts[0] !== 'study' || !PROGRAMMES.has(parts[1]) || !/^year_\d+$/.test(parts[2] ?? '') || parts.length < 4) {
    return null;
  }
  const routeParts = [PROGRAMMES.get(parts[1]), ...parts.slice(2).map(slug)];
  return `src/content/docs/${routeParts.join('/')}.md`;
}

export function classifyPreviousPage(pageId, representedPageIds = new Set(), source = '') {
  if (representedPageIds.has(pageId)) return 'represented';
  if (pageId.endsWith(':placeholder') && /This course is a placeholder/i.test(source)) return 'empty';
  return destinationForPageId(pageId) ? 'migrate' : 'excluded';
}

function humanLabel(value) {
  return value
    .replace(/\.pdf$/i, '')
    .replaceAll('_', ' ')
    .replace(/\b\w/g, (character) => character.toUpperCase());
}

function markdownRouteForPageId(pageId) {
  const destination = destinationForPageId(pageId);
  return destination
    ? `/${destination.replace(/^src\/content\/docs\//, '').replace(/\.md$/, '')}/`
    : `https://msvincognito.nl/wiki/${pageId.replaceAll(':', '/')}`;
}

export function convertCourseSource({ pageId, sourceUrl, source, pdfs }) {
  const titleMatch = source.match(/^={2,6}\s*(.*?)\s*={2,6}\s*$/m);
  if (!titleMatch) throw new Error(`course source has no DokuWiki heading: ${pageId}`);
  const sourceTitle = titleMatch[1].trim();
  const title = /^Course Title$/i.test(sourceTitle) ? humanLabel(pageId.split(':').at(-1)) : sourceTitle;
  let body = source.replace(titleMatch[0], '').trim();

  body = body.replace(/\{\{(?:filelist|medialist)>([^}]+)\}\}/g, (_match, expression) => {
    let namespace = expression.split(/[&*]/, 1)[0].replace(/^:/, '').replace(/:$/, '');
    if (namespace.includes('@NS@') || namespace.includes('@PAGE@')) namespace = pageId;
    const matches = pdfs.filter(({ mediaId }) => mediaId.startsWith(`${namespace}:`));
    if (matches.length === 0) return 'No recovered PDF files were found for this source folder.';
    return matches
      .map(({ mediaId, publicUrl }) => `- [${humanLabel(mediaId.split(':').at(-1))}](${publicUrl})`)
      .join('\n');
  });
  body = body
    .replace(/^={2,6}\s*(.*?)\s*={2,6}\s*$/gm, '## $1')
    .replace(/\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g, (_match, target, text) => {
      const href = /^https?:\/\//.test(target) ? target : markdownRouteForPageId(target);
      return `[${text || target}](${href})`;
    })
    .replace(/^\s+([*-])\s+/gm, '$1 ')
    .replace(/\\\\\s*$/gm, '')
    .replace(/^~~NOCACHE~~\s*$/gm, '')
    .replace(/\{\{[^}]+\}\}/g, '')
    .replace(/\n{3,}/g, '\n\n')
    .trim();

  return `---
title: ${title}
description: Historical course details for ${title}, recovered from the previous Incognito wiki.
---

:::caution[Historical information]
This information originated in the previous wiki and may be outdated. Check the current Maastricht University course information before relying on it.
:::

${body}

---

Source: [Previous wiki page](${sourceUrl})
`;
}

function pagePath(outputRoot, pageId) {
  const parts = pageId.split(':');
  if (parts.some((part) => !part || part === '.' || part === '..' || part.includes('/') || part.includes('\\'))) {
    throw new Error(`unsafe DokuWiki page ID: ${pageId}`);
  }
  return join(outputRoot, 'pages', ...parts.slice(0, -1), `${parts.at(-1)}.txt`);
}

export async function crawlPreviousWiki({ capturedAt, outputRoot, fetchText, onProgress = () => {} }) {
  const queuedNamespaces = [''];
  const seenNamespaces = new Set();
  const pageIds = new Set();

  while (queuedNamespaces.length > 0) {
    queuedNamespaces.sort();
    const namespace = queuedNamespaces.shift();
    if (seenNamespaces.has(namespace)) continue;
    seenNamespaces.add(namespace);
    const query = namespace ? `&idx=${encodeURIComponent(namespace)}` : '';
    const html = await fetchText(`/wiki/start?do=index${query}`);
    const entries = extractSitemapEntries(html);
    onProgress({ phase: 'sitemap', completed: seenNamespaces.size, discovered: queuedNamespaces.length + seenNamespaces.size });
    for (const child of entries.namespaces) {
      if (!seenNamespaces.has(child)) queuedNamespaces.push(child);
    }
    for (const id of entries.pageIds) pageIds.add(id);
  }

  const pages = [];
  const failures = [];
  for (const id of [...pageIds].sort()) {
    const url = `https://msvincognito.nl/wiki/${id.replaceAll(':', '/')}`;
    const rawPath = `/wiki/_export/raw/${encodeURIComponent(id)}`;
    try {
      const source = await fetchText(rawPath);
      const localPath = pagePath(outputRoot, id);
      await mkdir(join(localPath, '..'), { recursive: true });
      await writeFile(localPath, source);
      pages.push({
        id,
        url,
        localPath: localPath.slice(outputRoot.length + 1),
        bytes: Buffer.byteLength(source),
        sha256: createHash('sha256').update(source).digest('hex'),
      });
    } catch (error) {
      failures.push({ id, url, error: error instanceof Error ? error.message : String(error) });
    }
    onProgress({ phase: 'pages', completed: pages.length + failures.length, discovered: pageIds.size });
  }
  const manifest = {
    source: 'https://msvincognito.nl/wiki/',
    capturedAt,
    namespaces: [...seenNamespaces].sort(),
    pages,
    failures,
  };
  await mkdir(outputRoot, { recursive: true });
  await writeFile(join(outputRoot, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`);
  return manifest;
}