import { createHash } from 'node:crypto';
import { mkdir, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';

import { parseHTML } from 'linkedom';
import TurndownService from 'turndown';

export const LIVE_WIKI_BASE_URL = 'https://wiki.msvincognito.nl/';

const ASSET_EXTENSION = /\.(?:avif|bmp|css|csv|docx?|eot|gif|ico|jpe?g|js|json|map|mp3|mp4|ogg|otf|pdf|png|pptx?|rar|rss|svg|tar|tiff?|tsv|txt|wav|webm|webmanifest|webp|woff2?|xlsx?|xml|zip)$/i;
const NON_CONTENT_SELECTOR = [
  'script',
  'style',
  'header',
  'nav',
  'aside',
  'form',
  'button',
  'footer',
  'noscript',
  'iframe',
  'dialog',
  '[role="navigation"]',
  '[role="contentinfo"]',
].join(',');

class PageLimitError extends Error {}
class SoftNotFoundError extends Error {}

export function normalizeWikiUrl(input, relativeTo = LIVE_WIKI_BASE_URL) {
  let url;

  try {
    url = new URL(input, relativeTo);
  } catch {
    return null;
  }

  const baseUrl = new URL(LIVE_WIKI_BASE_URL);
  if (url.origin !== baseUrl.origin || !['http:', 'https:'].includes(url.protocol)) {
    return null;
  }

  url.search = '';
  url.hash = '';
  url.pathname = url.pathname.replace(/\/{2,}/g, '/');

  if (url.pathname !== '/') {
    url.pathname = url.pathname.replace(/\/+$/, '');
  }

  if (ASSET_EXTENSION.test(url.pathname)) {
    return null;
  }

  return url.href;
}

export function routeToCapturePath(input) {
  const normalizedUrl = normalizeWikiUrl(input);
  if (!normalizedUrl) {
    throw new TypeError(`Cannot map non-content URL to a capture path: ${input}`);
  }

  const { pathname } = new URL(normalizedUrl);
  if (pathname === '/') {
    return 'index.md';
  }

  const route = pathname
    .split('/')
    .filter(Boolean)
    .map((segment) => sanitizePathSegment(segment))
    .join('/');

  return `${route}.md`;
}

export function extractPage(html, url) {
  const { document } = parseHTML(html);
  const content = document.querySelector('main')
    ?? document.querySelector('article')
    ?? document.body;

  if (!content) {
    throw new Error(`No readable content root found at ${url}`);
  }

  const links = new Set();
  const currentUrl = normalizeWikiUrl(url);
  for (const anchor of content.querySelectorAll('a[href]')) {
    const normalizedLink = normalizeWikiUrl(anchor.getAttribute('href'), url);
    if (normalizedLink && normalizedLink !== currentUrl) {
      links.add(normalizedLink);
      anchor.setAttribute('href', normalizedLink);
    }
  }

  const heading = content.querySelector('h1, h2, h3');
  const title = cleanText(heading?.textContent)
    || cleanText(document.querySelector('title')?.textContent)
    || titleFromUrl(url);

  for (const element of content.querySelectorAll(NON_CONTENT_SELECTOR)) {
    element.remove();
  }
  removeLiveWikiChrome(content);

  const turndown = new TurndownService({
    bulletListMarker: '-',
    codeBlockStyle: 'fenced',
    emDelimiter: '*',
    headingStyle: 'atx',
  });
  turndown.addRule('lineBreakWithoutTrailingWhitespace', {
    filter: 'br',
    replacement: () => '<br>\n',
  });
  const markdown = `${turndown.turndown(content).trim().replace(/[ \t]+$/gm, '')}\n`;

  return {
    title,
    markdown,
    links: [...links].sort(),
  };
}

export async function crawlWiki({
  capturedAt = new Date().toISOString().slice(0, 10),
  delayMs = 150,
  fetchImpl = globalThis.fetch,
  manifestPath,
  maxPages = 500,
  outputRoot,
  startUrls = [LIVE_WIKI_BASE_URL],
  waitImpl = wait,
} = {}) {
  if (!outputRoot || !manifestPath) {
    throw new TypeError('crawlWiki requires outputRoot and manifestPath.');
  }
  if (!/^\d{4}-\d{2}-\d{2}$/.test(capturedAt)) {
    throw new TypeError('capturedAt must use YYYY-MM-DD format.');
  }
  if (!Number.isInteger(maxPages) || maxPages < 1) {
    throw new TypeError('maxPages must be a positive integer.');
  }

  const queue = [];
  const discovered = new Set();
  for (const startUrl of startUrls) {
    discover(startUrl, LIVE_WIKI_BASE_URL, discovered, queue, maxPages);
  }

  const pages = [];
  const failures = [];
  let requestCount = 0;

  await mkdir(outputRoot, { recursive: true });

  while (queue.length > 0) {
    queue.sort();
    const url = queue.shift();

    if (requestCount > 0) {
      await waitImpl(delayMs);
    }
    requestCount += 1;
    const capturePath = routeToCapturePath(url);
    const destination = path.join(outputRoot, ...capturePath.split('/'));

    try {
      const response = await fetchImpl(url, {
        headers: {
          accept: 'text/html,application/xhtml+xml',
          'user-agent': 'Incognito-Wiki research capture (+https://msvincognito.nl/)',
        },
        redirect: 'follow',
      });

      if (!response) {
        throw new Error('No response');
      }
      if (!response.ok) {
        throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());
      }

      const contentType = response.headers.get('content-type') ?? '';
      if (contentType && !/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
        throw new Error(`Unexpected content type: ${contentType}`);
      }

      const page = extractPage(await response.text(), url);
      if (page.title === '404' && /This page could not be found\./i.test(page.markdown)) {
        throw new SoftNotFoundError('Soft 404: This page could not be found.');
      }
      for (const link of page.links) {
        discover(link, url, discovered, queue, maxPages);
      }

      const content = `> Source: ${url}\n> Captured: ${capturedAt}\n\n${page.markdown}`;

      await mkdir(path.dirname(destination), { recursive: true });
      await writeFile(destination, content, 'utf8');

      pages.push({
        url,
        title: page.title,
        path: toPosixPath(path.relative(path.dirname(manifestPath), destination)),
        contentHash: createHash('sha256').update(content).digest('hex'),
      });
    } catch (error) {
      if (error instanceof PageLimitError) {
        throw error;
      }
      await rm(destination, { force: true });
      if (error instanceof SoftNotFoundError) {
        continue;
      }
      failures.push({
        url,
        error: error instanceof Error ? error.message : String(error),
      });
    }
  }

  const manifest = {
    capturedAt,
    baseUrl: LIVE_WIKI_BASE_URL,
    pages: pages.sort(compareByUrl),
    failures: failures.sort(compareByUrl),
  };

  await mkdir(path.dirname(manifestPath), { recursive: true });
  await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');

  return manifest;
}

function cleanText(value = '') {
  return value.replace(/\s+/g, ' ').trim();
}

function removeLiveWikiChrome(content) {
  for (const element of content.querySelectorAll('[class]')) {
    const classes = new Set(element.classList);
    const text = cleanText(element.textContent);

    const isPreviousNextNavigation = classes.has('mt-16');
    const isRepeatedPageFooter = classes.has('hidden')
      && classes.has('text-sm')
      && classes.has('lg:block')
      && /Star on GitHub|Create Issues/.test(text);

    if (isPreviousNextNavigation || isRepeatedPageFooter) {
      element.remove();
    }
  }
}

function sanitizePathSegment(segment) {
  let decoded;
  try {
    decoded = decodeURIComponent(segment);
  } catch {
    decoded = segment;
  }

  const safe = decoded
    .normalize('NFKC')
    .replace(/[^\p{Letter}\p{Number}._-]+/gu, '-')
    .replace(/^-+|-+$/g, '');

  if (!safe || safe === '.' || safe === '..') {
    throw new Error(`Unsafe live-wiki route segment: ${segment}`);
  }

  return safe;
}

function titleFromUrl(url) {
  const { pathname } = new URL(url);
  const lastSegment = pathname.split('/').filter(Boolean).at(-1) ?? 'Home';
  return decodeURIComponent(lastSegment)
    .replace(/[-_]+/g, ' ')
    .replace(/\b\p{Letter}/gu, (letter) => letter.toUpperCase());
}

function discover(input, relativeTo, discovered, queue, maxPages) {
  const url = normalizeWikiUrl(input, relativeTo);
  if (!url || discovered.has(url)) {
    return;
  }

  discovered.add(url);
  if (discovered.size > maxPages) {
    throw new PageLimitError(`Discovered more than ${maxPages} content pages; refusing to truncate the crawl.`);
  }
  queue.push(url);
}

function compareByUrl(left, right) {
  return left.url.localeCompare(right.url);
}

function toPosixPath(input) {
  return input.split(path.sep).join('/');
}

function wait(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}