/**
 * Content extraction: HTML on disk -> one typed corpus document.
 *
 * The corpus is the single source of truth the whole application reads. It is
 * generated, never hand-edited, and it is the only place where "what content
 * exists" is answered. The renderer, the ranker and the sitemap all consume
 * this one file, so they cannot disagree with each other about the site.
 *
 * Deliberately dependency-free. A parser would be more correct in general, but
 * this input is a known, fixed set of files we control, and a build step that
 * cannot rot is worth more here than one that handles arbitrary HTML.
 *
 * Usage:  node build/extract-corpus.mjs <sourceDir> <outFile>
 */

import { readdir, readFile, writeFile, stat } from 'node:fs/promises';
import { join, relative, sep } from 'node:path';

/** Elements whose *content* is never prose. */
const NON_PROSE = /<(script|style|noscript|svg|template)\b[^>]*>[\s\S]*?<\/\1>/gi;

/**
 * Page chrome. Navigation, headers and footers repeat on every page, so
 * including them makes every summary begin with the same words and every
 * embedding drift toward the same point. Stripping them is the difference
 * between "Home Free Resources Getting Started Guide..." and a real sentence.
 */
const CHROME = /<(nav|header|footer|aside)\b[^>]*>[\s\S]*?<\/\1>/gi;
const TAGS = /<[^>]+>/g;

const ENTITIES = {
  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
  mdash: '—', ndash: '–', hellip: '…', rsquo: '’',
  lsquo: '‘', ldquo: '“', rdquo: '”', deg: '°', phi: 'φ',
};

function decodeEntities(s) {
  return s
    .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)))
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
    .replace(/&([a-z]+);/gi, (m, name) => ENTITIES[name.toLowerCase()] ?? m);
}

function textOf(html, stripChrome = false) {
  let h = html.replace(NON_PROSE, ' ');
  if (stripChrome) h = h.replace(CHROME, ' ');
  return decodeEntities(h.replace(TAGS, ' '))
    .replace(/\s+/g, ' ')
    .trim();
}

/**
 * Pull the first paragraph that actually reads like prose.
 *
 * "First 320 characters of the page" is the naive approach and it produces
 * navigation. A real summary is the first <p> that is long enough to be a
 * sentence and is not a breadcrumb trail.
 */
function summaryOf(html) {
  const paras = [...html.replace(NON_PROSE, ' ').replace(CHROME, ' ')
    .matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/gi)]
    .map((m) => textOf(m[1]))
    .filter((t) => t.length >= 60 && (t.match(/[.!?]/g) || []).length >= 1);
  if (paras.length) return paras[0];

  // No usable paragraph: fall back to body prose with the chrome removed.
  const body = html.match(/<(?:main|article)\b[^>]*>([\s\S]*?)<\/(?:main|article)>/i)
    ?? html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
  return body ? textOf(body[1], true) : '';
}

/** Strip the site-name prefix editors leave on every <title>. */
function cleanTitle(t) {
  return t
    .replace(/^\s*MEGAMIND\s*[|—–-]\s*/i, '')
    .replace(/\s*[|—–]\s*(MEGAMIND|feedthejoe\.com|Feed The Joe).*$/i, '')
    .replace(/^\s*(Home|Concepts|Theory)\s*\/\s*/i, '')
    .trim();
}

function firstMatch(html, re) {
  const m = html.match(re);
  return m ? textOf(m[1]) : '';
}

function allMatches(html, re) {
  return [...html.matchAll(re)].map((m) => textOf(m[1])).filter(Boolean);
}

/**
 * Topic assignment. Topics are the unit the Beta posteriors are kept over, so
 * this partition directly determines what the model can learn a preference
 * *about*. Too few and it cannot discriminate; too many and every posterior
 * stays uninformed. Five is right for this corpus size.
 */
function topicFor(relPath) {
  const p = relPath.replace(/\\/g, '/');
  if (/^ch\d+-/.test(p)) return 'chronicle';
  if (p.startsWith('concepts/')) return 'theory';
  if (p.startsWith('blog/')) return 'notes';
  if (p.startsWith('papers') || p.startsWith('theory/') || p.startsWith('model/')) return 'research';
  return 'profile';
}

function kindFor(relPath) {
  const p = relPath.replace(/\\/g, '/');
  if (/^ch\d+-/.test(p)) return 'chapter';
  if (p.startsWith('concepts/')) return 'concept';
  if (p.startsWith('blog/')) return 'post';
  return 'page';
}

/** Files that are infrastructure, not content. */
const EXCLUDE = [
  /\.bak/i, /^404\./, /^sitemap\./, /backup/i,
  /^index\.html\./, /\.pre-/,
  // Off-entity content. An automated SEO pass read the domain name as a food
  // brand and generated a catering content farm here: wedding catering, private
  // chef, meal prep. It is not Joseph's work and it actively confuses entity
  // resolution — Google cannot hold "SDVOSB software engineer" and "wedding
  // caterer" as one entity. Excluded from the corpus; archived, not deleted.
  /^guides\//, /^blog\//,
];

async function* walk(dir) {
  for (const entry of await readdir(dir, { withFileTypes: true })) {
    const full = join(dir, entry.name);
    if (entry.isDirectory()) {
      if (entry.name.startsWith('.') || entry.name === 'assets') continue;
      yield* walk(full);
    } else if (entry.name.endsWith('.html')) {
      yield full;
    }
  }
}

/** Turn a file path into the URL nginx will serve it at (try_files gives us extensionless). */
function urlFor(relPath) {
  const p = relPath.replace(/\\/g, '/');
  if (p === 'index.html') return '/';
  if (p.endsWith('/index.html')) return `/${p.slice(0, -'index.html'.length)}`;
  return `/${p.replace(/\.html$/, '')}`;
}

async function main() {
  const [sourceDir, outFile] = process.argv.slice(2);
  if (!sourceDir || !outFile) {
    console.error('usage: node build/extract-corpus.mjs <sourceDir> <outFile>');
    process.exit(1);
  }

  const items = [];
  const skipped = [];

  for await (const file of walk(sourceDir)) {
    const rel = relative(sourceDir, file).split(sep).join('/');
    if (EXCLUDE.some((re) => re.test(rel))) { skipped.push(rel); continue; }

    const html = await readFile(file, 'utf8');
    const info = await stat(file);

    const title = firstMatch(html, /<title[^>]*>([\s\S]*?)<\/title>/i);
    const h1 = firstMatch(html, /<h1[^>]*>([\s\S]*?)<\/h1>/i);
    const headings = allMatches(html, /<h2[^>]*>([\s\S]*?)<\/h2>/gi).slice(0, 12);

    // Prose: prefer <main>/<article> when present, else the whole body.
    const bodyMatch = html.match(/<(?:main|article)\b[^>]*>([\s\S]*?)<\/(?:main|article)>/i)
      ?? html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
    const text = bodyMatch ? textOf(bodyMatch[1], true) : textOf(html, true);
    const wordCount = text ? text.split(/\s+/).length : 0;

    if (wordCount < 120) { skipped.push(`${rel} (thin: ${wordCount} words)`); continue; }

    // Template shells — "Free Resources", "Sponsor With Us", "Explore X" — are
    // navigation pages wearing an article's clothes. They carry no work, and on
    // an identity anchor they read as filler.
    if (/^(free resources|sponsor with us|explore|get in touch|about megamind|megamind capabilities|guides? (&|and) resources)/i
        .test(cleanTitle(h1 || title || ''))) {
      skipped.push(`${rel} (template shell)`);
      continue;
    }

    items.push({
      id: rel.replace(/\.html$/, '').replace(/\//g, '--'),
      url: urlFor(rel),
      source: rel,
      title: cleanTitle(h1 || title || rel).slice(0, 200) || cleanTitle(title) || rel,
      documentTitle: title,
      topic: topicFor(rel),
      kind: kindFor(rel),
      headings,
      // The text the embedder sees. Title and headings are repeated at the head
      // because they carry disproportionate topical signal for a 256-token model.
      embedText: [title, h1, ...headings, text].filter(Boolean).join('. ').slice(0, 4000),
      summary: (summaryOf(html) || text).slice(0, 300),
      wordCount,
      publishedMs: info.mtimeMs,
    });
  }

  items.sort((a, b) => a.id.localeCompare(b.id));

  const corpus = {
    generatedAt: new Date().toISOString(),
    itemCount: items.length,
    topics: [...new Set(items.map((i) => i.topic))].sort(),
    items,
  };

  await writeFile(outFile, `${JSON.stringify(corpus, null, 2)}\n`);

  const byTopic = {};
  for (const i of items) byTopic[i.topic] = (byTopic[i.topic] ?? 0) + 1;

  console.log(`corpus: ${items.length} items -> ${outFile}`);
  console.log(`topics: ${Object.entries(byTopic).map(([k, v]) => `${k}=${v}`).join('  ')}`);
  console.log(`words:  ${items.reduce((s, i) => s + i.wordCount, 0).toLocaleString()} total, ` +
    `median ${items.map((i) => i.wordCount).sort((a, b) => a - b)[items.length >> 1]}`);
  console.log(`skipped ${skipped.length}: ${skipped.slice(0, 6).join(', ')}${skipped.length > 6 ? ' ...' : ''}`);
}

main();