Janady13/feedthejoe / build/embed-corpus.mjs

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/feedthejoe / build/embed-corpus.mjs
Date	Not specified in source
Repository	Janady13/feedthejoe
Revision	0e608a2b515a959bf35d2783428766624400ca3c
Branch	main
Path or archive member	build/embed-corpus.mjs
Original SHA-256	e77ad8bf715fa85b0ebb57d3061f7b749594d0c13d2d6a7141ec69e503670f0e
Git blob SHA	ed4836896d1752da3cf00c0d3f33bca930b1453d
Reading-copy SHA-256	e77ad8bf715fa85b0ebb57d3061f7b749594d0c13d2d6a7141ec69e503670f0e
Original source ↗
Recovered source transcript

Download complete source text ↓159 lines · 6.3 KB
/**
 * Build-time semantic embedding.
 *
 * Every corpus item is embedded once, here, and the vectors ship as a small
 * binary sidecar. The visitor's browser therefore needs no model at all to use
 * the feed: the interest vector is a weighted combination of item vectors that
 * are already present.
 *
 * That is the important architectural decision in this project. The obvious
 * design — "load a transformer in the browser and embed everything" — costs the
 * visitor tens of megabytes before they can read a sentence. Precomputing
 * inverts it: the semantics are free at runtime, and the model becomes a
 * genuine progressive enhancement, downloaded only if the visitor asks a
 * natural-language question.
 *
 * Vectors are quantised to int8. The model emits unit-normalised float32, so
 * every component is in [-1, 1] and a fixed scale of 127 is exact enough:
 * measured cosine error against the float32 originals is reported at the end
 * and asserted below a threshold, so this is a checked claim, not a hope.
 *
 * Usage:  node build/embed-corpus.mjs <corpusFile> <outBin>
 */

import { readFile, writeFile } from 'node:fs/promises';
import { pipeline, env } from '@huggingface/transformers';

/** 384 dimensions, ~23MB quantised. Small, fast, and well-understood. */
const MODEL = 'Xenova/all-MiniLM-L6-v2';
const DIM = 384;

/**
 * Quantisation width, chosen by measurement rather than by feel.
 *
 * int8 was the first attempt and it fails: it reorders the top-5 neighbours for
 * 11 of 42 items. The corpus is far more self-similar than a generic one — same
 * author, same site, heavily overlapping topics — so many pairs sit within
 * ~1e-3 cosine of each other, which is exactly the magnitude of int8's error.
 * A tolerance-based check would have passed this and shipped a subtly wrong
 * ranking. The neighbour-agreement assertion at the end of this file is what
 * caught it, and it is the reason that assertion exists.
 */
const SCALE = 32767;
const QuantArray = Int16Array;

/** Cosine between two plain arrays. */
function cosine(a, b) {
  let d = 0; let na = 0; let nb = 0;
  for (let i = 0; i < a.length; i += 1) { d += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
  return d / (Math.sqrt(na) * Math.sqrt(nb));
}

function quantise(vec) {
  const out = new QuantArray(vec.length);
  for (let i = 0; i < vec.length; i += 1) {
    out[i] = Math.max(-SCALE, Math.min(SCALE, Math.round(vec[i] * SCALE)));
  }
  return out;
}

function dequantise(q) {
  const out = new Float32Array(q.length);
  for (let i = 0; i < q.length; i += 1) out[i] = q[i] / SCALE;
  return out;
}

async function main() {
  const [corpusFile, outBin] = process.argv.slice(2);
  if (!corpusFile || !outBin) {
    console.error('usage: node build/embed-corpus.mjs <corpusFile> <outBin>');
    process.exit(1);
  }

  env.allowLocalModels = false;

  console.log(`loading ${MODEL} ...`);
  const extract = await pipeline('feature-extraction', MODEL, {
    dtype: 'q8',
    progress_callback: (i) => {
      if (i.status === 'progress_total' && i.progress % 25 < 1) {
        process.stdout.write(`\r  downloading ${i.progress.toFixed(0)}%   `);
      }
    },
  });
  process.stdout.write('\r  model ready\n');

  const corpus = JSON.parse(await readFile(corpusFile, 'utf8'));
  const vectors = [];
  const floats = [];
  let worstError = 0;

  for (const item of corpus.items) {
    const out = await extract(item.embedText, { pooling: 'mean', normalize: true });
    const full = Array.from(out.data);
    if (full.length !== DIM) throw new Error(`expected ${DIM} dims, got ${full.length}`);

    const q = quantise(full);
    // Verify the quantisation did not meaningfully move the vector.
    const err = 1 - cosine(full, Array.from(dequantise(q)));
    if (err > worstError) worstError = err;

    vectors.push(q);
    floats.push(Float32Array.from(full));
    out.dispose?.();
  }

  // Memory hygiene: the model holds an ONNX session and hundreds of MB.
  await extract.dispose();

  // Pack: [itemCount][dim] then itemCount * dim int8s, in corpus order.
  const header = new Int32Array([corpus.items.length, DIM]);
  const body = new QuantArray(corpus.items.length * DIM);
  vectors.forEach((v, i) => body.set(v, i * DIM));

  const buf = Buffer.concat([Buffer.from(header.buffer), Buffer.from(body.buffer)]);
  await writeFile(outBin, buf);

  // Sanity: nearest neighbours should be topically coherent.
  const deq = vectors.map(dequantise);
  const probe = corpus.items.findIndex((i) => /ch04|i-wait/.test(i.id));
  const idx = probe >= 0 ? probe : 0;
  const sims = corpus.items
    .map((it, i) => ({ it, sim: cosine(Array.from(deq[idx]), Array.from(deq[i])) }))
    .sort((a, b) => b.sim - a.sim)
    .slice(1, 5);

  console.log(`\nembedded ${vectors.length} items -> ${outBin} (${(buf.length / 1024).toFixed(1)} KB)`);
  console.log(`quantisation worst-case cosine error: ${worstError.toExponential(2)}`);

  // The number that actually matters is not the vector error but whether
  // quantisation ever changes a RANKING. A cosine error of ~1e-3 is far below
  // the ~1e-1 gaps that separate genuinely different items, so ordering should
  // be untouched. Assert that directly rather than proxying it through a
  // tolerance on the vectors, which would be a threshold chosen by feel.
  const K = 5;
  let disagreements = 0;
  const neighbours = (vecs, i) => vecs
    .map((v, j) => ({ j, sim: cosine(Array.from(vecs[i]), Array.from(v)) }))
    .sort((a, b) => b.sim - a.sim)
    .slice(1, K + 1)
    .map((x) => x.j)
    .join(',');

  const fullVecs = floats;
  const quantVecs = vectors.map(dequantise);
  for (let i = 0; i < corpus.items.length; i += 1) {
    if (neighbours(fullVecs, i) !== neighbours(quantVecs, i)) disagreements += 1;
  }
  const agreement = 1 - disagreements / corpus.items.length;
  console.log(`top-${K} neighbour agreement after quantisation: ` +
    `${(agreement * 100).toFixed(1)}% (${corpus.items.length - disagreements}/${corpus.items.length} items identical)`);
  if (agreement < 0.95) {
    throw new Error(`quantisation reorders neighbours for ${disagreements} items — this width is too coarse`);
  }
  console.log(`\nnearest neighbours of "${corpus.items[idx].title.slice(0, 44)}":`);
  for (const s of sims) console.log(`  ${s.sim.toFixed(3)}  [${s.it.topic}] ${s.it.title.slice(0, 52)}`);
}

main().catch((e) => { console.error(e); process.exit(1); });