Janady13/feedthejoe / src/adapters/corpus.js

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/feedthejoe / src/adapters/corpus.js
Date	Not specified in source
Repository	Janady13/feedthejoe
Revision	0e608a2b515a959bf35d2783428766624400ca3c
Branch	main
Path or archive member	src/adapters/corpus.js
Original SHA-256	1b55db684b8257f8e6ad301d1a0fb6ec663485e6989621da5f2ebe0f557de538
Git blob SHA	375752b4e22bc27a8ee93d54ecea812c9f9c9289
Reading-copy SHA-256	1b55db684b8257f8e6ad301d1a0fb6ec663485e6989621da5f2ebe0f557de538

Original source ↗

Recovered source transcript

Download complete source text ↓47 lines · 1.6 KB

/**
 * Corpus adapter: fetch the generated corpus and its embedding sidecar, and
 * hand back items the core can score.
 *
 * The embeddings arrive as a packed int16 binary rather than JSON. 42 x 384
 * numbers as JSON text is ~180 KB of digits and commas that then have to be
 * parsed into objects; as a typed array it is 31.5 KB and needs no parsing at
 * all. The header is [itemCount, dim] as int32, then itemCount*dim int16s in
 * corpus order.
 */

/** Must match SCALE in build/embed-corpus.mjs. */
const SCALE = 32767;

/**
 * @param {string} corpusUrl
 * @param {string} embeddingsUrl
 * @returns {Promise<{items: import('../ports/types.js').Item[], dim: number}>}
 */
export async function loadCorpus(corpusUrl, embeddingsUrl) {
  const [corpusRes, embedRes] = await Promise.all([
    fetch(corpusUrl, { credentials: 'omit' }),
    fetch(embeddingsUrl, { credentials: 'omit' }),
  ]);
  if (!corpusRes.ok) throw new Error(`corpus ${corpusRes.status}`);
  if (!embedRes.ok) throw new Error(`embeddings ${embedRes.status}`);

  const corpus = await corpusRes.json();
  const buf = await embedRes.arrayBuffer();

  const header = new Int32Array(buf, 0, 2);
  const [count, dim] = header;
  if (count !== corpus.items.length) {
    throw new Error(`embedding count ${count} does not match corpus ${corpus.items.length}`);
  }

  const packed = new Int16Array(buf, 8, count * dim);
  const items = corpus.items.map((item, i) => {
    const e = new Float32Array(dim);
    const off = i * dim;
    for (let d = 0; d < dim; d += 1) e[d] = packed[off + d] / SCALE;
    return { ...item, embedding: e };
  });

  return { items, dim };
}