Janady13/missionmatch / internal/platform/eval/metrics.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/platform/eval/metrics.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/platform/eval/metrics.go
Original SHA-256	bd7d4a916baa3dae0517b8ded118101a093c506aa1b304ef5652fec9a61a9412
Git blob SHA	6db9ae62f00224d82e31e82d489cfad1bb869001
Reading-copy SHA-256	bd7d4a916baa3dae0517b8ded118101a093c506aa1b304ef5652fec9a61a9412

Original source ↗

Every recorded occurrence

These 12 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/platform/eval/metrics.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/platform/eval/metrics.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓115 lines · 4.2 KB

// Package eval holds the pure, deterministic retrieval evaluation metrics used to gate the RAG layer.
//
// These are OFFLINE evaluation numbers (recall, MRR, nDCG), not a surfaced score, not money, and not a
// candidate match score, so float64 is the right type for a metric value here. float64 must never leak into
// any production scoring path; the matching engine stays on shopspring/decimal. This package has no DB and no
// I/O: it is a set of pure functions over a ranked list of ids and a set of relevant ids, so it is exactly
// reproducible and exhaustively unit testable without a database or the inference plane.
//
// A ranked list is the ordered ids the retriever returned, best first. The relevant set is the gold judgment:
// the ids a human says SHOULD come back for the query. The metrics measure how well the ranking surfaced the
// relevant ids, which is what catches a regression in embeddings, HNSW tuning, the mission gate, or the RRF k
// constant before it ships.
package eval

import "math"

// RecallAtK is the fraction of the relevant ids that appear in the top k of the ranked list. It answers did we
// FIND the right people at all, within the cutoff. The denominator is the count of relevant ids, so a query
// with no relevant judgments is defined as 1.0 (nothing was missed, vacuously). A k at or below zero recalls
// nothing, so the result is 0.0 unless there were no relevant ids to find. Duplicate ids in the ranked list
// are counted once, so a retriever cannot inflate recall by repeating a hit.
func RecallAtK(ranked []string, relevant map[string]bool, k int) float64 {
	total := countRelevant(relevant)
	if total == 0 {
		return 1.0
	}
	if k <= 0 {
		return 0.0
	}
	seen := make(map[string]bool, k)
	found := 0
	for i, id := range ranked {
		if i >= k {
			break
		}
		if seen[id] {
			continue
		}
		seen[id] = true
		if relevant[id] {
			found++
		}
	}
	return float64(found) / float64(total)
}

// MRR is the reciprocal of the rank (one based) of the FIRST relevant id in the ranked list, or 0.0 when no
// relevant id is present. A relevant id at rank one scores 1.0, at rank two 0.5, at rank three one third, and
// so on. It rewards putting a right answer near the top, which is what a person reads first. With no relevant
// judgments at all there is no first relevant id, so the score is 0.0.
func MRR(ranked []string, relevant map[string]bool) float64 {
	for i, id := range ranked {
		if relevant[id] {
			return 1.0 / float64(i+1)
		}
	}
	return 0.0
}

// NDCGAtK is the binary gain normalized discounted cumulative gain at cutoff k. Each relevant id contributes a
// gain of one discounted by log2 of its one based rank, so a relevant id deeper in the list is worth less. The
// raw DCG is normalized by the ideal DCG, the DCG of the best possible ordering (every relevant id packed at
// the top), bounded by k, so the score is between 0.0 (no relevant id in the top k) and 1.0 (the relevant ids
// fill the top k in any order). With no relevant judgments the ideal DCG is zero, so nDCG is defined as 1.0
// (a perfect, vacuous ranking). A k at or below zero yields 0.0 unless there were no relevant ids. Duplicate
// ids are credited once, on their first appearance, so a repeated hit cannot inflate the gain.
func NDCGAtK(ranked []string, relevant map[string]bool, k int) float64 {
	total := countRelevant(relevant)
	if total == 0 {
		return 1.0
	}
	if k <= 0 {
		return 0.0
	}
	seen := make(map[string]bool, k)
	dcg := 0.0
	for i, id := range ranked {
		if i >= k {
			break
		}
		if seen[id] {
			continue
		}
		seen[id] = true
		if relevant[id] {
			dcg += 1.0 / math.Log2(float64(i+2)) // rank is i+1, discount uses log2(rank+1) = log2(i+2)
		}
	}
	// Ideal DCG: the relevant ids packed into the first min(total, k) positions, each with gain one.
	ideal := total
	if k < ideal {
		ideal = k
	}
	idcg := 0.0
	for i := 0; i < ideal; i++ {
		idcg += 1.0 / math.Log2(float64(i+2))
	}
	if idcg == 0 {
		return 1.0
	}
	return dcg / idcg
}

// countRelevant counts the ids marked relevant (value true), ignoring any explicitly false entries so a qrels
// map that records non relevant judgments does not distort the denominator.
func countRelevant(relevant map[string]bool) int {
	n := 0
	for _, ok := range relevant {
		if ok {
			n++
		}
	}
	return n
}