Janady13/missionmatch / internal/platform/magstore/eval.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/platform/magstore/eval.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/platform/magstore/eval.go
Original SHA-256	bdf795cb857c12539366e492955edd249677dd37d20609435626bb86967e6cea
Git blob SHA	677e432b2232cceed177215b99cd6ab2fcfb88d8
Reading-copy SHA-256	bdf795cb857c12539366e492955edd249677dd37d20609435626bb86967e6cea

Original source ↗

Every recorded occurrence

These 10 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/platform/magstore/eval.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/platform/magstore/eval.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓96 lines · 3.7 KB

package magstore

import (
	"context"
	"errors"

	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/learn"
)

// Quality gate tolerances and the metric bin count.
//
// Ranking (AUC) is the primary discrimination metric: a challenger must not rank materially worse than the
// champion (AUCTolerance absorbs floating point noise and a tiny held out split). Calibration (ECE) is guarded
// two ways so the gate is honest against a DEGENERATE champion: the prior only champion is a CONSTANT predictor
// (AUC near 0.5), and a constant predictor's ECE is a lucky baseline (it is just the gap between the constant
// and the overall base rate), not a meaningful bar. So the challenger's calibration passes when it is either not
// materially worse than the champion (ECETolerance, the mature champion versus challenger case) OR within an
// absolute cold start bound (ECECeiling, so a genuine discriminating challenger is not blocked by a constant
// champion's deceptively low ECE, while a truly miscalibrated challenger past the ceiling still fails). A later
// isotonic recalibration tightens ECE further.
const (
	ECETolerance = 0.03
	ECECeiling   = 0.15
	AUCTolerance = 0.02
	MetricBins   = 10
)

// QualityResult is the held out comparison of a champion and a challenger: the split size, each model's ranking
// (AUC) and calibration (ECE), and whether the challenger clears the quality gate.
type QualityResult struct {
	HeldOut  int
	ChampAUC float64
	ChalAUC  float64
	ChampECE float64
	ChalECE  float64
	Pass     bool
}

// EvaluateHeldOut reads the pooled decided bids and holds out a fraction as a test split, taking the tail of
// the read's stable id order so the split is deterministic and reproducible (the same ledger always yields the
// same held out set), then scores the split under both models and computes AUC and ECE for each. The challenger
// passes when it ranks at least as well as the champion (AUC within tolerance) and calibrates at least as well
// (ECE within tolerance). It is pure over the stored models and the label read; the fairness gate is a separate
// check the caller runs through the compliance pool.
func EvaluateHeldOut(ctx context.Context, pool *pgxpool.Pool, champ, chal Model, holdout float64) (QualityResult, error) {
	rows, err := ReadLabeledBids(ctx, pool)
	if err != nil {
		return QualityResult{}, err
	}
	decided := make([]learn.LabeledRow, 0, len(rows))
	for _, r := range rows {
		if learn.IsDecided(r.Result) {
			decided = append(decided, r)
		}
	}
	if len(decided) == 0 {
		return QualityResult{}, errors.New("magstore: no decided bids to evaluate")
	}
	if holdout <= 0 || holdout >= 1 {
		holdout = 0.30
	}
	cut := len(decided) - int(float64(len(decided))*holdout)
	if cut >= len(decided) {
		cut = len(decided) - 1 // always hold out at least one row
	}
	if cut < 0 {
		cut = 0
	}
	test := decided[cut:] // the tail of the stable id order, a deterministic held out split

	champProbs := make([]float64, len(test))
	chalProbs := make([]float64, len(test))
	y := make([]int, len(test))
	for i, r := range test {
		champProbs[i] = champ.Score(r.Features)
		chalProbs[i] = chal.Score(r.Features)
		if r.Result == learn.ResultWin {
			y[i] = 1
		}
	}
	res := QualityResult{
		HeldOut:  len(test),
		ChampAUC: learn.AUC(champProbs, y),
		ChalAUC:  learn.AUC(chalProbs, y),
		ChampECE: learn.ECE(champProbs, y, MetricBins),
		ChalECE:  learn.ECE(chalProbs, y, MetricBins),
	}
	aucNotWorse := res.ChalAUC >= res.ChampAUC-AUCTolerance
	eceNotWorse := res.ChalECE <= res.ChampECE+ECETolerance // relative: not worse than the champion
	eceAcceptable := res.ChalECE <= ECECeiling              // absolute: within the cold start calibration bound
	res.Pass = aucNotWorse && (eceNotWorse || eceAcceptable)
	return res, nil
}