Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / cmd/eval-model/main.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | b82c7100a492dec67dfd1eda6b3f53def859be35 |
| Branch | mag-substrate |
| Path or archive member | cmd/eval-model/main.go |
| Original SHA-256 | b61a80b7deadfeed8db10c55c948bf94bb224f8386c654cde391effab3e5dfa7 |
| Git blob SHA | 2ecc1f7e9974234f98b5885cce341d5d2fab1b35 |
| Reading-copy SHA-256 | b61a80b7deadfeed8db10c55c948bf94bb224f8386c654cde391effab3e5dfa7 |
Every recorded occurrence
These 3 records contain identical original source bytes. Their paths and revisions remain part of the archive.
| Original record | Branch or archive | Revision / identifier | Original location |
|---|---|---|---|
| Janady13/missionmatch / cmd/eval-model/main.go | mag-substrate | b82c7100a492dec67dfd1eda6b3f53def859be35 | Open original |
| Janady13/missionmatch / cmd/eval-model/main.go | phase2-connectors | e038d4c687aec16f8bd9bc4d907eea14d91f8704 | Open original |
| Janady13/missionmatch / cmd/eval-model/main.go | mag-substrate | b82c7100a492dec67dfd1eda6b3f53def859be35 | Open original |
Recovered source transcript
Download complete source text ↓396 lines · 17.8 KB
// Command eval-model is the offline EVALUATOR and the human promotion gate of the MAG learning loop. It replays
// a challenger version of a MAG model against a held out split of the decided bid labels, compares its
// calibration (ECE) and ranking (AUC) to the current champion, and runs the no adverse impact four fifths
// fairness gate. Only when BOTH gates pass AND a human passes the promote flag does it flip the champion; by
// default it is a pure report. It NEVER computes a per candidate score: the model is opportunity level and the
// frozen decimal engine (internal/modules/matching, ModelVersion v1.4.0) stays the only per candidate scorer.
//
// Two gates guard a promotion. The QUALITY gate: on the held out split the challenger must rank at least as well
// as the champion (AUC not materially lower) and calibrate at least as well (ECE not materially higher), so a
// promotion never ships a worse model. The FAIRNESS gate: because the win probability model emits
// opportunity_score and CANNOT drive candidate selection (the frozen engine does), promoting it cannot change
// the four fifths selection ratios; the gate confirms that structural invariant AND reads the current adverse
// impact state across every org (through the compliance pool, the only path to the eeo schema), refusing to
// promote while any group's four fifths ratio is already below 0.80 unless an operator explicitly overrides,
// on the governance principle that a model change does not ship into an unresolved disparate impact condition.
//
// Env (server side only): MM_MIGRATOR_DATABASE_URL (the BYPASSRLS migrator DSN, reads labels and models, flips
// champion) and MM_COMPLIANCE_DATABASE_URL (the NOBYPASSRLS compliance DSN, the only path to the eeo schema).
// Flags: -model, -challenger V (required), -holdout F (the held out fraction, default 0.30), -promote (default
// false, a dry run report unless set), -allow-adverse-impact (override the fairness gate, logged loudly).
//
// Build and run FIPS validated: CGO_ENABLED=0 GOFIPS140=v1.0.0 go build, run with GODEBUG=fips140=on.
package main
import (
"context"
"crypto/fips140"
"crypto/pbkdf2"
"crypto/sha256"
"flag"
"log"
"os"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/missionworkforce/missionmatch/internal/modules/eeo"
"github.com/missionworkforce/missionmatch/internal/platform/database"
"github.com/missionworkforce/missionmatch/internal/platform/magstore"
"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)
func main() {
kind := flag.String("kind", "win_probability", "which MAG model to evaluate: win_probability (opportunity_score) or reranker (display_order)")
modelKey := flag.String("model", "", "the MAG model_key to evaluate; defaults per kind")
challenger := flag.Int("challenger", 0, "the challenger version to evaluate (required)")
holdout := flag.Float64("holdout", 0.30, "the held out fraction of the decided labels or slates")
promote := flag.Bool("promote", false, "promote the challenger to champion when both gates pass")
allowAdverse := flag.Bool("allow-adverse-impact", false, "override the fairness gate (a deliberate governance decision, logged)")
flag.Parse()
if err := fipsSelfTest(); err != nil {
log.Fatalf("eval-model: FIPS self test (PBKDF2 HMAC SHA 256) failed: %v", err)
}
log.Printf("eval-model: fips_enabled=%v module=%s", fips140.Enabled(), fips140.Version())
if *challenger <= 0 {
log.Fatal("eval-model: set -challenger V (the challenger version to evaluate)")
}
migDSN := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
if migDSN == "" {
log.Fatal("eval-model: set MM_MIGRATOR_DATABASE_URL (the BYPASSRLS migrator DSN)")
}
compDSN := strings.TrimSpace(os.Getenv("MM_COMPLIANCE_DATABASE_URL"))
if compDSN == "" {
log.Fatal("eval-model: set MM_COMPLIANCE_DATABASE_URL (the compliance role DSN, the only path to the eeo schema)")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
migPool, err := database.NewPool(ctx, migDSN)
if err != nil {
log.Fatalf("eval-model: connect migrator: %v", err)
}
defer migPool.Close()
if err := database.AssertMigratorRole(ctx, migPool); err != nil {
log.Fatalf("eval-model: migrator role check: %v", err)
}
compPool, err := database.NewPool(ctx, compDSN)
if err != nil {
log.Fatalf("eval-model: connect compliance: %v", err)
}
defer compPool.Close()
if err := database.AssertRole(ctx, compPool, "missionmatch_compliance"); err != nil {
log.Fatalf("eval-model: compliance pool role check: %v", err)
}
if *kind == "reranker" {
evalReranker(ctx, migPool, compPool, resolveModelKey(*modelKey, magstore.RerankModelKey), *challenger, *holdout, *promote, *allowAdverse)
return
}
winKey := resolveModelKey(*modelKey, "opportunity_win_probability")
// Load the champion and the challenger, and confirm both emit opportunity_score, so an evaluation can never
// compare or promote something that could surface a candidate score.
champVersion, err := magstore.ChampionVersion(ctx, migPool, winKey)
if err != nil {
log.Fatalf("eval-model: %v", err)
}
champ, err := magstore.LoadModel(ctx, migPool, winKey, champVersion)
if err != nil {
log.Fatalf("eval-model: load champion: %v", err)
}
chal, err := magstore.LoadModel(ctx, migPool, winKey, *challenger)
if err != nil {
log.Fatalf("eval-model: load challenger: %v", err)
}
if champ.OutputKind != "opportunity_score" || chal.OutputKind != "opportunity_score" {
log.Fatalf("eval-model: refusing to evaluate a non opportunity_score model (champion %q, challenger %q); this loop fits only opportunity level win probability", champ.OutputKind, chal.OutputKind)
}
log.Printf("eval-model: champion v%d vs challenger v%d for %q", champVersion, *challenger, winKey)
// The QUALITY gate: score the held out split under both models and compare calibration and ranking.
quality, err := magstore.EvaluateHeldOut(ctx, migPool, champ, chal, *holdout)
if err != nil {
log.Fatalf("eval-model: quality evaluation: %v", err)
}
logQuality(quality)
// The FAIRNESS gate: the current adverse impact state across every org, read through the compliance pool.
fairness, err := evaluateFairness(ctx, compPool)
if err != nil {
log.Fatalf("eval-model: fairness evaluation: %v", err)
}
logFairness(fairness)
// The gate decision. Fairness may be overridden by an explicit operator flag (logged loudly); quality may
// not, so a promotion never ships a measurably worse model.
fairnessPass := fairness.Clean || *allowAdverse
if fairness.Clean {
log.Print("eval-model: FAIRNESS gate PASS (no group below the 0.80 four fifths threshold; the opportunity level model cannot change candidate selection)")
} else if *allowAdverse {
log.Printf("eval-model: FAIRNESS gate OVERRIDDEN by -allow-adverse-impact despite %d flagged group(s); this is a recorded governance decision", fairness.FlaggedGroups)
} else {
log.Printf("eval-model: FAIRNESS gate FAIL (%d group(s) below the 0.80 four fifths threshold across %d org(s)); resolve the disparate impact or pass -allow-adverse-impact to override", fairness.FlaggedGroups, fairness.OrgsWithFlag)
}
log.Printf("eval-model: QUALITY gate %s", passFail(quality.Pass))
if !*promote {
log.Print("eval-model: report only (no -promote); nothing changed")
return
}
if !quality.Pass || !fairnessPass {
log.Print("eval-model: NOT promoting; a gate did not pass")
return
}
from, err := magstore.PromoteChampion(ctx, migPool, winKey, *challenger)
if err != nil {
log.Fatalf("eval-model: promote: %v", err)
}
log.Printf("eval-model: PROMOTED %s v%d to champion (replacing v%d); the serve path now reflects the learned coefficients", winKey, *challenger, from)
}
func logQuality(q magstore.QualityResult) {
log.Printf("eval-model: held out %d bids", q.HeldOut)
log.Printf("eval-model: AUC champion %.4f vs challenger %.4f (higher is better ranking)", q.ChampAUC, q.ChalAUC)
log.Printf("eval-model: ECE champion %.4f vs challenger %.4f (lower is better calibration)", q.ChampECE, q.ChalECE)
}
// fourFifthsFloor is the EEOC four fifths threshold; a group impact ratio below it is adverse impact.
// rankFairnessSlack absorbs small sample noise in the ratio estimate when comparing the challenger ranking to
// the champion ranking (a regression must be real, not noise).
const (
fourFifthsFloor = 0.80
rankFairnessSlack = 0.05
)
// evalReranker evaluates a display order re ranker challenger: the held out RANKING quality (does it order
// approved above rejected at least as well as the champion) AND the four fifths no adverse impact check
// recomputed UNDER the challenger ranking per org (does its top of slate selection skew who surfaces first).
// It promotes only when both gates pass and a human passes the flag. The re ranker emits an ordering, never a
// per candidate score; a human still approves every member and the frozen v1.4.0 engine is untouched.
func evalReranker(ctx context.Context, migPool, compPool *pgxpool.Pool, modelKey string, challenger int, holdout float64, promote, allowAdverse bool) {
champVersion, err := magstore.ChampionVersion(ctx, migPool, modelKey)
if err != nil {
log.Fatalf("eval-model: %v", err)
}
champ, err := magstore.LoadModel(ctx, migPool, modelKey, champVersion)
if err != nil {
log.Fatalf("eval-model: load champion: %v", err)
}
chal, err := magstore.LoadModel(ctx, migPool, modelKey, challenger)
if err != nil {
log.Fatalf("eval-model: load challenger: %v", err)
}
if champ.OutputKind != "display_order" || chal.OutputKind != "display_order" {
log.Fatalf("eval-model: refusing to evaluate a non display_order model as a re ranker (champion %q, challenger %q)", champ.OutputKind, chal.OutputKind)
}
champCoeffs := magstore.RerankCoeffs(champ)
chalCoeffs := magstore.RerankCoeffs(chal)
log.Printf("eval-model: re ranker champion v%d vs challenger v%d for %q", champVersion, challenger, modelKey)
// The QUALITY gate: held out pairwise ranking accuracy.
quality, err := magstore.EvaluateRerankerQuality(ctx, migPool, champCoeffs, chalCoeffs, holdout)
if err != nil {
log.Fatalf("eval-model: ranking evaluation: %v", err)
}
log.Printf("eval-model: held out %d slates, pairwise ranking accuracy champion %.4f vs challenger %.4f (higher orders approved above rejected better)",
quality.HeldOutGroups, quality.ChampAccuracy, quality.ChalAccuracy)
// The FAIRNESS gate: recompute the four fifths ratio UNDER the challenger ranking per org, comparing the
// challenger's top of slate selection to the champion's. A display order model influences selection, so this
// is a genuine per ranking recomputation, not a baseline check.
fair := evaluateRerankFairness(ctx, migPool, compPool, champCoeffs, chalCoeffs)
fairnessPass := fair.Clean || allowAdverse
if fair.Clean {
log.Printf("eval-model: FAIRNESS gate PASS (no org drops below the 0.80 four fifths floor or regresses from the engine order under the challenger ranking; %d org(s) checked)", fair.OrgsChecked)
} else if allowAdverse {
log.Printf("eval-model: FAIRNESS gate OVERRIDDEN by -allow-adverse-impact despite %d org(s) worsened under the challenger ranking; a recorded governance decision", fair.OrgsWorsened)
} else {
log.Printf("eval-model: FAIRNESS gate FAIL (%d org(s) drop below 0.80 or regress from the engine order under the challenger ranking, worst ratio %.4f); resolve or pass -allow-adverse-impact", fair.OrgsWorsened, fair.WorstRatio)
}
log.Printf("eval-model: QUALITY gate %s", passFail(quality.Pass))
if !promote {
log.Print("eval-model: report only (no -promote); nothing changed")
return
}
if !quality.Pass || !fairnessPass {
log.Print("eval-model: NOT promoting; a gate did not pass")
return
}
from, err := magstore.PromoteChampion(ctx, migPool, modelKey, challenger)
if err != nil {
log.Fatalf("eval-model: promote: %v", err)
}
log.Printf("eval-model: PROMOTED re ranker %s v%d to champion (replacing v%d); the slate display order now reflects the learned weights, the engine score unchanged", modelKey, challenger, from)
}
// rerankFairness is the outcome of recomputing the four fifths ratio under a challenger ranking across orgs.
type rerankFairness struct {
OrgsChecked int
OrgsWorsened int
WorstRatio float64
Clean bool
}
// evaluateRerankFairness reads the decided slates (migrator, no demographics), and for each org computes the
// challenger's and the champion's top of slate selection, then recomputes the four fifths minimum impact ratio
// under each through the compliance pool (the only demographic path; only candidate ids cross). An org fails
// when the challenger drops below the 0.80 floor while the champion did not, or regresses from the champion
// beyond the noise slack. Clean is true when no org fails.
func evaluateRerankFairness(ctx context.Context, migPool, compPool *pgxpool.Pool, champCoeffs, chalCoeffs []float64) rerankFairness {
groups, err := magstore.ReadSlateGroups(ctx, migPool)
if err != nil {
log.Fatalf("eval-model: read slates for fairness: %v", err)
}
svc := eeo.NewService(tenant.New(compPool))
res := rerankFairness{WorstRatio: 1.0}
for org, orgGroups := range magstore.GroupsByOrg(groups) {
champSel := magstore.CandidateSelection(champCoeffs, orgGroups)
chalSel := magstore.CandidateSelection(chalCoeffs, orgGroups)
champSection, cerr := svc.AdverseImpactForSelection(tenant.WithOrg(ctx, org), champSel)
if cerr != nil {
log.Printf("eval-model: fairness read for org %s failed: %v", org, cerr)
continue
}
chalSection, cerr := svc.AdverseImpactForSelection(tenant.WithOrg(ctx, org), chalSel)
if cerr != nil {
log.Printf("eval-model: fairness read for org %s failed: %v", org, cerr)
continue
}
res.OrgsChecked++
champMin := champSection.MinImpactRatio()
chalMin := chalSection.MinImpactRatio()
if chalMin < res.WorstRatio {
res.WorstRatio = chalMin
}
// The org passes when the challenger stays above the floor OR is no worse than the champion (the engine
// order that ships today); it fails only when the challenger introduces or worsens adverse impact.
orgPass := chalMin >= fourFifthsFloor || chalMin >= champMin-rankFairnessSlack
if !orgPass {
res.OrgsWorsened++
log.Printf("eval-model: org %s four fifths under challenger ranking %.4f vs champion %.4f (below floor or regressed)", org, chalMin, champMin)
}
}
res.Clean = res.OrgsWorsened == 0
return res
}
// resolveModelKey returns an explicit model_key flag when set, else the default for the kind.
func resolveModelKey(flagKey, def string) string {
if s := strings.TrimSpace(flagKey); s != "" {
return s
}
return def
}
// fairnessResult is the current adverse impact state across every org with AEDT activity.
type fairnessResult struct {
OrgsChecked int
OrgsWithFlag int
FlaggedGroups int
Clean bool
}
// evaluateFairness reads the current adverse impact report for every org with AEDT activity through the
// compliance pool (the only path to the eeo schema), counting the groups whose four fifths ratio is already
// below 0.80. It is Clean when no group is flagged in any org. The win probability model cannot itself change
// these ratios (it emits opportunity_score, never a candidate selection), so this is a baseline safety check,
// not a per model recomputation: a model governance action does not ship while a disparate impact is unresolved.
func evaluateFairness(ctx context.Context, compPool *pgxpool.Pool) (fairnessResult, error) {
svc := eeo.NewService(tenant.New(compPool))
orgs, err := orgsWithActivity(ctx, compPool)
if err != nil {
return fairnessResult{}, err
}
now := time.Now().UTC()
var res fairnessResult
for _, org := range orgs {
report, rerr := svc.AdverseImpact(tenant.WithOrg(ctx, org.String()), now)
if rerr != nil {
// One org failing must not abort the gate; log the org id (not PII) and continue conservatively.
log.Printf("eval-model: fairness read for org %s failed: %v", org, rerr)
continue
}
res.OrgsChecked++
flagged := countFlagged(report)
if flagged > 0 {
res.OrgsWithFlag++
res.FlaggedGroups += flagged
}
}
res.Clean = res.FlaggedGroups == 0
return res, nil
}
// countFlagged counts the demographic groups flagged below the 0.80 four fifths threshold across both analysis
// stages (selection and screening) and all three cuts (gender, race, intersectional) of an adverse impact
// report.
func countFlagged(r eeo.Report) int {
n := 0
for _, section := range []eeo.AnalysisSection{r.Selection, r.Screening} {
for _, group := range [][]eeo.ImpactRow{section.ByGender, section.ByRace, section.Intersectional} {
for _, row := range group {
if row.Flagged {
n++
}
}
}
}
return n
}
func logFairness(f fairnessResult) {
log.Printf("eval-model: fairness checked %d org(s), %d with a flagged group, %d flagged group(s) total",
f.OrgsChecked, f.OrgsWithFlag, f.FlaggedGroups)
}
// orgsWithActivity enumerates the orgs to check from the distinct organization_id values in audit_logs, which
// is outside RLS and granted to the compliance role, so the gate needs no cross org read of tenant data.
func orgsWithActivity(ctx context.Context, pool *pgxpool.Pool) ([]uuid.UUID, error) {
rows, err := pool.Query(ctx,
`SELECT DISTINCT organization_id FROM audit_logs WHERE organization_id IS NOT NULL ORDER BY organization_id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []uuid.UUID
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
func passFail(ok bool) string {
if ok {
return "PASS"
}
return "FAIL"
}
// fipsSelfTest derives one PBKDF2 HMAC SHA 256 key under the active crypto module and fails closed if it errors.
func fipsSelfTest() error {
_, err := pbkdf2.Key(sha256.New, "eval-model-self-test", []byte("eval-model-salt"), 600000, 32)
return err
}
func firstNonEmpty(vs ...string) string {
for _, v := range vs {
if s := strings.TrimSpace(v); s != "" {
return s
}
}
return ""
}