Janady13/missionmatch / internal/integration/mag_train_eval_integration_test.go

1//go:build integration 2 3// MAG learning loop integration. Proves at runtime against a REAL Postgres and the migrated 0036 coefficient 4// store what the unit tests cannot: the offline TRAINER fits a learned win probability challenger from the 5// pooled decided bid_outcomes labels, the EVALUATOR compares it to the prior only champion on a held out split, 6// a human promotion flips the champion, and the EXISTING serve path (IntentOpportunityWinProbability through the 7// orchestrator) then reflects the LEARNED coefficient, moving the served probability off the prior only base 8// rate toward the seeded market signal, with the frozen decimal engine untouched (no candidate score anywhere). 9// 10// The seeded signal is honest market structure: a HIGH win rate agency segment (mostly wins) and a LOW win rate 11// segment (mostly losses), pooled, so the segment base rate feature genuinely predicts the label and the fitted 12// coefficient is positive. NONE of it is a protected attribute. 13// 14// Run: go test -tags=integration ./internal/integration/ -run MAGLearningLoop -v 15// Env: MM_APP_DSN, MM_MIGRATOR_DSN, MM_COMPLIANCE_DSN (compliance optional; the fairness read is skipped without 16// it). The orchestrator applies migration 0036 (goose v36) first. 17package integration 18 19import ( 20 "context" 21 "strconv" 22 "strings" 23 "testing" 24 "time" 25 26 "github.com/google/uuid" 27 28 "github.com/missionworkforce/missionmatch/internal/modules/ai" 29 "github.com/missionworkforce/missionmatch/internal/modules/bidoutcomes" 30 "github.com/missionworkforce/missionmatch/internal/modules/eeo" 31 "github.com/missionworkforce/missionmatch/internal/platform/audit" 32 "github.com/missionworkforce/missionmatch/internal/platform/magstore" 33 "github.com/missionworkforce/missionmatch/internal/platform/tenant" 34) 35 36// seedSegmentOutcomes records n wins and m losses for one org tied to a single agency segment, the real 37// recording path under RLS. It is the honest market signal the trainer learns from. 38func seedSegmentOutcomes(t *testing.T, svc *bidoutcomes.Service, org uuid.UUID, agency string, wins, losses int) { 39 t.Helper() 40 req := seedRequirementWithAgency(t, org, agency) 41 for i := 0; i < wins; i++ { 42 recordOutcome(t, svc, org, req, bidoutcomes.ResultWin) 43 } 44 for i := 0; i < losses; i++ { 45 recordOutcome(t, svc, org, req, bidoutcomes.ResultLoss) 46 } 47} 48 49// servedProbability serves the win probability MAG intent for an org and agency through the orchestrator with a 50// nil inference client (so the answer is the deterministic rendering) and returns the calibrated probability the 51// answer states, parsed from the "for the segment: <p>." rendering. 52func servedProbability(t *testing.T, org uuid.UUID, agency string) (string, string) { 53 t.Helper() 54 orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) 55 resp, err := orch.Answer(ctxOrg(org), "", ai.IntentOpportunityWinProbability, []any{agency, ""}) 56 if err != nil { 57 t.Fatalf("win probability answer: %v", err) 58 } 59 if resp.Refused { 60 t.Fatalf("an authenticated win probability question must not be refused, got %+v", resp) 61 } 62 return parseServedProbability(t, resp.Answer), resp.Answer 63} 64 65// parseServedProbability pulls the calibrated probability out of the deterministic rendering, which reads 66// "Calibrated opportunity_score for the segment: 0.xx. Drivers ...". 67func parseServedProbability(t *testing.T, answer string) string { 68 t.Helper() 69 const marker = "for the segment: " 70 i := strings.Index(answer, marker) 71 if i < 0 { 72 t.Fatalf("the served answer must render the calibrated probability, got %q", answer) 73 } 74 rest := answer[i+len(marker):] 75 // the probability renders as 0.xx followed by a period and a space before "Drivers"; take the value token up 76 // to the first space and drop the trailing sentence period. 77 if k := strings.Index(rest, " "); k >= 0 { 78 return strings.TrimSuffix(rest[:k], ".") 79 } 80 return strings.TrimSpace(rest) 81} 82 83// mustFloat parses a probability token the served answer rendered, failing the test on a malformed value. 84func mustFloat(t *testing.T, s string) float64 { 85 t.Helper() 86 f, err := strconv.ParseFloat(s, 64) 87 if err != nil { 88 t.Fatalf("could not parse the served probability %q: %v", s, err) 89 } 90 return f 91} 92 93// TestMAGLearningLoopTrainsEvaluatesPromotesAndServes drives the whole loop end to end on the real database. 94func TestMAGLearningLoopTrainsEvaluatesPromotesAndServes(t *testing.T) { 95 const modelKey = "opportunity_win_probability" 96 const priorOnlyVersion = 1 // migration 0036 seeds v1 as the prior only champion 97 run := uuid.NewString()[:8] 98 highAgency := "HIGHSEG " + run 99 lowAgency := "LOWSEG " + run 100 101 // Staging persists prior promotions, so force the champion back to the prior only v1 first, making the 102 // before and after comparison deterministic and the test idempotent across re runs. 103 if _, err := magstore.PromoteChampion(context.Background(), migrator, modelKey, priorOnlyVersion); err != nil { 104 t.Fatalf("reset champion to prior only v%d: %v", priorOnlyVersion, err) 105 } 106 priorChampion, err := magstore.ChampionVersion(context.Background(), migrator, modelKey) 107 if err != nil { 108 t.Fatalf("champion version: %v", err) 109 } 110 if priorChampion != priorOnlyVersion { 111 t.Fatalf("the champion must reset to the prior only v%d, got v%d", priorOnlyVersion, priorChampion) 112 } 113 // This test promotes a learned challenger to champion on the shared staging store. Restore the prior only v1 114 // as champion when the test ends (however it ends), so it never leaks a learned champion into the other MAG 115 // tests, which assume the prior only base rate. The default production champion is the prior only v1; a 116 // learned model becomes champion only by a deliberate human promotion through cmd/eval-model. 117 t.Cleanup(func() { 118 if _, cerr := magstore.PromoteChampion(context.Background(), migrator, modelKey, priorOnlyVersion); cerr != nil { 119 t.Logf("cleanup: could not restore the prior only champion v%d: %v", priorOnlyVersion, cerr) 120 } 121 }) 122 123 // Serve BEFORE training for the high segment, the prior only baseline (near 0.60, betas zero). 124 orgHigh := seedOrg(t) 125 orgLow := seedOrg(t) 126 svc := bidoutcomes.NewService(appDB, nil) 127 // A strong, honest signal: the high segment wins far more than it loses, the low segment the reverse. This 128 // dominates any residue from other tests, so the pooled fit learns a POSITIVE segment_base_rate coefficient. 129 seedSegmentOutcomes(t, svc, orgHigh, highAgency, 30, 5) 130 seedSegmentOutcomes(t, svc, orgLow, lowAgency, 5, 30) 131 132 probBeforeStr, answerBefore := servedProbability(t, orgHigh, highAgency) 133 probBefore := mustFloat(t, probBeforeStr) 134 t.Logf("served probability before training (prior only): %s", probBeforeStr) 135 // The prior only segment_base_rate coefficient is zero, so the driver contributes zero regardless of the 136 // gathered value; the prior only calibrated probability is near the 0.60 base rate. 137 if !strings.Contains(answerBefore, "segment_base_rate contributes 0") { 138 t.Errorf("the prior only answer must show a zero segment_base_rate contribution, got %q", answerBefore) 139 } 140 if probBefore < 0.5 || probBefore > 0.7 { 141 t.Errorf("the prior only probability must be near the 0.60 base rate, got %v", probBefore) 142 } 143 144 // TRAIN: fit a challenger over the pooled decided labels. min-events 10 so the seeded 70 clears the floor. 145 res, err := magstore.Train(context.Background(), migrator, modelKey, 1.0, 10, false) 146 if err != nil { 147 t.Fatalf("train: %v", err) 148 } 149 if res.Skipped { 150 t.Fatalf("the fit must not be skipped with %d seeded decided bids: %s", res.N, res.SkipReason) 151 } 152 if res.Version <= priorChampion { 153 t.Fatalf("the challenger version %d must be newer than the prior champion %d", res.Version, priorChampion) 154 } 155 // The learned segment_base_rate coefficient must be POSITIVE: a higher segment win rate predicts a win. 156 idx := -1 157 for i, name := range res.FeatureNames { 158 if name == magstore.SegmentBaseRateFeature { 159 idx = i 160 } 161 } 162 if idx < 0 { 163 t.Fatalf("the fit must include the segment_base_rate feature, got %v", res.FeatureNames) 164 } 165 if res.Coeffs[idx] <= 0 { 166 t.Fatalf("the learned segment_base_rate coefficient must be positive, got %v", res.Coeffs[idx]) 167 } 168 t.Logf("challenger v%d learned segment_base_rate coefficient %.6f over %d bids", res.Version, res.Coeffs[idx], res.N) 169 170 // The challenger stays a challenger until promotion: the champion must NOT have moved yet. 171 stillPrior, err := magstore.ChampionVersion(context.Background(), migrator, modelKey) 172 if err != nil { 173 t.Fatalf("champion version after train: %v", err) 174 } 175 if stillPrior != priorChampion { 176 t.Fatalf("the trainer must NOT flip champion; champion moved from %d to %d", priorChampion, stillPrior) 177 } 178 179 // EVALUATE: the held out replay must compute AUC and ECE for both, and the learned challenger must rank 180 // strictly better than the constant prior only champion (whose AUC is 0.5 on any split). 181 champ, err := magstore.LoadModel(context.Background(), migrator, modelKey, priorChampion) 182 if err != nil { 183 t.Fatalf("load champion: %v", err) 184 } 185 chal, err := magstore.LoadModel(context.Background(), migrator, modelKey, res.Version) 186 if err != nil { 187 t.Fatalf("load challenger: %v", err) 188 } 189 quality, err := magstore.EvaluateHeldOut(context.Background(), migrator, champ, chal, 0.30) 190 if err != nil { 191 t.Fatalf("evaluate: %v", err) 192 } 193 t.Logf("held out %d bids, AUC champion %.4f vs challenger %.4f, ECE champion %.4f vs challenger %.4f", 194 quality.HeldOut, quality.ChampAUC, quality.ChalAUC, quality.ChampECE, quality.ChalECE) 195 if quality.HeldOut == 0 { 196 t.Fatal("the held out split must be non empty") 197 } 198 if quality.ChalAUC <= quality.ChampAUC { 199 t.Errorf("the learned challenger must rank better than the constant prior only champion, AUC challenger %.4f vs champion %.4f", quality.ChalAUC, quality.ChampAUC) 200 } 201 if !quality.Pass { 202 t.Errorf("the learned challenger must clear the quality gate, got %+v", quality) 203 } 204 205 // The FAIRNESS read must be callable through the compliance pool (the only path to the eeo schema). The 206 // win probability model is opportunity level and cannot change candidate selection, so a clean seed shows no 207 // flagged group. Skip when no compliance DSN is configured. 208 if compliance != nil { 209 eeoSvc := eeo.NewService(tenant.New(compliance)) 210 if _, ferr := eeoSvc.AdverseImpact(tenant.WithOrg(context.Background(), orgHigh.String()), time.Now().UTC()); ferr != nil { 211 t.Fatalf("the fairness read must be callable through the compliance pool: %v", ferr) 212 } 213 } 214 215 // PROMOTE: the human governance action flips the champion and records an append only audit row. 216 from, err := magstore.PromoteChampion(context.Background(), migrator, modelKey, res.Version) 217 if err != nil { 218 t.Fatalf("promote: %v", err) 219 } 220 if from != priorChampion { 221 t.Errorf("the promotion must replace the prior champion %d, reported replacing %d", priorChampion, from) 222 } 223 nowChampion, err := magstore.ChampionVersion(context.Background(), migrator, modelKey) 224 if err != nil { 225 t.Fatalf("champion version after promote: %v", err) 226 } 227 if nowChampion != res.Version { 228 t.Fatalf("the champion must be the promoted challenger v%d, got v%d", res.Version, nowChampion) 229 } 230 // The promotion audit row exists (global governance action, no org). 231 var auditCount int 232 if err := migrator.QueryRow(context.Background(), 233 `SELECT count(*) FROM audit_logs WHERE action='aedt.model.promoted' AND entity_id=$1 AND (new_values->>'to_version')::int=$2`, 234 modelKey, res.Version).Scan(&auditCount); err != nil { 235 t.Fatalf("read promotion audit: %v", err) 236 } 237 if auditCount == 0 { 238 t.Fatal("the promotion must append an aedt.model.promoted governance audit row") 239 } 240 241 // SERVE AFTER PROMOTION: the EXISTING serve path now reflects the learned coefficient. The segment_base_rate 242 // driver must gather the real 0.857 segment win rate (proving the pgtype.Numeric decode fix) and contribute a 243 // non zero positive term, and the served probability must move UP off the prior only 0.60 baseline toward the 244 // high segment signal. 245 probAfterStr, answerAfter := servedProbability(t, orgHigh, highAgency) 246 probAfter := mustFloat(t, probAfterStr) 247 t.Logf("served probability after promotion (learned): %s", probAfterStr) 248 if strings.Contains(answerAfter, "segment_base_rate contributes 0 ") { 249 t.Errorf("the learned answer must show a non zero segment_base_rate contribution, got %q", answerAfter) 250 } 251 if !strings.Contains(answerAfter, "value 0.8") { 252 t.Errorf("the serve path must gather the real high segment win rate near 0.857, not zero; got %q", answerAfter) 253 } 254 if probAfter <= probBefore { 255 t.Errorf("the served probability must move up once the learned model is champion, before %v after %v", probBefore, probAfter) 256 } 257 258 // NO HISTORY under the LEARNED champion: a segment with no decided bids must rest on the base rate (the 259 // feature is imputed at the model training mean), NOT extrapolate off a misleading zero to a spuriously low 260 // number. Serve a fresh agency with no bids and assert the probability stays in a plausible base rate band. 261 noHistoryAgency := "NOHIST " + run 262 noHistOrg := seedOrg(t) 263 probNoHistStr, answerNoHist := servedProbability(t, noHistOrg, noHistoryAgency) 264 probNoHist := mustFloat(t, probNoHistStr) 265 t.Logf("served probability for a no history segment under the learned champion: %s", probNoHistStr) 266 // The imputed value must rest in a plausible base rate band, distinctly NOT the low extrapolation a zero 267 // feature would produce (which lands near 0.04 for this signal). The imputed mean surfaces as the value the 268 // segment_base_rate driver is scored at. 269 if probNoHist < 0.3 || probNoHist > 0.75 { 270 t.Errorf("a no history segment must rest on the base rate under a learned champion, got %v in %q", probNoHist, answerNoHist) 271 } 272 if strings.Contains(answerNoHist, "segment_base_rate contributes 0 ") { 273 t.Errorf("a no history segment under a learned champion must impute the training mean, not a zero contribution, got %q", answerNoHist) 274 } 275 276 // The frozen engine path stays untouched: the answer is an opportunity_score, discloses it is not a per 277 // candidate match score, and leaks no engine score. 278 if !strings.Contains(answerAfter, "opportunity_score") { 279 t.Errorf("the served answer must remain an opportunity_score, got %q", answerAfter) 280 } 281 if !strings.Contains(strings.ToLower(answerAfter), "not a per candidate match score") { 282 t.Errorf("the served answer must disclose it is not a per candidate match score, got %q", answerAfter) 283 } 284 for _, scoreWord := range []string{"total_score", "band=", "scorecandidate"} { 285 if strings.Contains(strings.ToLower(answerAfter), scoreWord) { 286 t.Errorf("the win probability answer must leak no engine score, found %q in %q", scoreWord, answerAfter) 287 } 288 } 289} 290