Janady13/missionmatch / internal/integration/mag_win_probability_integration_test.go

1//go:build integration 2 3// MAG (Model Augmented Generation) win probability integration. Proves at runtime against a REAL Postgres and 4// the migrated 0036 coefficient store what the unit tests cannot: the first model backed intent on the closed 5// Registry gathers its feature (the advisory bid_win_rate_by_segment base rate) through the guarded 6// ExecuteIntent path RLS scoped, loads the prior only v0 champion coefficients from aedt_learned_models, scores 7// a CALIBRATED probability in shopspring/decimal, narrates the ordered driver contributions and the prior only 8// reliability disclosure through the deterministic fallback (a nil inference client), computes NO candidate 9// score, and emits a counts only aedt.ai.model_answered audit carrying model_key, version, output_kind, and 10// feature_count and NO value or PII. It also proves the raw ExecuteIntent path rejects the model intent. 11// 12// Run: go test -tags=integration ./internal/integration/ -run MAGWinProbability -v 13// Env: MM_APP_DSN, MM_MIGRATOR_DSN (required). The orchestrator applies migration 0036 (goose v36) first. 14package integration 15 16import ( 17 "context" 18 "strings" 19 "testing" 20 21 "github.com/google/uuid" 22 "github.com/jackc/pgx/v5" 23 24 "github.com/missionworkforce/missionmatch/internal/modules/ai" 25 "github.com/missionworkforce/missionmatch/internal/modules/bidoutcomes" 26 "github.com/missionworkforce/missionmatch/internal/platform/audit" 27) 28 29// TestMAGWinProbabilityPriorOnlyReturnsCalibratedBaseRate seeds a segment with decided bid outcomes so the 30// advisory base rate is defined, then runs the win probability MAG intent through the orchestrator with a nil 31// inference client (so the answer is the deterministic rendering). It asserts the answer carries a calibrated 32// probability in [0, 1] near the seeded base rate (prior only, betas zero, so the number is the shrunk market 33// base rate), the ordered driver contributions, the prior only reliability disclosure, and NO candidate score, 34// and that the audit holds model_key, version, output_kind, and feature_count and NO value or PII. 35func TestMAGWinProbabilityPriorOnlyReturnsCalibratedBaseRate(t *testing.T) { 36 org := seedOrg(t) 37 run := uuid.NewString()[:8] 38 agency := "DEPT OF THE NAVY " + run // unique per run so the segment is isolated 39 40 // Seed a segment with decided bid outcomes so the base rate is defined: two wins and one loss for the 41 // agency, so the advisory win rate is 2/3. The prior only v0 model returns the calibrated intercept base 42 // rate (0.60) regardless, because every learned coefficient is zero; the base rate is the FEATURE, which 43 // the trace surfaces as a driver. 44 req := seedRequirementWithAgency(t, org, agency) 45 svc := bidoutcomes.NewService(appDB, nil) 46 recordOutcome(t, svc, org, req, bidoutcomes.ResultWin) 47 recordOutcome(t, svc, org, req, bidoutcomes.ResultWin) 48 recordOutcome(t, svc, org, req, bidoutcomes.ResultLoss) 49 50 orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) 51 resp, err := orch.Answer(ctxOrg(org), "", ai.IntentOpportunityWinProbability, []any{agency, ""}) 52 if err != nil { 53 t.Fatalf("win probability answer: %v", err) 54 } 55 56 // (a) The answer is on the graph route and is not refused for an authenticated tenant. 57 if resp.Refused { 58 t.Fatalf("an authenticated win probability question must not be refused, got %+v", resp) 59 } 60 if resp.Route != ai.RouteGraph { 61 t.Errorf("the model intent must dispatch to the graph route, got %q", resp.Route) 62 } 63 64 // (b) The answer carries a calibrated opportunity_score probability near the seeded base rate. Prior only 65 // v0: the intercept is logit(0.60), betas zero, so the calibrated probability is near 0.60. The answer 66 // names it as an opportunity_score, never a candidate score. 67 if !strings.Contains(resp.Answer, "opportunity_score") { 68 t.Errorf("the answer must name the opportunity_score output kind, got %q", resp.Answer) 69 } 70 if !strings.Contains(resp.Answer, "0.6") { 71 t.Errorf("the prior only calibrated probability must be near 0.60, got %q", resp.Answer) 72 } 73 74 // (c) The ordered driver contributions are present: the intercept (the base) and the segment base rate 75 // feature. 76 if !strings.Contains(resp.Answer, "intercept contributes") { 77 t.Errorf("the answer must carry the ordered intercept driver contribution, got %q", resp.Answer) 78 } 79 if !strings.Contains(resp.Answer, "segment_base_rate contributes") { 80 t.Errorf("the answer must carry the ordered segment base rate driver contribution, got %q", resp.Answer) 81 } 82 83 // (d) The prior only reliability disclosure is present. 84 if !strings.Contains(resp.Answer, "prior only estimate") { 85 t.Errorf("the answer must carry the prior only reliability disclosure, got %q", resp.Answer) 86 } 87 88 // (e) The frozen engine path is untouched: no per candidate score, no band, no total_score, and the 89 // disclosure explicitly names that this is not a per candidate match score. 90 if !strings.Contains(strings.ToLower(resp.Answer), "not a per candidate match score") { 91 t.Errorf("the answer must disclose it is not a per candidate match score, got %q", resp.Answer) 92 } 93 for _, scoreWord := range []string{"total_score", "band=", "candidatescore", "scorecandidate"} { 94 if strings.Contains(strings.ToLower(resp.Answer), scoreWord) { 95 t.Errorf("the win probability answer must report no engine score, leaked %q in %q", scoreWord, resp.Answer) 96 } 97 } 98 99 // (f) The counts only audit holds model_key, version, output_kind, and feature_count and NO value or PII. 100 var payloads []string 101 if err := appDB.RunInTenantTx(ctxOrg(org), func(ctx context.Context, tx pgx.Tx) error { 102 rows, qerr := tx.Query(ctx, 103 "SELECT new_values::text FROM audit_logs WHERE organization_id=$1 AND action='aedt.ai.model_answered'", org) 104 if qerr != nil { 105 return qerr 106 } 107 defer rows.Close() 108 for rows.Next() { 109 var p string 110 if serr := rows.Scan(&p); serr != nil { 111 return serr 112 } 113 payloads = append(payloads, p) 114 } 115 return rows.Err() 116 }); err != nil { 117 t.Fatalf("read model audit: %v", err) 118 } 119 if len(payloads) == 0 { 120 t.Fatal("a win probability answer must emit an aedt.ai.model_answered audit event") 121 } 122 for _, p := range payloads { 123 if !strings.Contains(p, ai.WinProbabilityModelKey) { 124 t.Errorf("the model audit must record the model_key, got %q", p) 125 } 126 for _, want := range []string{"version", "output_kind", "feature_count", "opportunity_score"} { 127 if !strings.Contains(p, want) { 128 t.Errorf("the model audit must record %q, got %q", want, p) 129 } 130 } 131 // NO probability value, NO agency, NO answer text, NO candidate id in the audit payload. The word 132 // probability legitimately appears inside the model_key opportunity_win_probability, so the leak check 133 // looks for a decimal value (a probability renders as 0.xxx) and for answer or candidate references, 134 // not the bare word. 135 if strings.Contains(p, agency) { 136 t.Errorf("the model audit must hold no agency, leaked it in %q", p) 137 } 138 for _, leak := range []string{"0.", "answer", "candidate"} { 139 if strings.Contains(p, leak) { 140 t.Errorf("the model audit must hold no value, answer text, or candidate id, leaked %q in %q", leak, p) 141 } 142 } 143 } 144} 145 146// TestMAGWinProbabilityNoBidHistoryStillReturnsPrior proves the prior only floor holds honestly when a segment 147// has NO decided bid history: the feature is a zero base rate, but the model still returns the calibrated 148// intercept base rate (near 0.60), because the prior is the intercept, not the feature. The answer states the 149// no history rationale honestly. 150func TestMAGWinProbabilityNoBidHistoryStillReturnsPrior(t *testing.T) { 151 org := seedOrg(t) 152 agency := "AGENCY WITH NO BIDS " + uuid.NewString()[:8] 153 154 orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) 155 resp, err := orch.Answer(ctxOrg(org), "", ai.IntentOpportunityWinProbability, []any{agency, ""}) 156 if err != nil { 157 t.Fatalf("win probability answer: %v", err) 158 } 159 if resp.Refused || resp.Route != ai.RouteGraph { 160 t.Fatalf("a bound tenant win probability question must answer on the graph route, got %+v", resp) 161 } 162 // The prior only base rate is still returned near 0.60 (the intercept), even with a zero base rate feature. 163 if !strings.Contains(resp.Answer, "0.6") { 164 t.Errorf("with no bid history the prior only base rate must still be near 0.60, got %q", resp.Answer) 165 } 166 // The feature rationale states the no history honestly (via the segment base rate driver line and the 167 // disclosure). The segment base rate driver contributes zero (coefficient zero), so it is present but zero. 168 if !strings.Contains(resp.Answer, "segment_base_rate contributes 0") { 169 t.Errorf("the segment base rate driver must contribute zero with a zero coefficient, got %q", resp.Answer) 170 } 171} 172 173// TestMAGWinProbabilityRejectsRawIntentPath proves the defense in depth guard: the generic ExecuteIntent (the 174// raw path that returns database rows verbatim) hard rejects the win probability model intent, so a model 175// intent can never run as raw SQL even if a future caller tried it. 176func TestMAGWinProbabilityRejectsRawIntentPath(t *testing.T) { 177 intent, ok := ai.IntentByName(ai.IntentOpportunityWinProbability) 178 if !ok { 179 t.Fatalf("the win probability intent %q must be registered", ai.IntentOpportunityWinProbability) 180 } 181 _, err := ai.ExecuteIntent(ctxOrg(seedOrg(t)), appDB, intent, []any{"agency", "naics"}) 182 if err == nil { 183 t.Fatal("ExecuteIntent must reject the win probability model intent on the raw path, got nil error") 184 } 185 if !strings.Contains(err.Error(), "ExecuteModelIntent") { 186 t.Errorf("the raw path rejection of a model intent must name ExecuteModelIntent, got %v", err) 187 } 188} 189 190// TestMAGWinProbabilityNaturalLanguageRoute proves the conservative classifier binds a quoted agency for a 191// win probability question and routes it to the model intent, returning the calibrated probability. It uses a 192// nil inference client so the answer is deterministic. 193func TestMAGWinProbabilityNaturalLanguageRoute(t *testing.T) { 194 org := seedOrg(t) 195 run := uuid.NewString()[:8] 196 agency := "DEPT OF THE ARMY " + run 197 req := seedRequirementWithAgency(t, org, agency) 198 svc := bidoutcomes.NewService(appDB, nil) 199 recordOutcome(t, svc, org, req, bidoutcomes.ResultWin) 200 recordOutcome(t, svc, org, req, bidoutcomes.ResultLoss) 201 202 orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) 203 // A free form win probability question that quotes the agency: the classifier binds it and routes to the 204 // model intent (no explicit intent name). 205 question := "what is the win probability for \"" + agency + "\"" 206 resp, err := orch.Answer(ctxOrg(org), question, "", nil) 207 if err != nil { 208 t.Fatalf("natural language win probability answer: %v", err) 209 } 210 if resp.Refused { 211 t.Fatalf("the natural language win probability question must not be refused, got %+v", resp) 212 } 213 if resp.Route != ai.RouteGraph { 214 t.Errorf("the classified win probability question must dispatch to the graph route, got %q", resp.Route) 215 } 216 if !strings.Contains(resp.Answer, "opportunity_score") || !strings.Contains(resp.Answer, "0.6") { 217 t.Errorf("the classified answer must carry the calibrated opportunity_score near 0.60, got %q", resp.Answer) 218 } 219} 220 221// verifyChampionSeeded is a small guard that the migrator applied the 0036 seed: exactly one champion row for 222// the win probability model, with the opportunity_score output kind and the intercept and segment base rate 223// coefficients. It reads through the migrator so it sees the global reference tables directly. 224func verifyChampionSeeded(t *testing.T) { 225 t.Helper() 226 var version int 227 var outputKind string 228 if err := migrator.QueryRow(context.Background(), 229 `SELECT version, output_kind FROM aedt_learned_models WHERE model_key = $1 AND champion = true`, 230 ai.WinProbabilityModelKey).Scan(&version, &outputKind); err != nil { 231 t.Fatalf("the 0036 champion seed must be present: %v", err) 232 } 233 if outputKind != "opportunity_score" { 234 t.Errorf("the seeded champion output kind must be opportunity_score, got %q", outputKind) 235 } 236 var nCoef int 237 if err := migrator.QueryRow(context.Background(), 238 `SELECT count(*) FROM aedt_model_coefficients WHERE model_key = $1 AND version = $2`, 239 ai.WinProbabilityModelKey, version).Scan(&nCoef); err != nil { 240 t.Fatalf("read seeded coefficients: %v", err) 241 } 242 if nCoef < 2 { 243 t.Errorf("the seed must carry the intercept and the segment base rate coefficient, got %d rows", nCoef) 244 } 245} 246 247// TestMAGChampionSeeded is the seed guard, run standalone so a missing 0036 seed fails fast with a clear 248// message rather than surfacing as a no champion error deep in a serving test. 249func TestMAGChampionSeeded(t *testing.T) { 250 verifyChampionSeeded(t) 251} 252