Janady13/missionmatch / internal/modules/ai/mag_test.go

1package ai 2 3import ( 4 "context" 5 "strings" 6 "testing" 7 8 "github.com/shopspring/decimal" 9 10 "github.com/missionworkforce/missionmatch/internal/platform/tenant" 11) 12 13// TestWinProbabilityModelIntentRegisteredAndSafe proves the MAG win probability intent is wired safely into 14// the closed Registry: it is registered, it is a model intent (Executor ExecutorModel), the whole Registry 15// still validates (the MAG branch asserts its static shape), its OutputKind is the legal opportunity_score, 16// its ModelKey is non empty, and its FeatureIntents are all registered names. This is the unit guarantee that 17// a model intent cannot ship with an illegal shape. 18func TestWinProbabilityModelIntentRegisteredAndSafe(t *testing.T) { 19 intent, ok := IntentByName(IntentOpportunityWinProbability) 20 if !ok { 21 t.Fatalf("intent %q is not registered", IntentOpportunityWinProbability) 22 } 23 if intent.Executor != ExecutorModel { 24 t.Fatalf("the win probability intent must be an ExecutorModel intent, got %q", intent.Executor) 25 } 26 if intent.Model == nil { 27 t.Fatal("the win probability intent must carry a ModelSpec") 28 } 29 if intent.Model.OutputKind != OutputKindOpportunityScore { 30 t.Errorf("the win probability intent must emit opportunity_score, got %q", intent.Model.OutputKind) 31 } 32 if intent.Model.ModelKey != WinProbabilityModelKey { 33 t.Errorf("the win probability intent model key must be %q, got %q", WinProbabilityModelKey, intent.Model.ModelKey) 34 } 35 if strings.TrimSpace(intent.SQL) != "" { 36 t.Error("a model intent must carry no reachable SQL") 37 } 38 // The whole Registry must still validate with the model intent and the new allowlist entries. 39 if err := ValidateRegistry(); err != nil { 40 t.Fatalf("ValidateRegistry failed with the win probability model intent: %v", err) 41 } 42 // The three MAG coefficient tables must be in the read allowlist (the reviewed decision). 43 for _, tbl := range []string{"aedt_learned_models", "aedt_model_coefficients", "aedt_calibration_bins"} { 44 if !ReadAllowlist[tbl] { 45 t.Errorf("the MAG coefficient table %q must be in the AI ReadAllowlist", tbl) 46 } 47 } 48} 49 50// TestIsModelIntentClassification pins the routing classification: the win probability intent is a model 51// intent, and it is NOT a plain SQL intent; the existing TAG and graph intents are NOT model intents. A 52// misclassification would route a model intent through the raw SQL path (which rejects it) or run a SQL intent 53// through the model executor. 54func TestIsModelIntentClassification(t *testing.T) { 55 if !IsModelIntent(IntentOpportunityWinProbability) { 56 t.Fatal("the win probability intent must be a model intent") 57 } 58 // A model intent is not a SQL intent. 59 intent, _ := IntentByName(IntentOpportunityWinProbability) 60 if intent.IsSQLIntent() { 61 t.Error("the win probability intent must not be a SQL intent") 62 } 63 // Existing intents are NOT model intents. 64 for _, name := range []string{IntentBidWinRateBySegment, IntentLaborRateBand, "count_candidates_by_clearance"} { 65 if IsModelIntent(name) { 66 t.Errorf("%q is not a model intent, it must not classify as one", name) 67 } 68 it, _ := IntentByName(name) 69 if !it.IsSQLIntent() { 70 t.Errorf("%q must be a SQL intent", name) 71 } 72 } 73 // An unknown name is not a model intent. 74 if IsModelIntent("no_such_intent") { 75 t.Error("an unknown name must not be a model intent") 76 } 77} 78 79// TestValidateRegistryRejectsIllegalModelIntent proves the MAG branch of ValidateRegistry rejects a crafted 80// model intent with an illegal OutputKind (a candidate score, the exact thing the enum forbids), an empty 81// model key, an eeo feature intent, or a feature intent that is not in the Registry. It calls the branch 82// directly on crafted intents so the closed Registry is never mutated. This is the load bearing proof that a 83// candidate score is structurally impossible on the MAG surface. 84func TestValidateRegistryRejectsIllegalModelIntent(t *testing.T) { 85 registered := map[string]bool{IntentBidWinRateBySegment: true} 86 bad := map[string]QueryIntent{ 87 "candidate score output kind": { 88 Name: "crafted_candidate_scorer", 89 Executor: ExecutorModel, 90 Model: &ModelSpec{ModelKey: "k", OutputKind: "candidate_score", FeatureIntents: []string{IntentBidWinRateBySegment}}, 91 }, 92 "empty output kind": { 93 Name: "crafted_empty_kind", 94 Executor: ExecutorModel, 95 Model: &ModelSpec{ModelKey: "k", OutputKind: "", FeatureIntents: []string{IntentBidWinRateBySegment}}, 96 }, 97 "empty model key": { 98 Name: "crafted_no_key", 99 Executor: ExecutorModel, 100 Model: &ModelSpec{ModelKey: "", OutputKind: OutputKindOpportunityScore, FeatureIntents: []string{IntentBidWinRateBySegment}}, 101 }, 102 "eeo feature intent": { 103 Name: "crafted_eeo_feature", 104 Executor: ExecutorModel, 105 Model: &ModelSpec{ModelKey: "k", OutputKind: OutputKindConfidence, FeatureIntents: []string{"eeo.candidate_demographics"}}, 106 }, 107 "unregistered feature intent": { 108 Name: "crafted_unregistered", 109 Executor: ExecutorModel, 110 Model: &ModelSpec{ModelKey: "k", OutputKind: OutputKindDisplayOrder, FeatureIntents: []string{"no_such_intent"}}, 111 }, 112 "no feature intents": { 113 Name: "crafted_no_features", 114 Executor: ExecutorModel, 115 Model: &ModelSpec{ModelKey: "k", OutputKind: OutputKindOpportunityScore, FeatureIntents: nil}, 116 }, 117 "reachable sql on a model intent": { 118 Name: "crafted_with_sql", 119 Executor: ExecutorModel, 120 SQL: "SELECT 1 FROM candidates", 121 Model: &ModelSpec{ModelKey: "k", OutputKind: OutputKindOpportunityScore, FeatureIntents: []string{IntentBidWinRateBySegment}}, 122 }, 123 "nil model spec": { 124 Name: "crafted_nil_spec", 125 Executor: ExecutorModel, 126 Model: nil, 127 }, 128 } 129 for name, it := range bad { 130 if err := validateModelIntent(it, registered); err == nil { 131 t.Errorf("validateModelIntent must reject %q", name) 132 } 133 } 134 // A legal model intent passes. 135 good := QueryIntent{ 136 Name: "crafted_ok", 137 Executor: ExecutorModel, 138 Model: &ModelSpec{ModelKey: "k", OutputKind: OutputKindOpportunityScore, FeatureIntents: []string{IntentBidWinRateBySegment}}, 139 } 140 if err := validateModelIntent(good, registered); err != nil { 141 t.Errorf("a legal model intent must pass: %v", err) 142 } 143 // The OutputKind enum forbids a candidate score at the validOutputKind check directly. 144 for _, kind := range []string{"candidate_score", "match_score", "band", "total_score", ""} { 145 if validOutputKind(kind) { 146 t.Errorf("validOutputKind must reject %q; a candidate score is forbidden by construction", kind) 147 } 148 } 149 for _, kind := range []string{OutputKindOpportunityScore, OutputKindDisplayOrder, OutputKindConfidence} { 150 if !validOutputKind(kind) { 151 t.Errorf("validOutputKind must accept the legal kind %q", kind) 152 } 153 } 154} 155 156// TestExecuteIntentRejectsModelIntentOnRawPath proves the defense in depth guard: the generic ExecuteIntent 157// (the raw SQL path) hard rejects the win probability model intent, so a model intent can never run as raw 158// SQL even if a future caller tried it. It needs no database (the rejection precedes any query). 159func TestExecuteIntentRejectsModelIntentOnRawPath(t *testing.T) { 160 intent, ok := IntentByName(IntentOpportunityWinProbability) 161 if !ok { 162 t.Fatalf("the win probability intent %q must be registered", IntentOpportunityWinProbability) 163 } 164 // A non nil db is required so the rejection is on the model kind, not the nil db guard. The db pointer's 165 // Pool is never dereferenced because the model check precedes the query. 166 _, err := ExecuteIntent(context.Background(), &tenant.DB{}, intent, []any{"agency", "naics"}) 167 if err == nil { 168 t.Fatal("ExecuteIntent must reject the win probability model intent on the raw path, got nil error") 169 } 170 if !strings.Contains(err.Error(), "ExecuteModelIntent") { 171 t.Errorf("the raw path rejection of a model intent must name ExecuteModelIntent, got %v", err) 172 } 173} 174 175// TestScoreModelIntentGolden is the golden test of the MAG serving math: on a fixed champion (a known 176// intercept and a known feature coefficient) and a fixed feature vector, the decimal log odds, the sigmoid, 177// and the contribution trace are exactly as expected, and the contributions sum to the log odds. It proves 178// the surfaced numbers are decimal and the additive explanation is faithful. 179func TestScoreModelIntentGolden(t *testing.T) { 180 // A champion with intercept logit(0.60) approx 0.4054651081 and a feature coefficient of 0.5. No 181 // calibration bins, so the calibration is the identity map. 182 intercept := decimal.RequireFromString("0.4054651081") 183 model := championModel{ 184 ModelKey: "test_model", 185 Version: 1, 186 OutputKind: OutputKindOpportunityScore, 187 Intercept: intercept, 188 Coefficients: map[string]decimal.Decimal{ 189 "segment_base_rate": decimal.RequireFromString("0.5"), 190 }, 191 } 192 feats := []Contribution{ 193 {FeatureName: "segment_base_rate", Value: decimal.RequireFromString("0.4"), Rationale: "the segment base rate"}, 194 } 195 196 value, trace := scoreModelIntent(model, feats) 197 198 // The log odds is intercept + 0.5 * 0.4 = 0.4054651081 + 0.20 = 0.6054651081. 199 // The trace: the intercept term (value 1, coefficient and contribution = intercept) plus the feature term 200 // (value 0.4, coefficient 0.5, contribution 0.20). 201 if len(trace) != 2 { 202 t.Fatalf("the trace must carry the intercept plus one feature, got %d entries", len(trace)) 203 } 204 if trace[0].FeatureName != interceptFeatureName { 205 t.Errorf("the first trace entry must be the intercept, got %q", trace[0].FeatureName) 206 } 207 if !trace[0].Contribution.Equal(intercept) { 208 t.Errorf("the intercept contribution must be %s, got %s", intercept, trace[0].Contribution) 209 } 210 wantFeatureContribution := decimal.RequireFromString("0.20") 211 if !trace[1].Contribution.Equal(wantFeatureContribution) { 212 t.Errorf("the feature contribution must be 0.20, got %s", trace[1].Contribution) 213 } 214 if !trace[1].Coefficient.Equal(decimal.RequireFromString("0.5")) { 215 t.Errorf("the feature coefficient must be 0.5, got %s", trace[1].Coefficient) 216 } 217 218 // The contributions must sum to the log odds (the additive explanation is faithful). 219 sum := decimal.Zero 220 for _, c := range trace { 221 sum = sum.Add(c.Contribution) 222 } 223 wantLogOdds := decimal.RequireFromString("0.6054651081") 224 if !sum.Equal(wantLogOdds) { 225 t.Errorf("the contributions must sum to the log odds %s, got %s", wantLogOdds, sum) 226 } 227 228 // The calibrated probability is sigmoid(0.6054651081) approx 0.6469. With no calibration bins the identity 229 // map passes the raw through. The surfaced value is a decimal in [0, 1]. 230 if value.LessThan(decimal.Zero) || value.GreaterThan(decimal.NewFromInt(1)) { 231 t.Fatalf("the probability must be in [0, 1], got %s", value) 232 } 233 // sigmoid(0.6054651081) = 1 / (1 + e^-0.6054651081). e^-0.6054651081 approx 0.54580, so p approx 0.64695. 234 lo := decimal.RequireFromString("0.646") 235 hi := decimal.RequireFromString("0.648") 236 if value.LessThan(lo) || value.GreaterThan(hi) { 237 t.Errorf("the calibrated probability must be near 0.647, got %s", value) 238 } 239} 240 241// TestScoreModelIntentPriorOnlyReturnsBaseRate proves the prior only invariant: with every learned coefficient 242// zero (the v0 seed), the model returns the sigmoid of the intercept alone (the calibrated base rate) no 243// matter what the feature value is, because a zero coefficient makes the feature contribution zero. This is 244// the correctness proof that the prior only v0 returns the shrunk base rate. 245func TestScoreModelIntentPriorOnlyReturnsBaseRate(t *testing.T) { 246 intercept := decimal.RequireFromString("0.4054651081") // logit(0.60) 247 model := championModel{ 248 OutputKind: OutputKindOpportunityScore, 249 Intercept: intercept, 250 Coefficients: map[string]decimal.Decimal{"segment_base_rate": decimal.Zero}, // prior only: beta = 0 251 } 252 // Two very different feature values must yield the SAME probability, because the coefficient is zero. 253 valueA, _ := scoreModelIntent(model, []Contribution{{FeatureName: "segment_base_rate", Value: decimal.RequireFromString("0.10")}}) 254 valueB, _ := scoreModelIntent(model, []Contribution{{FeatureName: "segment_base_rate", Value: decimal.RequireFromString("0.90")}}) 255 if !valueA.Equal(valueB) { 256 t.Errorf("a prior only model (zero coefficient) must return the same base rate for any feature, got %s and %s", valueA, valueB) 257 } 258 // The base rate is sigmoid(logit(0.60)) = 0.60. Allow a small rounding tolerance. 259 lo := decimal.RequireFromString("0.599") 260 hi := decimal.RequireFromString("0.601") 261 if valueA.LessThan(lo) || valueA.GreaterThan(hi) { 262 t.Errorf("the prior only base rate must be near 0.60, got %s", valueA) 263 } 264} 265 266// TestApplyCalibrationIdentityAndMap proves the calibration behavior: an empty bin set is the identity map (the 267// raw passes through), and a bin that covers the raw remaps it to the bin's empirical rate. The result is 268// always clamped to [0, 1]. 269func TestApplyCalibrationIdentityAndMap(t *testing.T) { 270 raw := decimal.RequireFromString("0.647") 271 // Empty bins: identity. 272 if got := applyCalibration(nil, raw); !got.Equal(raw) { 273 t.Errorf("an empty calibration set must be the identity map, got %s", got) 274 } 275 // A bin covering the raw remaps it. 276 bins := []calibrationBin{{Lo: decimal.RequireFromString("0.60"), Hi: decimal.RequireFromString("0.70"), EmpiricalRate: decimal.RequireFromString("0.55")}} 277 if got := applyCalibration(bins, raw); !got.Equal(decimal.RequireFromString("0.55")) { 278 t.Errorf("a covering bin must remap the raw to its empirical rate, got %s", got) 279 } 280 // A raw outside every bin passes through (identity for the uncovered range). 281 if got := applyCalibration(bins, decimal.RequireFromString("0.20")); !got.Equal(decimal.RequireFromString("0.20")) { 282 t.Errorf("a raw outside every bin must pass through, got %s", got) 283 } 284} 285