Janady13/missionmatch / internal/integration/planner_integration_test.go

1//go:build integration 2 3package integration 4 5import ( 6 "context" 7 "os" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/google/uuid" 13 "github.com/jackc/pgx/v5" 14 15 "github.com/missionworkforce/missionmatch/internal/modules/ai" 16 "github.com/missionworkforce/missionmatch/internal/platform/audit" 17) 18 19// TestPlannerComposesACompoundCrossPlaneQuestion is the live keystone proof of THE PLANNER: a COMPOUND 20// question that names BOTH the market plane and the bench plane and carries the bindable params (a quoted 21// agency, a bare naics, and the requirement uuid) routes to the planner, which SELECTS a set of closed 22// Registry intents, runs each through its existing guarded path, and composes ONE answer behind the 23// faithfulness gate. The model authors no SQL and supplies no param value; every value comes from a 24// deterministic extractor. The proof is STRUCTURAL, not on exact LLM text, because generation is live. 25// 26// It seeds a per run org with a requirement (exercising the bench gap), a bench that exercises both sides of 27// the gap, and a market segment (an agency and an award under it). It then poses a compound question that 28// does NOT match the single shot graph classifier (no capture, recompete, incumbent, or competitor keyword), 29// so it reaches the planner, and asserts: the answer composes a market line AND a suppressed bench line, the 30// bench counts come through RenderCount (no raw sub k number leaks), no score is computed, and the counts 31// only aedt.ai.plan_answered audit holds only names, kinds, counts, and verdicts with NO value and NO PII. 32func TestPlannerComposesACompoundCrossPlaneQuestion(t *testing.T) { 33 if os.Getenv("MM_MEGAMIND_URL") == "" { 34 t.Skip("MM_MEGAMIND_URL not set; the planner needs the live inference plane to propose a plan") 35 } 36 plane := testPlane(t) 37 org := seedOrg(t) 38 k := ai.ConfiguredSuppressionThreshold() 39 run := uuid.NewString()[:8] 40 agency := "DEPT OF THE AIR FORCE " + run // unique per run so the market segment is isolated 41 naics := "541512" 42 43 // A requirement linked to a tenant opportunity whose federal_ref points at a global opportunity_records row 44 // carrying the naics, so the bench gap and the segment both bind. Reuses the G6 seed helper. 45 req := seedRequirementForCapture(t, org, agency, "secret", naics, run) 46 47 // A mandatory required skill so the structural gap's skill arm is meaningful. 48 skill := seedSkillByName(t, "PlannerSkill-"+run) 49 seedRequiredSkill(t, org, req, skill, true) 50 51 // MEETS: k candidates hold the required clearance and the skill. DOES NOT MEET: k candidates hold none, so 52 // both gap buckets are at or above k and the bench legs publish real counts. 53 for i := 0; i < k; i++ { 54 c := seedCandidate(t, org, "secret", "active", true) 55 seedCandidateSkill(t, org, c.ID, skill) 56 } 57 for i := 0; i < k; i++ { 58 seedCandidate(t, org, "none", "active", true) 59 } 60 61 // The market segment: awards under the requirement's agency and naics, so a market intent has structure to 62 // report whichever one the planner selects. 63 recent := time.Now().UTC().AddDate(0, -3, 0) 64 insideWindow := time.Now().UTC().AddDate(0, 6, 0) 65 seedMarketAward(t, "PL-INC-"+run, "PIID-PL-IN", agency, naics, "UEIINCUMBENTP", "Planner Incumbent LLC", "6000000.00", insideWindow, false) 66 seedMarketAward(t, "PL-CF-"+run, "PIID-PL-CF", agency, naics, "UEICHALLENGP2", "Planner Challenger LLC", "2000000.00", recent, false) 67 68 orch := ai.NewOrchestrator(appDB, plane, audit.New(appDB)) 69 70 // A COMPOUND question that names the market plane and the bench plane and carries the bindable params, but 71 // avoids every single shot graph classifier keyword (no capture, recompete, incumbent, competitor, vendor, 72 // teaming, buying, or bench coverage phrase), so it reaches the planner rather than a single intent route. 73 question := `Across the federal market and our bench, for the "` + agency + `" in naics ` + naics + 74 `, give an integrated picture for requirement ` + req.String() + `.` 75 76 resp, err := orch.Answer(ctxOrg(org), question, "", nil) 77 if err != nil { 78 t.Fatalf("planner answer: %v", err) 79 } 80 if resp.Refused { 81 t.Fatalf("an authenticated compound question must not be refused, got %+v", resp) 82 } 83 if resp.Route != ai.RoutePlan { 84 t.Fatalf("a compound cross plane question must dispatch to the plan route, got %q (answer %q)", resp.Route, resp.Answer) 85 } 86 87 lower := strings.ToLower(resp.Answer) 88 89 // (b) The answer composes a market line AND a suppressed bench line. The deterministic union rendering keys 90 // each line by its intent name, so a market intent name and a bench intent name both present prove the cross 91 // plane composition. We accept any market intent and any bench intent the planner selected. 92 marketIntents := []string{ 93 ai.IntentSegmentCompetitorField, ai.IntentAgencyVendorConcentration, ai.IntentAgencySpendTrend, 94 ai.IntentVendorCooccurrenceField, ai.IntentRecompeteClock, ai.IntentRecompetePipeline, 95 ai.IntentRecompeteTeamingField, ai.IntentAgencyBuyingPattern, ai.IntentAgencyBuyingPatternDetail, 96 ai.IntentIncumbentOfRecord, 97 } 98 benchIntentNames := []string{ 99 ai.IntentBenchGapForRequirement, ai.IntentBenchReadinessDistribution, ai.IntentBenchSkillCoverage, 100 ai.IntentBenchClearanceCoverage, ai.IntentCapturePictureForRequirement, "proposal_ready_for_requirement", 101 } 102 103 // Read the ran intents from the audit (the structural truth of what the plan executed), and assert the plan 104 // spanned both planes. 105 intents := readPlanIntents(t, org) 106 if len(intents) == 0 { 107 t.Fatalf("the planner must run at least one intent; the answer was %q", resp.Answer) 108 } 109 if !containsAnyOf(intents, marketIntents) { 110 t.Errorf("the plan must run a market plane intent, ran %v", intents) 111 } 112 if !containsAnyOf(intents, benchIntentNames) { 113 t.Errorf("the plan must run a bench plane intent, ran %v", intents) 114 } 115 116 // (c) No raw sub k number leaks: when a bench cell is suppressed it renders as the sentinel, never a count. 117 // Both seeded buckets are at or above k, so the bench legs should publish real counts and not suppress; but 118 // if any cell suppresses, it must be the sentinel and never a bare sub k integer. We assert the sentinel 119 // discipline holds: a suppressed marker present means the withheld phrasing is present too. 120 if strings.Contains(resp.Answer, ai.SuppressionSentinel) && !strings.Contains(lower, "below reporting threshold") { 121 t.Errorf("a suppressed bench cell must render as the sentinel, got %q", resp.Answer) 122 } 123 124 // (d) No engine score leaks: a total_score or a band value must never appear (the planner computes no score). 125 for _, scoreWord := range []string{"total_score", "band="} { 126 if strings.Contains(lower, scoreWord) { 127 t.Errorf("the plan answer must report no engine score, leaked %q in %q", scoreWord, resp.Answer) 128 } 129 } 130 131 // (e) The counts only aedt.ai.plan_answered audit holds only names, kinds, counts, and verdicts, with NO 132 // param value, NO candidate id, NO requirement id, and NO answer text. 133 payloads := readAudits(t, org, "aedt.ai.plan_answered") 134 if len(payloads) == 0 { 135 t.Fatal("a planned answer must emit an aedt.ai.plan_answered audit event") 136 } 137 for _, p := range payloads { 138 if !strings.Contains(p, "intents") || !strings.Contains(p, "available_kinds") || !strings.Contains(p, "self_consistency") { 139 t.Errorf("the plan audit must record the ran intents, the available kinds, and the self consistency outcome, got %q", p) 140 } 141 // NO requirement id, NO agency value, NO naics value, NO candidate marker, NO answer text in the audit. 142 if strings.Contains(p, req.String()) { 143 t.Errorf("the plan audit must hold no requirement id, leaked it in %q", p) 144 } 145 if strings.Contains(p, agency) { 146 t.Errorf("the plan audit must hold no agency value, leaked it in %q", p) 147 } 148 for _, leak := range []string{"candidate", "answer", "question", "541512"} { 149 if strings.Contains(p, leak) { 150 t.Errorf("the plan audit must hold no candidate, answer, question, or param value, leaked %q in %q", leak, p) 151 } 152 } 153 } 154} 155 156// TestPlannerLeavesTheSingleShotRouteUnchanged is the regression guard: a non compound single intent question 157// still takes the unchanged single shot path (the structured TAG route here), never the planner. The planner 158// is the fallback for compound questions, not a replacement for the working single shot routes. 159func TestPlannerLeavesTheSingleShotRouteUnchanged(t *testing.T) { 160 if os.Getenv("MM_MEGAMIND_URL") == "" { 161 t.Skip("MM_MEGAMIND_URL not set; skipping the planner regression guard") 162 } 163 org := seedOrg(t) 164 for i := 0; i < 3; i++ { 165 seedCandidate(t, org, "secret", "active", true) 166 } 167 orch := ai.NewOrchestrator(appDB, testPlane(t), audit.New(appDB)) 168 169 // A single plane structured intent question (a named TAG intent) takes the TAG route, byte for byte as 170 // before, never the planner. 171 resp, err := orch.Answer(ctxOrg(org), "", "count_candidates_by_clearance", []any{"secret"}) 172 if err != nil { 173 t.Fatalf("single intent answer: %v", err) 174 } 175 if resp.Route == ai.RoutePlan { 176 t.Errorf("a single intent question must not route to the planner, got route %q", resp.Route) 177 } 178 if resp.Route != ai.RouteTAG { 179 t.Errorf("a named TAG intent must take the unchanged TAG route, got %q", resp.Route) 180 } 181} 182 183// TestPlannerRefusesAnEEOLacedCompoundQuestion proves the EEO guard runs FIRST on the plan path: a compound 184// question that names a protected attribute is refused (route plan, no answer), never planned and never run. 185func TestPlannerRefusesAnEEOLacedCompoundQuestion(t *testing.T) { 186 if os.Getenv("MM_MEGAMIND_URL") == "" { 187 t.Skip("MM_MEGAMIND_URL not set; skipping the planner EEO refusal proof") 188 } 189 org := seedOrg(t) 190 orch := ai.NewOrchestrator(appDB, testPlane(t), audit.New(appDB)) 191 192 // A compound question (market AND bench plane words) that also asks about a protected attribute. The EEO and 193 // protected attribute guards run first, so it refuses before any plan is proposed or run. 194 question := `Across the market and our bench, break the bench down by race and tell me the incumbent for the agency.` 195 resp, err := orch.Answer(ctxOrg(org), question, "", nil) 196 if err != nil { 197 t.Fatalf("an eeo laced compound question must refuse cleanly, got error %v", err) 198 } 199 if !resp.Refused { 200 t.Fatalf("an eeo laced compound question must be refused, got %+v", resp) 201 } 202 if resp.Answer != "" { 203 t.Errorf("a refusal must ship no answer, got %q", resp.Answer) 204 } 205} 206 207// readPlanIntents reads the ran intent names from the most recent aedt.ai.plan_answered audit for an org. It 208// is the structural truth of what the plan executed, independent of the live model's prose. 209func readPlanIntents(t *testing.T, org uuid.UUID) []string { 210 t.Helper() 211 var intents []string 212 payloads := readAudits(t, org, "aedt.ai.plan_answered") 213 allNames := registryIntentNames() 214 for _, p := range payloads { 215 for _, name := range allNames { 216 if strings.Contains(p, `"`+name+`"`) { 217 intents = append(intents, name) 218 } 219 } 220 } 221 return intents 222} 223 224// readAudits reads the new_values text of every audit row for an org and action. 225func readAudits(t *testing.T, org uuid.UUID, action string) []string { 226 t.Helper() 227 var payloads []string 228 if err := appDB.RunInTenantTx(ctxOrg(org), func(ctx context.Context, tx pgx.Tx) error { 229 rows, qerr := tx.Query(ctx, 230 "SELECT new_values::text FROM audit_logs WHERE organization_id=$1 AND action=$2", org, action) 231 if qerr != nil { 232 return qerr 233 } 234 defer rows.Close() 235 for rows.Next() { 236 var p string 237 if serr := rows.Scan(&p); serr != nil { 238 return serr 239 } 240 payloads = append(payloads, p) 241 } 242 return rows.Err() 243 }); err != nil { 244 t.Fatalf("read %s audit: %v", action, err) 245 } 246 return payloads 247} 248 249// registryIntentNames is the closed set of Registry intent names, used to read the ran intents out of the 250// audit payload by exact name match. 251func registryIntentNames() []string { 252 return []string{ 253 "count_candidates_by_clearance", "proposal_ready_for_requirement", "open_requirements_count", 254 "slate_decision_mix", ai.IntentSkillSubstitutionClosure, ai.IntentSkillGapBridge, 255 ai.IntentSegmentCompetitorField, ai.IntentIncumbentOfRecord, ai.IntentRecompetePipeline, 256 ai.IntentAgencyBuyingPattern, ai.IntentProcurementSuccession, ai.IntentRecompeteClock, 257 ai.IntentForecastRunway, ai.IntentAgencyBuyingPatternDetail, ai.IntentAgencyVendorConcentration, 258 ai.IntentAgencySpendTrend, ai.IntentVendorCooccurrenceField, ai.IntentPartnerOrCompetitorRatio, 259 ai.IntentRecompeteTeamingField, ai.IntentClearanceReachability, ai.IntentCompartmentRequirement, 260 ai.IntentBenchClearanceCoverage, ai.IntentBenchSkillCoverage, ai.IntentBenchReadinessDistribution, 261 ai.IntentBenchGapForRequirement, ai.IntentCapturePictureForRequirement, ai.IntentBidWinRateBySegment, 262 } 263} 264 265// containsAnyOf reports whether the haystack contains any of the needles. 266func containsAnyOf(haystack, needles []string) bool { 267 set := map[string]bool{} 268 for _, h := range haystack { 269 set[h] = true 270 } 271 for _, n := range needles { 272 if set[n] { 273 return true 274 } 275 } 276 return false 277} 278