Janady13/missionmatch / internal/modules/ai/planner_test.go

1package ai 2 3import ( 4 "context" 5 "reflect" 6 "sort" 7 "testing" 8) 9 10// representativeAvail is a full available bag: every extractor fired, so any Registry intent whose params are 11// all mapped should bind from it. The unit tests use it to prove that every selectable intent binds and that 12// the unmapped or unavailable cases drop. 13func representativeAvail() availableParams { 14 return availableParams{ 15 Agency: "DEPT OF THE AIR FORCE", 16 AgencyOK: true, 17 NAICS: "541512", 18 NAICSOK: true, 19 UUID: "11111111-1111-1111-1111-111111111111", 20 UUIDOK: true, 21 HeldClearance: "secret", 22 HeldOK: true, 23 UEI: "ABC123DEF456", 24 UEIOK: true, 25 WindowMonths: 24, 26 Compartments: "SI, TK", 27 CompartmentsOK: true, 28 OccupationCode: "113021", 29 OccupationOK: true, 30 Area: "HI", 31 AreaOK: true, 32 } 33} 34 35// TestExtractAvailableParams proves the bag is built deterministically from the question text alone, through 36// the existing extractors, and that an absent token leaves its slot unset rather than guessed. 37func TestExtractAvailableParams(t *testing.T) { 38 // A compound question that carries a quoted agency, a bare naics, a canonical uuid, a held clearance phrase, 39 // a quoted uei, and an explicit window in months. 40 q := `For the "DEPT OF THE AIR FORCE" in naics 541512, who on our bench can staff requirement ` + 41 `11111111-1111-1111-1111-111111111111 with a secret clearance and partner "ABC123DEF456" over a 24 month window` 42 a := extractAvailableParams(q) 43 if !a.AgencyOK || a.Agency != "DEPT OF THE AIR FORCE" { 44 t.Errorf("agency must bind from the quoted span, got %q ok=%v", a.Agency, a.AgencyOK) 45 } 46 if !a.NAICSOK || a.NAICS != "541512" { 47 t.Errorf("naics must bind from the bare six digit token, got %q ok=%v", a.NAICS, a.NAICSOK) 48 } 49 if !a.UUIDOK || a.UUID != "11111111-1111-1111-1111-111111111111" { 50 t.Errorf("uuid must bind from the canonical id, got %q ok=%v", a.UUID, a.UUIDOK) 51 } 52 if !a.HeldOK || a.HeldClearance != "secret" { 53 t.Errorf("held clearance must bind from the named phrase, got %q ok=%v", a.HeldClearance, a.HeldOK) 54 } 55 if !a.UEIOK || a.UEI != "ABC123DEF456" { 56 t.Errorf("uei must bind from the quoted twelve char token, got %q ok=%v", a.UEI, a.UEIOK) 57 } 58 if a.WindowMonths != 24 { 59 t.Errorf("window months must read the stated window, got %d", a.WindowMonths) 60 } 61 62 // A bare question carrying nothing extractable leaves every slot unset; the window defaults to zero so the 63 // intent applies its own clamp. 64 empty := extractAvailableParams("how many candidates are proposal ready") 65 if empty.AgencyOK || empty.NAICSOK || empty.UUIDOK || empty.HeldOK || empty.UEIOK || empty.CompartmentsOK { 66 t.Errorf("an unextractable question must leave every slot unset, got %+v", empty) 67 } 68 if empty.WindowMonths != 0 { 69 t.Errorf("an unstated window must default to zero, got %d", empty.WindowMonths) 70 } 71} 72 73// TestBindIntentArgsEveryRegistryIntent walks EVERY intent in the closed Registry and asserts bindIntentArgs 74// either binds it from the representative bag (when every param name is mapped) or correctly drops it (when a 75// param name is unmapped). It also pins that the bound arg count matches the param count and that an intent 76// with no params binds an empty arg list. This is the load bearing safety proof: an intent that binds gets a 77// value for every required param, and no intent binds a param the model could have invented. 78func TestBindIntentArgsEveryRegistryIntent(t *testing.T) { 79 avail := representativeAvail() 80 for _, intent := range Registry { 81 args, ok := bindIntentArgs(intent, avail) 82 // Every param name a registered intent declares is mapped in paramBindings, so against the full 83 // representative bag every intent must bind. If a future intent adds an unmapped param name, this fails 84 // loudly, which is the intended tripwire (that intent is then simply not selectable until it is mapped). 85 if !ok { 86 t.Errorf("intent %q has an unmapped or unavailable param against the full bag; map it or it is never selectable", intent.Name) 87 continue 88 } 89 if len(args) != len(intent.Params) { 90 t.Errorf("intent %q bound %d args for %d params", intent.Name, len(args), len(intent.Params)) 91 } 92 if len(intent.Params) == 0 && args != nil { 93 t.Errorf("intent %q with no params must bind a nil arg list, got %v", intent.Name, args) 94 } 95 } 96} 97 98// TestBindIntentArgsDropsOnMissingParam proves a node drops when a required param is not available: a market 99// intent that needs an agency and a naics drops when the bag carries neither, and binds when both are present. 100func TestBindIntentArgsDropsOnMissingParam(t *testing.T) { 101 intent, ok := IntentByName(IntentSegmentCompetitorField) 102 if !ok { 103 t.Fatalf("the segment competitor field intent must be registered") 104 } 105 106 // No agency and no naics: the node drops. 107 if _, bound := bindIntentArgs(intent, availableParams{}); bound { 108 t.Errorf("a segment intent must drop when its agency and naics are not available") 109 } 110 111 // Agency only, naics missing: still drops (every required param must be available). 112 if _, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true}); bound { 113 t.Errorf("a segment intent must drop when only the agency is available") 114 } 115 116 // Both present plus the clamp zero lookback: binds with three args (agency, naics, lookback clamp). 117 args, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true}) 118 if !bound { 119 t.Fatalf("a segment intent must bind when its agency and naics are both available") 120 } 121 if len(args) != 3 || args[0] != "X" || args[1] != "541512" || args[2] != 0 { 122 t.Errorf("the segment intent must bind agency, naics, and a clamp zero lookback, got %v", args) 123 } 124} 125 126// TestValidatePlanDropsOutOfSetAndUnbindable proves the validator drops a name that is not in the Registry 127// (the closed enum check) and an intent whose params cannot bind, keeping only the bound nodes in the 128// deterministic order. 129func TestValidatePlanDropsOutOfSetAndUnbindable(t *testing.T) { 130 // The bag binds an agency and a naics (so a market segment intent binds) and a uuid (so a requirement keyed 131 // intent binds), but NOT a held clearance, so clearance_reachability is unbindable and must drop. 132 avail := availableParams{ 133 Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true, 134 UUID: "22222222-2222-2222-2222-222222222222", UUIDOK: true, 135 } 136 names := []string{ 137 "not_a_real_intent", // out of set: dropped by the closed enum check 138 IntentClearanceReachability, // in set but unbindable here (no held clearance): dropped 139 IntentSegmentCompetitorField, // binds (agency, naics, clamp) 140 IntentProposalReadyName(), // binds (requirement uuid) 141 } 142 plan := validatePlan(names, avail) 143 144 got := make([]string, len(plan.Nodes)) 145 for i, n := range plan.Nodes { 146 got[i] = n.Intent 147 } 148 want := []string{IntentProposalReadyName(), IntentSegmentCompetitorField} // sorted by name 149 if !reflect.DeepEqual(got, want) { 150 t.Errorf("validatePlan must keep only the bound, in set intents in name order, got %v want %v", got, want) 151 } 152 for _, n := range plan.Nodes { 153 if n.Intent == "not_a_real_intent" { 154 t.Error("an out of set name must never reach the validated plan") 155 } 156 if n.Intent == IntentClearanceReachability { 157 t.Error("an unbindable intent must never reach the validated plan") 158 } 159 } 160} 161 162// IntentProposalReadyName returns the registered name of the proposal ready TAG intent, a requirement keyed 163// intent, used by the validator test to prove a bound requirement intent survives. It is a test helper, not a 164// production symbol. 165func IntentProposalReadyName() string { return "proposal_ready_for_requirement" } 166 167// TestValidatePlanCapsNodeCount proves the validator never returns more than the hard node cap even if the 168// proposer names more bindable intents than the cap. It feeds every requirement keyed intent (all bind from a 169// uuid) plus market intents and asserts the result is capped at maxPlanNodes. 170func TestValidatePlanCapsNodeCount(t *testing.T) { 171 avail := representativeAvail() 172 var allNames []string 173 for _, it := range Registry { 174 allNames = append(allNames, it.Name) 175 } 176 plan := validatePlan(allNames, avail) 177 if len(plan.Nodes) > maxPlanNodes { 178 t.Errorf("the validated plan must never exceed the hard node cap %d, got %d", maxPlanNodes, len(plan.Nodes)) 179 } 180} 181 182// TestProposeStablePlanMajority drives self consistency with a STUBBED proposer (no live plane), proving that 183// only the intents that appear in a MAJORITY of the k validated plans survive, and that total disagreement 184// yields an empty plan so the caller falls back. 185func TestProposeStablePlanMajority(t *testing.T) { 186 avail := representativeAvail() 187 188 // Three samples. The segment competitor field appears in all three (a clear majority); the recompete clock 189 // appears in two of three (a majority of three is two); the agency spend trend appears in only one (not a 190 // majority). The stable set is the first two, in name order. 191 samples := [][]string{ 192 {IntentSegmentCompetitorField, IntentRecompeteClock}, 193 {IntentSegmentCompetitorField, IntentRecompeteClock, IntentAgencySpendTrend}, 194 {IntentSegmentCompetitorField}, 195 } 196 call := 0 197 stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 198 s := samples[call%len(samples)] 199 call++ 200 return s, nil 201 } 202 plan := proposeStablePlan(context.Background(), "q", avail, 3, stub) 203 204 got := make([]string, len(plan.Nodes)) 205 for i, n := range plan.Nodes { 206 got[i] = n.Intent 207 } 208 want := []string{IntentRecompeteClock, IntentSegmentCompetitorField} // sorted by name, both at or above the majority of two 209 if !reflect.DeepEqual(got, want) { 210 t.Errorf("only the majority intents must survive self consistency, got %v want %v", got, want) 211 } 212 213 // Total disagreement: three samples that share no intent yields no majority, so the plan is empty and the 214 // caller falls back to the deterministic single intent or RAG path. 215 disagree := [][]string{{IntentSegmentCompetitorField}, {IntentRecompeteClock}, {IntentAgencySpendTrend}} 216 dcall := 0 217 dstub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 218 s := disagree[dcall%len(disagree)] 219 dcall++ 220 return s, nil 221 } 222 empty := proposeStablePlan(context.Background(), "q", avail, 3, dstub) 223 if len(empty.Nodes) != 0 { 224 t.Errorf("total disagreement must yield an empty plan, got %v", empty.Nodes) 225 } 226} 227 228// TestProposeStablePlanDropsOutOfSetSamples proves the closed enum holds through self consistency: a proposer 229// that returns an out of set name in every sample never produces a node (the validator drops it each time), so 230// the stable plan is empty even though the name had a clear majority. 231func TestProposeStablePlanDropsOutOfSetSamples(t *testing.T) { 232 stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 233 return []string{"injected_intent", "drop_table_users"}, nil 234 } 235 plan := proposeStablePlan(context.Background(), "q", representativeAvail(), 3, stub) 236 if len(plan.Nodes) != 0 { 237 t.Errorf("an out of set name must never survive self consistency, got %v", plan.Nodes) 238 } 239} 240 241// TestIsCompoundQuestion pins the broadened compound signal: a question that spans TWO OR MORE distinct 242// planes (bench, market, clearance, wage, skill, lifecycle) is compound and routes to the planner; a single 243// plane question (however many words it names within that one plane) is not, so it falls through to the 244// existing single shot route and is never regressed. 245func TestIsCompoundQuestion(t *testing.T) { 246 compound := []string{ 247 `who on our bench can staff this and who is the incumbent in the market`, // bench + market 248 `our bench gap versus the recompete pipeline for the agency`, // bench + market 249 `bench coverage and the competitor field for naics 541512`, // bench + market 250 `what is the labor rate and the ts/sci coverage`, // wage + clearance 251 `the incumbent and the prevailing wage for this agency`, // market + wage 252 `our bench and their ts/sci reciprocity`, // bench + clearance 253 `adjacent skill coverage and the competitor field`, // skill + market 254 } 255 for _, q := range compound { 256 if !isCompoundQuestion(q) { 257 t.Errorf("a question spanning two or more distinct planes must be compound, got false for %q", q) 258 } 259 } 260 261 single := []string{ 262 `how many candidates hold a secret clearance`, // no plane word pair 263 `who is the incumbent for the agency`, // market only 264 `what is our bench clearance coverage`, // bench only 265 `how many requirements are open`, // neither 266 `what is the prevailing wage for a data scientist`, // wage only 267 } 268 for _, q := range single { 269 if isCompoundQuestion(q) { 270 t.Errorf("a single plane or no plane question must not be compound, got true for %q", q) 271 } 272 } 273} 274 275// TestParseIntentNames proves the JSON array parser accepts the instructed bare array, tolerates surrounding 276// prose by extracting the first bracketed array, and returns nil on unparseable output so the caller falls 277// back to RAG. 278func TestParseIntentNames(t *testing.T) { 279 if got := parseIntentNames(`["segment_competitor_field","recompete_clock"]`); !reflect.DeepEqual(got, []string{"segment_competitor_field", "recompete_clock"}) { 280 t.Errorf("a bare JSON array must parse, got %v", got) 281 } 282 if got := parseIntentNames("Here is the plan: [\"recompete_clock\"] done."); !reflect.DeepEqual(got, []string{"recompete_clock"}) { 283 t.Errorf("a bracketed array inside prose must parse, got %v", got) 284 } 285 if got := parseIntentNames("no array here"); got != nil { 286 t.Errorf("unparseable output must return nil, got %v", got) 287 } 288 if got := parseIntentNames("[not valid json}"); got != nil { 289 t.Errorf("malformed json must return nil, got %v", got) 290 } 291} 292 293// TestAvailableKindsHoldsNoValue proves the audit and prompt available kinds carry only KIND names, never a 294// value, so a requirement id, an agency name, or a UEI can never leak through the kinds list. 295func TestAvailableKindsHoldsNoValue(t *testing.T) { 296 avail := representativeAvail() 297 kinds := availableKinds(avail) 298 want := []string{"agency", "area", "compartments", "held_clearance", "naics", "occupation_code", "uei", "uuid", "window_months"} 299 sort.Strings(want) 300 if !reflect.DeepEqual(kinds, want) { 301 t.Errorf("available kinds must be the sorted kind names, got %v want %v", kinds, want) 302 } 303 for _, k := range kinds { 304 for _, value := range []string{avail.Agency, avail.NAICS, avail.UUID, avail.UEI, avail.Compartments, avail.OccupationCode, avail.Area} { 305 if k == value { 306 t.Errorf("a kind name must never equal a param value, leaked %q", value) 307 } 308 } 309 } 310} 311 312// TestEdgeCatalogIsSafe is the PII firewall assertion for dependency edges: every edge SOURCE is a Registry 313// intent and is NOT a bench (person touching) intent, so no candidate or demographic value can ever be an edge 314// value, and every edge target actually declares the param the edge binds. This asserts the safety of the 315// whole chaining feature at the catalog, so a future edge that breaks the firewall fails the build. 316func TestEdgeCatalogIsSafe(t *testing.T) { 317 for _, e := range edgeCatalog { 318 if _, ok := IntentByName(e.FromIntent); !ok { 319 t.Errorf("edge source %q is not a Registry intent", e.FromIntent) 320 } 321 if IsBenchIntent(e.FromIntent) { 322 t.Errorf("edge source %q is a bench intent: a candidate value could flow through the chain, forbidden", e.FromIntent) 323 } 324 to, ok := IntentByName(e.ToIntent) 325 if !ok { 326 t.Errorf("edge target %q is not a Registry intent", e.ToIntent) 327 continue 328 } 329 declares := false 330 for _, p := range to.Params { 331 if p.Name == e.ToParam { 332 declares = true 333 } 334 } 335 if !declares { 336 t.Errorf("edge target %q does not declare the param %q the edge binds", e.ToIntent, e.ToParam) 337 } 338 } 339} 340 341// TestEdgeAwareBindingChainsUEI proves the planner CHAINS: partner_or_competitor_ratio, whose partner_uei the 342// question does not quote, binds when its edge source (incumbent_of_record) is also selected and the naics is 343// present; partner_uei is marked edge bound (a placeholder resolved at run time) and naics binds from the 344// question. The model supplied no value and no edge. 345func TestEdgeAwareBindingChainsUEI(t *testing.T) { 346 avail := availableParams{NAICS: "541512", NAICSOK: true} // naics present, NO quoted uei 347 selected := map[string]bool{IntentIncumbentOfRecord: true, IntentPartnerOrCompetitorRatio: true} 348 to, _ := IntentByName(IntentPartnerOrCompetitorRatio) 349 args, edge, idx, ok := bindIntentArgsWithEdges(to, avail, selected) 350 if !ok { 351 t.Fatal("partner_or_competitor_ratio must bind when its edge source is selected and naics is present") 352 } 353 if edge == nil || edge.FromIntent != IntentIncumbentOfRecord { 354 t.Fatalf("expected an edge from incumbent_of_record, got %v", edge) 355 } 356 if idx != 0 || args[0] != "" { 357 t.Errorf("expected partner_uei (idx 0) to be an edge bound placeholder, got idx %d args %v", idx, args) 358 } 359 if args[1] != "541512" { 360 t.Errorf("expected naics to bind from the question, got %v", args[1]) 361 } 362} 363 364// TestEdgeDropsWhenSourceNotSelected proves the target does NOT bind when its edge source is absent, so the 365// planner never fabricates a partner_uei out of nothing. 366func TestEdgeDropsWhenSourceNotSelected(t *testing.T) { 367 avail := availableParams{NAICS: "541512", NAICSOK: true} // naics only, no quoted uei 368 selected := map[string]bool{IntentPartnerOrCompetitorRatio: true} // the edge source is NOT selected 369 to, _ := IntentByName(IntentPartnerOrCompetitorRatio) 370 if _, _, _, ok := bindIntentArgsWithEdges(to, avail, selected); ok { 371 t.Error("partner_or_competitor_ratio must NOT bind without a quoted uei and without its edge source selected") 372 } 373} 374 375// TestValidatePlanTopoOrdersSourceBeforeTarget proves a plan naming both the source and the target orders the 376// source (edge free) BEFORE the target (edge bound), so the runner has the upstream result when the target runs. 377func TestValidatePlanTopoOrdersSourceBeforeTarget(t *testing.T) { 378 avail := availableParams{Agency: "NSA", AgencyOK: true, NAICS: "541512", NAICSOK: true} 379 plan := validatePlan([]string{IntentPartnerOrCompetitorRatio, IntentSegmentCompetitorField}, avail) 380 if len(plan.Nodes) != 2 { 381 t.Fatalf("expected 2 nodes (source and target), got %d: %+v", len(plan.Nodes), plan.Nodes) 382 } 383 if plan.Nodes[0].Intent != IntentSegmentCompetitorField { 384 t.Errorf("expected the edge source (segment_competitor_field) first, got %q", plan.Nodes[0].Intent) 385 } 386 if plan.Nodes[1].Intent != IntentPartnerOrCompetitorRatio || plan.Nodes[1].EdgeFrom != IntentSegmentCompetitorField { 387 t.Errorf("expected the edge target second with EdgeFrom set, got %q EdgeFrom %q", plan.Nodes[1].Intent, plan.Nodes[1].EdgeFrom) 388 } 389} 390 391// TestFirstRowColumn proves the edge value reader returns the first row's named string column and drops (ok 392// false) on an empty result, a missing column, or a non string cell, so a dependent node never binds a wrong 393// or empty value. 394func TestFirstRowColumn(t *testing.T) { 395 r := capturedResult{Columns: []string{"incumbent_uei", "amount"}, Rows: [][]any{{"ABC123UEI", 500}}} 396 if v, ok := firstRowColumn(r, "incumbent_uei"); !ok || v != "ABC123UEI" { 397 t.Errorf("expected ABC123UEI, got %q ok %v", v, ok) 398 } 399 if _, ok := firstRowColumn(r, "missing"); ok { 400 t.Error("a missing column must return ok false") 401 } 402 if _, ok := firstRowColumn(capturedResult{Columns: []string{"incumbent_uei"}, Rows: nil}, "incumbent_uei"); ok { 403 t.Error("an empty result must return ok false") 404 } 405 if _, ok := firstRowColumn(capturedResult{Columns: []string{"n"}, Rows: [][]any{{42}}}, "n"); ok { 406 t.Error("a non string cell must return ok false: never bind a non text value to a text param") 407 } 408} 409