Janady13/missionmatch / internal/modules/ai/planner_test.go

1package ai 2 3import ( 4 "context" 5 "reflect" 6 "sort" 7 "testing" 8) 9 10// representativeAvail is a full available bag: every extractor fired, so any Registry intent whose params are 11// all mapped should bind from it. The unit tests use it to prove that every selectable intent binds and that 12// the unmapped or unavailable cases drop. 13func representativeAvail() availableParams { 14 return availableParams{ 15 Agency: "DEPT OF THE AIR FORCE", 16 AgencyOK: true, 17 NAICS: "541512", 18 NAICSOK: true, 19 UUID: "11111111-1111-1111-1111-111111111111", 20 UUIDOK: true, 21 HeldClearance: "secret", 22 HeldOK: true, 23 UEI: "ABC123DEF456", 24 UEIOK: true, 25 WindowMonths: 24, 26 Compartments: "SI, TK", 27 CompartmentsOK: true, 28 OccupationCode: "113021", 29 OccupationOK: true, 30 Area: "HI", 31 AreaOK: true, 32 } 33} 34 35// TestExtractAvailableParams proves the bag is built deterministically from the question text alone, through 36// the existing extractors, and that an absent token leaves its slot unset rather than guessed. 37func TestExtractAvailableParams(t *testing.T) { 38 // A compound question that carries a quoted agency, a bare naics, a canonical uuid, a held clearance phrase, 39 // a quoted uei, and an explicit window in months. 40 q := `For the "DEPT OF THE AIR FORCE" in naics 541512, who on our bench can staff requirement ` + 41 `11111111-1111-1111-1111-111111111111 with a secret clearance and partner "ABC123DEF456" over a 24 month window` 42 a := extractAvailableParams(q) 43 if !a.AgencyOK || a.Agency != "DEPT OF THE AIR FORCE" { 44 t.Errorf("agency must bind from the quoted span, got %q ok=%v", a.Agency, a.AgencyOK) 45 } 46 if !a.NAICSOK || a.NAICS != "541512" { 47 t.Errorf("naics must bind from the bare six digit token, got %q ok=%v", a.NAICS, a.NAICSOK) 48 } 49 if !a.UUIDOK || a.UUID != "11111111-1111-1111-1111-111111111111" { 50 t.Errorf("uuid must bind from the canonical id, got %q ok=%v", a.UUID, a.UUIDOK) 51 } 52 if !a.HeldOK || a.HeldClearance != "secret" { 53 t.Errorf("held clearance must bind from the named phrase, got %q ok=%v", a.HeldClearance, a.HeldOK) 54 } 55 if !a.UEIOK || a.UEI != "ABC123DEF456" { 56 t.Errorf("uei must bind from the quoted twelve char token, got %q ok=%v", a.UEI, a.UEIOK) 57 } 58 if a.WindowMonths != 24 { 59 t.Errorf("window months must read the stated window, got %d", a.WindowMonths) 60 } 61 62 // A bare question carrying nothing extractable leaves every slot unset; the window defaults to zero so the 63 // intent applies its own clamp. 64 empty := extractAvailableParams("how many candidates are proposal ready") 65 if empty.AgencyOK || empty.NAICSOK || empty.UUIDOK || empty.HeldOK || empty.UEIOK || empty.CompartmentsOK { 66 t.Errorf("an unextractable question must leave every slot unset, got %+v", empty) 67 } 68 if empty.WindowMonths != 0 { 69 t.Errorf("an unstated window must default to zero, got %d", empty.WindowMonths) 70 } 71} 72 73// TestBindIntentArgsEveryRegistryIntent walks EVERY intent in the closed Registry and asserts bindIntentArgs 74// either binds it from the representative bag (when every param name is mapped) or correctly drops it (when a 75// param name is unmapped). It also pins that the bound arg count matches the param count and that an intent 76// with no params binds an empty arg list. This is the load bearing safety proof: an intent that binds gets a 77// value for every required param, and no intent binds a param the model could have invented. 78func TestBindIntentArgsEveryRegistryIntent(t *testing.T) { 79 avail := representativeAvail() 80 for _, intent := range Registry { 81 args, ok := bindIntentArgs(intent, avail) 82 // Every param name a registered intent declares is mapped in paramBindings, so against the full 83 // representative bag every intent must bind. If a future intent adds an unmapped param name, this fails 84 // loudly, which is the intended tripwire (that intent is then simply not selectable until it is mapped). 85 if !ok { 86 t.Errorf("intent %q has an unmapped or unavailable param against the full bag; map it or it is never selectable", intent.Name) 87 continue 88 } 89 if len(args) != len(intent.Params) { 90 t.Errorf("intent %q bound %d args for %d params", intent.Name, len(args), len(intent.Params)) 91 } 92 if len(intent.Params) == 0 && args != nil { 93 t.Errorf("intent %q with no params must bind a nil arg list, got %v", intent.Name, args) 94 } 95 } 96} 97 98// TestBindIntentArgsDropsOnMissingParam proves a node drops when a required param is not available: a market 99// intent that needs an agency and a naics drops when the bag carries neither, and binds when both are present. 100func TestBindIntentArgsDropsOnMissingParam(t *testing.T) { 101 intent, ok := IntentByName(IntentSegmentCompetitorField) 102 if !ok { 103 t.Fatalf("the segment competitor field intent must be registered") 104 } 105 106 // No agency and no naics: the node drops. 107 if _, bound := bindIntentArgs(intent, availableParams{}); bound { 108 t.Errorf("a segment intent must drop when its agency and naics are not available") 109 } 110 111 // Agency only, naics missing: still drops (every required param must be available). 112 if _, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true}); bound { 113 t.Errorf("a segment intent must drop when only the agency is available") 114 } 115 116 // Both present plus the clamp zero lookback: binds with three args (agency, naics, lookback clamp). 117 args, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true}) 118 if !bound { 119 t.Fatalf("a segment intent must bind when its agency and naics are both available") 120 } 121 if len(args) != 3 || args[0] != "X" || args[1] != "541512" || args[2] != 0 { 122 t.Errorf("the segment intent must bind agency, naics, and a clamp zero lookback, got %v", args) 123 } 124} 125 126// TestValidatePlanDropsOutOfSetAndUnbindable proves the validator drops a name that is not in the Registry 127// (the closed enum check) and an intent whose params cannot bind, keeping only the bound nodes in the 128// deterministic order. 129func TestValidatePlanDropsOutOfSetAndUnbindable(t *testing.T) { 130 // The bag binds an agency and a naics (so a market segment intent binds) and a uuid (so a requirement keyed 131 // intent binds), but NOT a held clearance, so clearance_reachability is unbindable and must drop. 132 avail := availableParams{ 133 Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true, 134 UUID: "22222222-2222-2222-2222-222222222222", UUIDOK: true, 135 } 136 names := []string{ 137 "not_a_real_intent", // out of set: dropped by the closed enum check 138 IntentClearanceReachability, // in set but unbindable here (no held clearance): dropped 139 IntentSegmentCompetitorField, // binds (agency, naics, clamp) 140 IntentProposalReadyName(), // binds (requirement uuid) 141 } 142 plan := validatePlan(names, avail) 143 144 got := make([]string, len(plan.Nodes)) 145 for i, n := range plan.Nodes { 146 got[i] = n.Intent 147 } 148 want := []string{IntentProposalReadyName(), IntentSegmentCompetitorField} // sorted by name 149 if !reflect.DeepEqual(got, want) { 150 t.Errorf("validatePlan must keep only the bound, in set intents in name order, got %v want %v", got, want) 151 } 152 for _, n := range plan.Nodes { 153 if n.Intent == "not_a_real_intent" { 154 t.Error("an out of set name must never reach the validated plan") 155 } 156 if n.Intent == IntentClearanceReachability { 157 t.Error("an unbindable intent must never reach the validated plan") 158 } 159 } 160} 161 162// IntentProposalReadyName returns the registered name of the proposal ready TAG intent, a requirement keyed 163// intent, used by the validator test to prove a bound requirement intent survives. It is a test helper, not a 164// production symbol. 165func IntentProposalReadyName() string { return "proposal_ready_for_requirement" } 166 167// TestValidatePlanCapsNodeCount proves the validator never returns more than the hard node cap even if the 168// proposer names more bindable intents than the cap. It feeds every requirement keyed intent (all bind from a 169// uuid) plus market intents and asserts the result is capped at maxPlanNodes. 170func TestValidatePlanCapsNodeCount(t *testing.T) { 171 avail := representativeAvail() 172 var allNames []string 173 for _, it := range Registry { 174 allNames = append(allNames, it.Name) 175 } 176 plan := validatePlan(allNames, avail) 177 if len(plan.Nodes) > maxPlanNodes { 178 t.Errorf("the validated plan must never exceed the hard node cap %d, got %d", maxPlanNodes, len(plan.Nodes)) 179 } 180} 181 182// TestProposeStablePlanMajority drives self consistency with a STUBBED proposer (no live plane), proving that 183// only the intents that appear in a MAJORITY of the k validated plans survive, and that total disagreement 184// yields an empty plan so the caller falls back. 185func TestProposeStablePlanMajority(t *testing.T) { 186 avail := representativeAvail() 187 188 // Three samples. The segment competitor field appears in all three (a clear majority); the recompete clock 189 // appears in two of three (a majority of three is two); the agency spend trend appears in only one (not a 190 // majority). The stable set is the first two, in name order. 191 samples := [][]string{ 192 {IntentSegmentCompetitorField, IntentRecompeteClock}, 193 {IntentSegmentCompetitorField, IntentRecompeteClock, IntentAgencySpendTrend}, 194 {IntentSegmentCompetitorField}, 195 } 196 call := 0 197 stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 198 s := samples[call%len(samples)] 199 call++ 200 return s, nil 201 } 202 plan := proposeStablePlan(context.Background(), "q", avail, 3, stub) 203 204 got := make([]string, len(plan.Nodes)) 205 for i, n := range plan.Nodes { 206 got[i] = n.Intent 207 } 208 want := []string{IntentRecompeteClock, IntentSegmentCompetitorField} // sorted by name, both at or above the majority of two 209 if !reflect.DeepEqual(got, want) { 210 t.Errorf("only the majority intents must survive self consistency, got %v want %v", got, want) 211 } 212 213 // Total disagreement: three samples that share no intent yields no majority, so the plan is empty and the 214 // caller falls back to the deterministic single intent or RAG path. 215 disagree := [][]string{{IntentSegmentCompetitorField}, {IntentRecompeteClock}, {IntentAgencySpendTrend}} 216 dcall := 0 217 dstub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 218 s := disagree[dcall%len(disagree)] 219 dcall++ 220 return s, nil 221 } 222 empty := proposeStablePlan(context.Background(), "q", avail, 3, dstub) 223 if len(empty.Nodes) != 0 { 224 t.Errorf("total disagreement must yield an empty plan, got %v", empty.Nodes) 225 } 226} 227 228// TestProposeStablePlanDropsOutOfSetSamples proves the closed enum holds through self consistency: a proposer 229// that returns an out of set name in every sample never produces a node (the validator drops it each time), so 230// the stable plan is empty even though the name had a clear majority. 231func TestProposeStablePlanDropsOutOfSetSamples(t *testing.T) { 232 stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { 233 return []string{"injected_intent", "drop_table_users"}, nil 234 } 235 plan := proposeStablePlan(context.Background(), "q", representativeAvail(), 3, stub) 236 if len(plan.Nodes) != 0 { 237 t.Errorf("an out of set name must never survive self consistency, got %v", plan.Nodes) 238 } 239} 240 241// TestIsCompoundQuestion pins the conservative compound signal: a question that names BOTH a bench plane word 242// and a market plane word is compound; a single plane question (only bench, or only market) is not, so it 243// falls through to the existing single shot route and is never regressed. 244func TestIsCompoundQuestion(t *testing.T) { 245 compound := []string{ 246 `who on our bench can staff this and who is the incumbent in the market`, 247 `our bench gap versus the recompete pipeline for the agency`, 248 `bench coverage and the competitor field for naics 541512`, 249 } 250 for _, q := range compound { 251 if !isCompoundQuestion(q) { 252 t.Errorf("a question naming a bench plane word AND a market plane word must be compound, got false for %q", q) 253 } 254 } 255 256 single := []string{ 257 `how many candidates hold a secret clearance`, // no plane word pair 258 `who is the incumbent for the agency`, // market only 259 `what is our bench clearance coverage`, // bench only 260 `how many requirements are open`, // neither 261 } 262 for _, q := range single { 263 if isCompoundQuestion(q) { 264 t.Errorf("a single plane or no plane question must not be compound, got true for %q", q) 265 } 266 } 267} 268 269// TestParseIntentNames proves the JSON array parser accepts the instructed bare array, tolerates surrounding 270// prose by extracting the first bracketed array, and returns nil on unparseable output so the caller falls 271// back to RAG. 272func TestParseIntentNames(t *testing.T) { 273 if got := parseIntentNames(`["segment_competitor_field","recompete_clock"]`); !reflect.DeepEqual(got, []string{"segment_competitor_field", "recompete_clock"}) { 274 t.Errorf("a bare JSON array must parse, got %v", got) 275 } 276 if got := parseIntentNames("Here is the plan: [\"recompete_clock\"] done."); !reflect.DeepEqual(got, []string{"recompete_clock"}) { 277 t.Errorf("a bracketed array inside prose must parse, got %v", got) 278 } 279 if got := parseIntentNames("no array here"); got != nil { 280 t.Errorf("unparseable output must return nil, got %v", got) 281 } 282 if got := parseIntentNames("[not valid json}"); got != nil { 283 t.Errorf("malformed json must return nil, got %v", got) 284 } 285} 286 287// TestAvailableKindsHoldsNoValue proves the audit and prompt available kinds carry only KIND names, never a 288// value, so a requirement id, an agency name, or a UEI can never leak through the kinds list. 289func TestAvailableKindsHoldsNoValue(t *testing.T) { 290 avail := representativeAvail() 291 kinds := availableKinds(avail) 292 want := []string{"agency", "area", "compartments", "held_clearance", "naics", "occupation_code", "uei", "uuid", "window_months"} 293 sort.Strings(want) 294 if !reflect.DeepEqual(kinds, want) { 295 t.Errorf("available kinds must be the sorted kind names, got %v want %v", kinds, want) 296 } 297 for _, k := range kinds { 298 for _, value := range []string{avail.Agency, avail.NAICS, avail.UUID, avail.UEI, avail.Compartments, avail.OccupationCode, avail.Area} { 299 if k == value { 300 t.Errorf("a kind name must never equal a param value, leaked %q", value) 301 } 302 } 303 } 304} 305