package ai import ( "context" "reflect" "sort" "testing" ) // representativeAvail is a full available bag: every extractor fired, so any Registry intent whose params are // all mapped should bind from it. The unit tests use it to prove that every selectable intent binds and that // the unmapped or unavailable cases drop. func representativeAvail() availableParams { return availableParams{ Agency: "DEPT OF THE AIR FORCE", AgencyOK: true, NAICS: "541512", NAICSOK: true, UUID: "11111111-1111-1111-1111-111111111111", UUIDOK: true, HeldClearance: "secret", HeldOK: true, UEI: "ABC123DEF456", UEIOK: true, WindowMonths: 24, Compartments: "SI, TK", CompartmentsOK: true, OccupationCode: "113021", OccupationOK: true, Area: "HI", AreaOK: true, } } // TestExtractAvailableParams proves the bag is built deterministically from the question text alone, through // the existing extractors, and that an absent token leaves its slot unset rather than guessed. func TestExtractAvailableParams(t *testing.T) { // A compound question that carries a quoted agency, a bare naics, a canonical uuid, a held clearance phrase, // a quoted uei, and an explicit window in months. q := `For the "DEPT OF THE AIR FORCE" in naics 541512, who on our bench can staff requirement ` + `11111111-1111-1111-1111-111111111111 with a secret clearance and partner "ABC123DEF456" over a 24 month window` a := extractAvailableParams(q) if !a.AgencyOK || a.Agency != "DEPT OF THE AIR FORCE" { t.Errorf("agency must bind from the quoted span, got %q ok=%v", a.Agency, a.AgencyOK) } if !a.NAICSOK || a.NAICS != "541512" { t.Errorf("naics must bind from the bare six digit token, got %q ok=%v", a.NAICS, a.NAICSOK) } if !a.UUIDOK || a.UUID != "11111111-1111-1111-1111-111111111111" { t.Errorf("uuid must bind from the canonical id, got %q ok=%v", a.UUID, a.UUIDOK) } if !a.HeldOK || a.HeldClearance != "secret" { t.Errorf("held clearance must bind from the named phrase, got %q ok=%v", a.HeldClearance, a.HeldOK) } if !a.UEIOK || a.UEI != "ABC123DEF456" { t.Errorf("uei must bind from the quoted twelve char token, got %q ok=%v", a.UEI, a.UEIOK) } if a.WindowMonths != 24 { t.Errorf("window months must read the stated window, got %d", a.WindowMonths) } // A bare question carrying nothing extractable leaves every slot unset; the window defaults to zero so the // intent applies its own clamp. empty := extractAvailableParams("how many candidates are proposal ready") if empty.AgencyOK || empty.NAICSOK || empty.UUIDOK || empty.HeldOK || empty.UEIOK || empty.CompartmentsOK { t.Errorf("an unextractable question must leave every slot unset, got %+v", empty) } if empty.WindowMonths != 0 { t.Errorf("an unstated window must default to zero, got %d", empty.WindowMonths) } } // TestBindIntentArgsEveryRegistryIntent walks EVERY intent in the closed Registry and asserts bindIntentArgs // either binds it from the representative bag (when every param name is mapped) or correctly drops it (when a // param name is unmapped). It also pins that the bound arg count matches the param count and that an intent // with no params binds an empty arg list. This is the load bearing safety proof: an intent that binds gets a // value for every required param, and no intent binds a param the model could have invented. func TestBindIntentArgsEveryRegistryIntent(t *testing.T) { avail := representativeAvail() for _, intent := range Registry { args, ok := bindIntentArgs(intent, avail) // Every param name a registered intent declares is mapped in paramBindings, so against the full // representative bag every intent must bind. If a future intent adds an unmapped param name, this fails // loudly, which is the intended tripwire (that intent is then simply not selectable until it is mapped). if !ok { t.Errorf("intent %q has an unmapped or unavailable param against the full bag; map it or it is never selectable", intent.Name) continue } if len(args) != len(intent.Params) { t.Errorf("intent %q bound %d args for %d params", intent.Name, len(args), len(intent.Params)) } if len(intent.Params) == 0 && args != nil { t.Errorf("intent %q with no params must bind a nil arg list, got %v", intent.Name, args) } } } // TestBindIntentArgsDropsOnMissingParam proves a node drops when a required param is not available: a market // intent that needs an agency and a naics drops when the bag carries neither, and binds when both are present. func TestBindIntentArgsDropsOnMissingParam(t *testing.T) { intent, ok := IntentByName(IntentSegmentCompetitorField) if !ok { t.Fatalf("the segment competitor field intent must be registered") } // No agency and no naics: the node drops. if _, bound := bindIntentArgs(intent, availableParams{}); bound { t.Errorf("a segment intent must drop when its agency and naics are not available") } // Agency only, naics missing: still drops (every required param must be available). if _, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true}); bound { t.Errorf("a segment intent must drop when only the agency is available") } // Both present plus the clamp zero lookback: binds with three args (agency, naics, lookback clamp). args, bound := bindIntentArgs(intent, availableParams{Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true}) if !bound { t.Fatalf("a segment intent must bind when its agency and naics are both available") } if len(args) != 3 || args[0] != "X" || args[1] != "541512" || args[2] != 0 { t.Errorf("the segment intent must bind agency, naics, and a clamp zero lookback, got %v", args) } } // TestValidatePlanDropsOutOfSetAndUnbindable proves the validator drops a name that is not in the Registry // (the closed enum check) and an intent whose params cannot bind, keeping only the bound nodes in the // deterministic order. func TestValidatePlanDropsOutOfSetAndUnbindable(t *testing.T) { // The bag binds an agency and a naics (so a market segment intent binds) and a uuid (so a requirement keyed // intent binds), but NOT a held clearance, so clearance_reachability is unbindable and must drop. avail := availableParams{ Agency: "X", AgencyOK: true, NAICS: "541512", NAICSOK: true, UUID: "22222222-2222-2222-2222-222222222222", UUIDOK: true, } names := []string{ "not_a_real_intent", // out of set: dropped by the closed enum check IntentClearanceReachability, // in set but unbindable here (no held clearance): dropped IntentSegmentCompetitorField, // binds (agency, naics, clamp) IntentProposalReadyName(), // binds (requirement uuid) } plan := validatePlan(names, avail) got := make([]string, len(plan.Nodes)) for i, n := range plan.Nodes { got[i] = n.Intent } want := []string{IntentProposalReadyName(), IntentSegmentCompetitorField} // sorted by name if !reflect.DeepEqual(got, want) { t.Errorf("validatePlan must keep only the bound, in set intents in name order, got %v want %v", got, want) } for _, n := range plan.Nodes { if n.Intent == "not_a_real_intent" { t.Error("an out of set name must never reach the validated plan") } if n.Intent == IntentClearanceReachability { t.Error("an unbindable intent must never reach the validated plan") } } } // IntentProposalReadyName returns the registered name of the proposal ready TAG intent, a requirement keyed // intent, used by the validator test to prove a bound requirement intent survives. It is a test helper, not a // production symbol. func IntentProposalReadyName() string { return "proposal_ready_for_requirement" } // TestValidatePlanCapsNodeCount proves the validator never returns more than the hard node cap even if the // proposer names more bindable intents than the cap. It feeds every requirement keyed intent (all bind from a // uuid) plus market intents and asserts the result is capped at maxPlanNodes. func TestValidatePlanCapsNodeCount(t *testing.T) { avail := representativeAvail() var allNames []string for _, it := range Registry { allNames = append(allNames, it.Name) } plan := validatePlan(allNames, avail) if len(plan.Nodes) > maxPlanNodes { t.Errorf("the validated plan must never exceed the hard node cap %d, got %d", maxPlanNodes, len(plan.Nodes)) } } // TestProposeStablePlanMajority drives self consistency with a STUBBED proposer (no live plane), proving that // only the intents that appear in a MAJORITY of the k validated plans survive, and that total disagreement // yields an empty plan so the caller falls back. func TestProposeStablePlanMajority(t *testing.T) { avail := representativeAvail() // Three samples. The segment competitor field appears in all three (a clear majority); the recompete clock // appears in two of three (a majority of three is two); the agency spend trend appears in only one (not a // majority). The stable set is the first two, in name order. samples := [][]string{ {IntentSegmentCompetitorField, IntentRecompeteClock}, {IntentSegmentCompetitorField, IntentRecompeteClock, IntentAgencySpendTrend}, {IntentSegmentCompetitorField}, } call := 0 stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { s := samples[call%len(samples)] call++ return s, nil } plan := proposeStablePlan(context.Background(), "q", avail, 3, stub) got := make([]string, len(plan.Nodes)) for i, n := range plan.Nodes { got[i] = n.Intent } want := []string{IntentRecompeteClock, IntentSegmentCompetitorField} // sorted by name, both at or above the majority of two if !reflect.DeepEqual(got, want) { t.Errorf("only the majority intents must survive self consistency, got %v want %v", got, want) } // Total disagreement: three samples that share no intent yields no majority, so the plan is empty and the // caller falls back to the deterministic single intent or RAG path. disagree := [][]string{{IntentSegmentCompetitorField}, {IntentRecompeteClock}, {IntentAgencySpendTrend}} dcall := 0 dstub := func(ctx context.Context, q string, a availableParams) ([]string, error) { s := disagree[dcall%len(disagree)] dcall++ return s, nil } empty := proposeStablePlan(context.Background(), "q", avail, 3, dstub) if len(empty.Nodes) != 0 { t.Errorf("total disagreement must yield an empty plan, got %v", empty.Nodes) } } // TestProposeStablePlanDropsOutOfSetSamples proves the closed enum holds through self consistency: a proposer // that returns an out of set name in every sample never produces a node (the validator drops it each time), so // the stable plan is empty even though the name had a clear majority. func TestProposeStablePlanDropsOutOfSetSamples(t *testing.T) { stub := func(ctx context.Context, q string, a availableParams) ([]string, error) { return []string{"injected_intent", "drop_table_users"}, nil } plan := proposeStablePlan(context.Background(), "q", representativeAvail(), 3, stub) if len(plan.Nodes) != 0 { t.Errorf("an out of set name must never survive self consistency, got %v", plan.Nodes) } } // TestIsCompoundQuestion pins the broadened compound signal: a question that spans TWO OR MORE distinct // planes (bench, market, clearance, wage, skill, lifecycle) is compound and routes to the planner; a single // plane question (however many words it names within that one plane) is not, so it falls through to the // existing single shot route and is never regressed. func TestIsCompoundQuestion(t *testing.T) { compound := []string{ `who on our bench can staff this and who is the incumbent in the market`, // bench + market `our bench gap versus the recompete pipeline for the agency`, // bench + market `bench coverage and the competitor field for naics 541512`, // bench + market `what is the labor rate and the ts/sci coverage`, // wage + clearance `the incumbent and the prevailing wage for this agency`, // market + wage `our bench and their ts/sci reciprocity`, // bench + clearance `adjacent skill coverage and the competitor field`, // skill + market } for _, q := range compound { if !isCompoundQuestion(q) { t.Errorf("a question spanning two or more distinct planes must be compound, got false for %q", q) } } single := []string{ `how many candidates hold a secret clearance`, // no plane word pair `who is the incumbent for the agency`, // market only `what is our bench clearance coverage`, // bench only `how many requirements are open`, // neither `what is the prevailing wage for a data scientist`, // wage only } for _, q := range single { if isCompoundQuestion(q) { t.Errorf("a single plane or no plane question must not be compound, got true for %q", q) } } } // TestParseIntentNames proves the JSON array parser accepts the instructed bare array, tolerates surrounding // prose by extracting the first bracketed array, and returns nil on unparseable output so the caller falls // back to RAG. func TestParseIntentNames(t *testing.T) { if got := parseIntentNames(`["segment_competitor_field","recompete_clock"]`); !reflect.DeepEqual(got, []string{"segment_competitor_field", "recompete_clock"}) { t.Errorf("a bare JSON array must parse, got %v", got) } if got := parseIntentNames("Here is the plan: [\"recompete_clock\"] done."); !reflect.DeepEqual(got, []string{"recompete_clock"}) { t.Errorf("a bracketed array inside prose must parse, got %v", got) } if got := parseIntentNames("no array here"); got != nil { t.Errorf("unparseable output must return nil, got %v", got) } if got := parseIntentNames("[not valid json}"); got != nil { t.Errorf("malformed json must return nil, got %v", got) } } // TestAvailableKindsHoldsNoValue proves the audit and prompt available kinds carry only KIND names, never a // value, so a requirement id, an agency name, or a UEI can never leak through the kinds list. func TestAvailableKindsHoldsNoValue(t *testing.T) { avail := representativeAvail() kinds := availableKinds(avail) want := []string{"agency", "area", "compartments", "held_clearance", "naics", "occupation_code", "uei", "uuid", "window_months"} sort.Strings(want) if !reflect.DeepEqual(kinds, want) { t.Errorf("available kinds must be the sorted kind names, got %v want %v", kinds, want) } for _, k := range kinds { for _, value := range []string{avail.Agency, avail.NAICS, avail.UUID, avail.UEI, avail.Compartments, avail.OccupationCode, avail.Area} { if k == value { t.Errorf("a kind name must never equal a param value, leaked %q", value) } } } } // TestEdgeCatalogIsSafe is the PII firewall assertion for dependency edges: every edge SOURCE is a Registry // intent and is NOT a bench (person touching) intent, so no candidate or demographic value can ever be an edge // value, and every edge target actually declares the param the edge binds. This asserts the safety of the // whole chaining feature at the catalog, so a future edge that breaks the firewall fails the build. func TestEdgeCatalogIsSafe(t *testing.T) { for _, e := range edgeCatalog { if _, ok := IntentByName(e.FromIntent); !ok { t.Errorf("edge source %q is not a Registry intent", e.FromIntent) } if IsBenchIntent(e.FromIntent) { t.Errorf("edge source %q is a bench intent: a candidate value could flow through the chain, forbidden", e.FromIntent) } to, ok := IntentByName(e.ToIntent) if !ok { t.Errorf("edge target %q is not a Registry intent", e.ToIntent) continue } declares := false for _, p := range to.Params { if p.Name == e.ToParam { declares = true } } if !declares { t.Errorf("edge target %q does not declare the param %q the edge binds", e.ToIntent, e.ToParam) } } } // TestEdgeAwareBindingChainsUEI proves the planner CHAINS: partner_or_competitor_ratio, whose partner_uei the // question does not quote, binds when its edge source (incumbent_of_record) is also selected and the naics is // present; partner_uei is marked edge bound (a placeholder resolved at run time) and naics binds from the // question. The model supplied no value and no edge. func TestEdgeAwareBindingChainsUEI(t *testing.T) { avail := availableParams{NAICS: "541512", NAICSOK: true} // naics present, NO quoted uei selected := map[string]bool{IntentIncumbentOfRecord: true, IntentPartnerOrCompetitorRatio: true} to, _ := IntentByName(IntentPartnerOrCompetitorRatio) args, edge, idx, ok := bindIntentArgsWithEdges(to, avail, selected) if !ok { t.Fatal("partner_or_competitor_ratio must bind when its edge source is selected and naics is present") } if edge == nil || edge.FromIntent != IntentIncumbentOfRecord { t.Fatalf("expected an edge from incumbent_of_record, got %v", edge) } if idx != 0 || args[0] != "" { t.Errorf("expected partner_uei (idx 0) to be an edge bound placeholder, got idx %d args %v", idx, args) } if args[1] != "541512" { t.Errorf("expected naics to bind from the question, got %v", args[1]) } } // TestEdgeDropsWhenSourceNotSelected proves the target does NOT bind when its edge source is absent, so the // planner never fabricates a partner_uei out of nothing. func TestEdgeDropsWhenSourceNotSelected(t *testing.T) { avail := availableParams{NAICS: "541512", NAICSOK: true} // naics only, no quoted uei selected := map[string]bool{IntentPartnerOrCompetitorRatio: true} // the edge source is NOT selected to, _ := IntentByName(IntentPartnerOrCompetitorRatio) if _, _, _, ok := bindIntentArgsWithEdges(to, avail, selected); ok { t.Error("partner_or_competitor_ratio must NOT bind without a quoted uei and without its edge source selected") } } // TestValidatePlanTopoOrdersSourceBeforeTarget proves a plan naming both the source and the target orders the // source (edge free) BEFORE the target (edge bound), so the runner has the upstream result when the target runs. func TestValidatePlanTopoOrdersSourceBeforeTarget(t *testing.T) { avail := availableParams{Agency: "NSA", AgencyOK: true, NAICS: "541512", NAICSOK: true} plan := validatePlan([]string{IntentPartnerOrCompetitorRatio, IntentSegmentCompetitorField}, avail) if len(plan.Nodes) != 2 { t.Fatalf("expected 2 nodes (source and target), got %d: %+v", len(plan.Nodes), plan.Nodes) } if plan.Nodes[0].Intent != IntentSegmentCompetitorField { t.Errorf("expected the edge source (segment_competitor_field) first, got %q", plan.Nodes[0].Intent) } if plan.Nodes[1].Intent != IntentPartnerOrCompetitorRatio || plan.Nodes[1].EdgeFrom != IntentSegmentCompetitorField { t.Errorf("expected the edge target second with EdgeFrom set, got %q EdgeFrom %q", plan.Nodes[1].Intent, plan.Nodes[1].EdgeFrom) } } // TestFirstRowColumn proves the edge value reader returns the first row's named string column and drops (ok // false) on an empty result, a missing column, or a non string cell, so a dependent node never binds a wrong // or empty value. func TestFirstRowColumn(t *testing.T) { r := capturedResult{Columns: []string{"incumbent_uei", "amount"}, Rows: [][]any{{"ABC123UEI", 500}}} if v, ok := firstRowColumn(r, "incumbent_uei"); !ok || v != "ABC123UEI" { t.Errorf("expected ABC123UEI, got %q ok %v", v, ok) } if _, ok := firstRowColumn(r, "missing"); ok { t.Error("a missing column must return ok false") } if _, ok := firstRowColumn(capturedResult{Columns: []string{"incumbent_uei"}, Rows: nil}, "incumbent_uei"); ok { t.Error("an empty result must return ok false") } if _, ok := firstRowColumn(capturedResult{Columns: []string{"n"}, Rows: [][]any{{42}}}, "n"); ok { t.Error("a non string cell must return ok false: never bind a non text value to a text param") } }