package ai import ( "context" "strings" "testing" ) // TestIsGraphIntent pins the closed set of graph intents: the two G3 skill traversals and the four G1 market // traversals are graph intents (dispatched to AnswerGraph), a TAG intent is not, and an unknown or empty name // is not. A future intent added to the graph set without being a real registered traversal, or a TAG intent // mislabeled graph, fails here. func TestIsGraphIntent(t *testing.T) { graph := []string{ IntentSkillSubstitutionClosure, IntentSkillGapBridge, IntentSegmentCompetitorField, IntentIncumbentOfRecord, IntentRecompetePipeline, IntentAgencyBuyingPattern, } for _, name := range graph { if !IsGraphIntent(name) { t.Errorf("%q must be a graph intent", name) } // Every graph intent must also be a real registered intent in the closed Registry. if _, ok := IntentByName(name); !ok { t.Errorf("graph intent %q must be registered in the closed Registry", name) } } notGraph := []string{ "count_candidates_by_clearance", // a TAG intent "proposal_ready_for_requirement", "open_requirements_count", "slate_decision_mix", "", // empty "definitely_not_an_intent", } for _, name := range notGraph { if IsGraphIntent(name) { t.Errorf("%q must NOT be a graph intent", name) } } } // TestGraphClassifyRoutesCaptureQuestions proves the conservative natural language router maps a free form // capture or skill question that clearly names a graph traversal AND carries the params to bind it // unambiguously to the right intent with the right args, the best effort NL path. It never authors a param the // user did not delimit: the agency must be quoted and the NAICS a bare six digit token. func TestGraphClassifyRoutesCaptureQuestions(t *testing.T) { cases := []struct { name string question string intent string args []any }{ { name: "competitor field with quoted agency and bare naics", question: `who are the competitors in "DEPT OF THE AIR FORCE" for 541512`, intent: IntentSegmentCompetitorField, args: []any{"DEPT OF THE AIR FORCE", "541512", 0}, }, { name: "incumbent question is a competitor segment question", question: `who is the incumbent on "DEPT OF DEFENSE" 541330 work`, intent: IntentSegmentCompetitorField, args: []any{"DEPT OF DEFENSE", "541330", 0}, }, { name: "recompete pipeline with quoted agency and explicit window", question: `show me the recompete pipeline for "DEPT OF THE NAVY" in the next 18 months`, intent: IntentRecompetePipeline, args: []any{"DEPT OF THE NAVY", 18}, }, { name: "recompete pipeline with quoted agency and default window", question: `what is coming up for recompete at "DEPT OF THE ARMY"`, intent: IntentRecompetePipeline, args: []any{"DEPT OF THE ARMY", 0}, }, { name: "agency buying pattern with quoted agency", question: `what does the agency buy, the buying pattern for "GSA"`, intent: IntentAgencyBuyingPattern, args: []any{"GSA", ""}, }, { name: "agency buying pattern with quoted agency and optional naics", question: `where should we position on the buying pattern for "GSA" 541519`, intent: IntentAgencyBuyingPattern, args: []any{"GSA", "541519"}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { gr := graphClassify(tc.question) if gr == nil { t.Fatalf("graphClassify(%q) must route to a graph intent", tc.question) } if gr.intent != tc.intent { t.Errorf("intent: got %q, want %q", gr.intent, tc.intent) } if len(gr.args) != len(tc.args) { t.Fatalf("args length: got %d %v, want %d %v", len(gr.args), gr.args, len(tc.args), tc.args) } for i := range tc.args { if gr.args[i] != tc.args[i] { t.Errorf("arg %d: got %v, want %v", i, gr.args[i], tc.args[i]) } } }) } } // TestGraphClassifyRoutesLifecycleQuestions proves the conservative NL router maps a free form G9 lifecycle // question that clearly names a traversal AND carries the params to bind it unambiguously to the right intent // with the right args. The succession needs an explicit opportunity UUID, the recompete clock needs a quoted // agency, and the forecast runway needs an explicit number of months; it never authors a param the user did // not delimit. func TestGraphClassifyRoutesLifecycleQuestions(t *testing.T) { oppID := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" cases := []struct { name string question string intent string args []any }{ { name: "procurement succession with an explicit opportunity uuid", question: `is this a recompete, walk the succession for opportunity ` + oppID, intent: IntentProcurementSuccession, args: []any{oppID}, }, { name: "lifecycle question with an explicit opportunity uuid", question: `show me the lifecycle of ` + oppID, intent: IntentProcurementSuccession, args: []any{oppID}, }, { name: "recompete clock with quoted agency and explicit window", question: `show me the recompete clock for "DEPT OF THE NAVY" over the next 18 months`, intent: IntentRecompeteClock, args: []any{"DEPT OF THE NAVY", 18}, }, { name: "recompete clock with quoted agency and default window", question: `what is the recompete clock for "DEPT OF THE ARMY"`, intent: IntentRecompeteClock, args: []any{"DEPT OF THE ARMY", 0}, }, { name: "forecast runway with an explicit window", question: `show me the forecast runway for the next 12 months`, intent: IntentForecastRunway, args: []any{12}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { gr := graphClassify(tc.question) if gr == nil { t.Fatalf("graphClassify(%q) must route to a graph intent", tc.question) } if gr.intent != tc.intent { t.Errorf("intent: got %q, want %q", gr.intent, tc.intent) } if len(gr.args) != len(tc.args) { t.Fatalf("args length: got %d %v, want %d %v", len(gr.args), gr.args, len(tc.args), tc.args) } for i := range tc.args { if gr.args[i] != tc.args[i] { t.Errorf("arg %d: got %v, want %v", i, gr.args[i], tc.args[i]) } } }) } } // TestGraphClassifyLifecycleFallsBackWhenParamsMissing is the conservatism test for the G9 routes: a lifecycle // question that clearly names a traversal but does NOT carry the params to bind it returns nil, so the caller // falls back to RAG rather than guessing an opportunity, an agency, or a window. The model never authors // params; an unbindable lifecycle question is a RAG question. func TestGraphClassifyLifecycleFallsBackWhenParamsMissing(t *testing.T) { fallback := []string{ // A succession or lifecycle question with no explicit opportunity UUID: nothing to bind. "is this a recompete or a new start", "walk the succession of this solicitation", "show me the lifecycle of the cloud contract", // A recompete clock question with no quoted agency: the agency is ambiguous. "show me the recompete clock for the next year", "what is the recompete window in general", // A forecast runway question with no explicit window: the horizon is ambiguous. "show me the forecast runway", "what is on the forecast pipeline", } for _, q := range fallback { if gr := graphClassify(q); gr != nil { t.Errorf("graphClassify(%q) must fall back (nil) when params cannot be bound, got intent %q args %v", q, gr.intent, gr.args) } } } // TestGraphClassifyBareRecompeteStillRoutesToPipeline pins the byte unchanged existing route: a bare recompete // question with no clock phrase still routes to the existing G1 recompete pipeline intent (not the new G9 // clock), so the Phase 0 route is untouched. The distinct recompete clock phrase is what selects the new // intent. func TestGraphClassifyBareRecompeteStillRoutesToPipeline(t *testing.T) { gr := graphClassify(`what is coming up for recompete at "DEPT OF THE ARMY"`) if gr == nil { t.Fatal("a bare recompete question with a quoted agency must still route") } if gr.intent != IntentRecompetePipeline { t.Errorf("a bare recompete question must route to the existing pipeline, got %q", gr.intent) } } // TestGraphClassifyFallsBackWhenParamsMissing is the load bearing conservatism test: a question that clearly // names a graph traversal but does NOT carry the params to bind it unambiguously returns nil, so the caller // falls back to the existing RAG path rather than guessing a segment, an agency, or a skill. The model never // authors params; an unbindable graph question is a RAG question. func TestGraphClassifyFallsBackWhenParamsMissing(t *testing.T) { fallback := []string{ // A competitor question with no quoted agency and no NAICS: nothing to bind. "who are the competitors in the cloud market", // A competitor question with a NAICS but no quoted agency: the segment key is incomplete. "who are the competitors for 541512", // A competitor question with a quoted agency but no NAICS: still incomplete. `who are the competitors in "DEPT OF THE AIR FORCE"`, // A recompete question with no quoted agency: the agency is ambiguous. "what is coming up for recompete next year", // A buying pattern question with no quoted agency. "what does the agency buy in general", // A skill adjacency question with no explicit skill id (a user never types a UUID): unbindable. "what substitutes for kubernetes", "show me the skill adjacency for cloud security", // A plain question that names no graph traversal at all. "who are the best cloud security candidates", "how many candidates hold a secret clearance", "what is the clearance hierarchy", } for _, q := range fallback { if gr := graphClassify(q); gr != nil { t.Errorf("graphClassify(%q) must fall back (nil) when params cannot be bound, got intent %q args %v", q, gr.intent, gr.args) } } } // TestGraphClassifySkillWithExplicitID proves the only unambiguous skill adjacency binding: a question that // carries a canonical UUID skill id routes to the substitution closure with that id bound and the default // depth (a bound zero, so the intent clamps it). Absent a UUID the prior test proves the fallback. func TestGraphClassifySkillWithExplicitID(t *testing.T) { id := "11111111-2222-3333-4444-555555555555" gr := graphClassify(`what substitutes for skill ` + id) if gr == nil { t.Fatalf("a skill question carrying an explicit UUID must route to the substitution closure") } if gr.intent != IntentSkillSubstitutionClosure { t.Errorf("intent: got %q, want %q", gr.intent, IntentSkillSubstitutionClosure) } if len(gr.args) != 2 || gr.args[0] != id || gr.args[1] != 0 { t.Errorf("args: got %v, want [%q 0]", gr.args, id) } } // TestClassifyUnchanged is the regression guard for default path invariance: the existing Classify cases for // Refuse, TAG, RAG, and Direct route exactly as before. Classify is untouched by the graph layer (the graph // NL routing lives in graphClassify, run BEFORE Classify in Answer), so a non graph question classifies byte // for byte as it did, and an EEO laced question still refuses. func TestClassifyUnchanged(t *testing.T) { cases := map[string]Route{ // Refuse: an eeo schema name or a protected attribute in plain words. "list candidates by race and gender": RouteRefuse, "show me candidates grouped by veteran status": RouteRefuse, "select from eeo.candidate_demographics": RouteRefuse, // TAG: an aggregation question. "how many candidates hold a secret clearance": RouteTAG, "what is the total number of open requirements": RouteTAG, "give me the breakdown of the slate decisions": RouteTAG, // RAG: a retrieval question. "who are the best cloud security candidates": RouteRAG, "find me a candidate with a TS SCI clearance": RouteRAG, "show me the top engineers": RouteRAG, // Direct: a stable rules question with no aggregation or retrieval keyword. "what is the clearance hierarchy": RouteDirect, "explain the scoring rules": RouteDirect, } for q, want := range cases { if got := Classify(q); got != want { t.Errorf("Classify(%q): got %q, want %q", q, got, want) } } } // TestAnswerDispatchesGraphIntentToAnswerGraph proves a named GRAPH intent dispatches to the structured graph // path: with a nil client and a nil db the dispatch reaches AnswerGraph (which runs ExecuteIntent), so the // resulting error is the no database error from ExecuteIntent, NOT the unknown intent error AnswerTAG would // return for a graph name it does not own. This pins the dispatch fork without needing a database. func TestAnswerDispatchesGraphIntentToAnswerGraph(t *testing.T) { o := NewOrchestrator(nil, nil, nil) _, err := o.Answer(context.Background(), "", IntentSegmentCompetitorField, []any{"DEPT OF THE AIR FORCE", "541512", 0}) if err == nil { t.Fatal("a graph intent with a nil db must error from ExecuteIntent, not silently succeed") } // ExecuteIntent returns "ai: no database" when db is nil; AnswerTAG would have returned ErrUnknownIntent // for a name it does not recognize, which it would not since the name is registered, but the point is the // graph fork ran AnswerGraph. The no database error confirms ExecuteIntent was reached through AnswerGraph. if !strings.Contains(err.Error(), "no database") { t.Errorf("expected the no database error from ExecuteIntent via AnswerGraph, got %v", err) } } // TestAnswerGraphRefusesEEO proves the structured graph path runs the EEO and protected attribute guards // FIRST and refuses on either, the orchestrator precedent. The refusal carries route graph, is marked // Refused, and ships no answer, so the graph path can never group, filter, or rank by a protected attribute, // even when a graph intent is named. func TestAnswerGraphRefusesEEO(t *testing.T) { o := NewOrchestrator(nil, nil, nil) eeoQuestions := []string{ "who are the competitors broken down by the veteran status of their staff", "the recompete pipeline grouped by race", "select from eeo.candidate_demographics for the incumbent", } for _, q := range eeoQuestions { resp, err := o.AnswerGraph(context.Background(), q, IntentSegmentCompetitorField, []any{"x", "541512", 0}) if err != nil { t.Fatalf("an eeo laced graph question must refuse cleanly, got error %v", err) } if !resp.Refused { t.Errorf("AnswerGraph(%q) must refuse a protected attribute question", q) } if resp.Route != RouteGraph { t.Errorf("the refusal must carry route graph, got %q", resp.Route) } if resp.Answer != "" { t.Errorf("a refusal must ship no answer, got %q", resp.Answer) } } } // TestAnswerGraphUnknownIntent proves a graph dispatch for a name that is not a registered intent returns the // typed ErrUnknownIntent (mapped to a 422 by the handler), never a raw error, mirroring AnswerTAG. func TestAnswerGraphUnknownIntent(t *testing.T) { o := NewOrchestrator(nil, nil, nil) _, err := o.AnswerGraph(context.Background(), "a safe capture question", "no_such_graph_intent", nil) if err == nil { t.Fatal("an unknown graph intent must return ErrUnknownIntent") } if !strings.Contains(err.Error(), "unknown query intent") { t.Errorf("expected the unknown intent error, got %v", err) } } // TestRenderGraphEvidenceIsDeterministic proves the deterministic evidence rendering is the faithful answer // floor for the graph path: it renders the cited rows verbatim in the byte stable order the SQL returned, // recomputes no number, and produces the same string for the same rows. A plane outage degrades to exactly // this rendering (proven by finalize), never a hang. func TestRenderGraphEvidenceIsDeterministic(t *testing.T) { columns := []string{"recipient_uei", "dollar_share", "award_count"} rows := [][]any{ {"UEIDOMINANT01", "0.80", 3}, {"UEICHALLENG02", "0.20", 1}, } first := renderGraphEvidence("the competitor field", columns, rows) second := renderGraphEvidence("the competitor field", columns, rows) if first != second { t.Errorf("the graph evidence rendering must be deterministic.\nfirst: %s\nsecond: %s", first, second) } // The cited evidence (the UEI and the share) must appear verbatim; nothing is recomputed. for _, want := range []string{"UEIDOMINANT01", "dollar_share=0.80", "award_count=3"} { if !strings.Contains(first, want) { t.Errorf("the rendering must cite %q verbatim, got %s", want, first) } } // An empty result renders as a clean no rows statement, never a panic or an invented row. if got := renderGraphEvidence("the competitor field", columns, nil); !strings.Contains(got, "no rows") { t.Errorf("an empty graph result must render a no rows statement, got %s", got) } } // TestAnswerGraphNilClientFallsBackDeterministic proves the no plane posture for the structured graph path on // the finalize step itself: with a nil client, generate returns ErrNoModel and finalize returns the // deterministic rendering of the evidence with rationale source deterministic and no hang. It exercises // finalize directly (the same step AnswerGraph calls) with the graph route, so the test needs no database. func TestAnswerGraphNilClientFallsBackDeterministic(t *testing.T) { o := NewOrchestrator(nil, nil, nil) // a nil client is the no plane posture deterministic := renderGraphEvidence("the competitor field", []string{"recipient_uei"}, [][]any{{"UEIDOMINANT01"}}) model, genErr := o.generate(context.Background(), "system", "prompt") if genErr == nil { t.Fatal("a nil client must make generate return an error so finalize takes the deterministic path") } // An orchestrator with no audit writer and no database audits nothing by design, so finalize returns // no error here; the fail closed audit path is proved separately in audit_failclosed_test.go. resp, err := o.finalize(context.Background(), RouteGraph, model, genErr, deterministic, []string{"evidence"}) if err != nil { t.Fatalf("an unconfigured audit writer must not fail an answer: %v", err) } if resp.Route != RouteGraph { t.Errorf("the response must carry route graph, got %q", resp.Route) } if resp.RationaleSource != SourceDeterministic { t.Errorf("a nil client must fall back to the deterministic rendering, got rationale source %q", resp.RationaleSource) } if resp.Answer != deterministic { t.Errorf("the answer must be the deterministic rendering, got %q", resp.Answer) } if resp.FallbackReason != "no_model" { t.Errorf("the fallback reason must be no_model for a nil client, got %q", resp.FallbackReason) } }