Janady13/missionmatch / internal/modules/ai/graph_route_test.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/ai/graph_route_test.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/modules/ai/graph_route_test.go
Original SHA-256	d5a94bd788026c51bea738b3eb54dc9b66f0fe4a10298bc8c454e82d7f5398fc
Git blob SHA	ef63f21e1b2238fd891f29be2e179a7c789c8113
Reading-copy SHA-256	d5a94bd788026c51bea738b3eb54dc9b66f0fe4a10298bc8c454e82d7f5398fc
Original source ↗
Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.
Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/ai/graph_route_test.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/modules/ai/graph_route_test.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Recovered source transcript

Download complete source text ↓411 lines · 18.2 KB
package ai

import (
	"context"
	"strings"
	"testing"
)

// TestIsGraphIntent pins the closed set of graph intents: the two G3 skill traversals and the four G1 market
// traversals are graph intents (dispatched to AnswerGraph), a TAG intent is not, and an unknown or empty name
// is not. A future intent added to the graph set without being a real registered traversal, or a TAG intent
// mislabeled graph, fails here.
func TestIsGraphIntent(t *testing.T) {
	graph := []string{
		IntentSkillSubstitutionClosure,
		IntentSkillGapBridge,
		IntentSegmentCompetitorField,
		IntentIncumbentOfRecord,
		IntentRecompetePipeline,
		IntentAgencyBuyingPattern,
	}
	for _, name := range graph {
		if !IsGraphIntent(name) {
			t.Errorf("%q must be a graph intent", name)
		}
		// Every graph intent must also be a real registered intent in the closed Registry.
		if _, ok := IntentByName(name); !ok {
			t.Errorf("graph intent %q must be registered in the closed Registry", name)
		}
	}
	notGraph := []string{
		"count_candidates_by_clearance", // a TAG intent
		"proposal_ready_for_requirement",
		"open_requirements_count",
		"slate_decision_mix",
		"", // empty
		"definitely_not_an_intent",
	}
	for _, name := range notGraph {
		if IsGraphIntent(name) {
			t.Errorf("%q must NOT be a graph intent", name)
		}
	}
}

// TestGraphClassifyRoutesCaptureQuestions proves the conservative natural language router maps a free form
// capture or skill question that clearly names a graph traversal AND carries the params to bind it
// unambiguously to the right intent with the right args, the best effort NL path. It never authors a param the
// user did not delimit: the agency must be quoted and the NAICS a bare six digit token.
func TestGraphClassifyRoutesCaptureQuestions(t *testing.T) {
	cases := []struct {
		name     string
		question string
		intent   string
		args     []any
	}{
		{
			name:     "competitor field with quoted agency and bare naics",
			question: `who are the competitors in "DEPT OF THE AIR FORCE" for 541512`,
			intent:   IntentSegmentCompetitorField,
			args:     []any{"DEPT OF THE AIR FORCE", "541512", 0},
		},
		{
			name:     "incumbent question is a competitor segment question",
			question: `who is the incumbent on "DEPT OF DEFENSE" 541330 work`,
			intent:   IntentSegmentCompetitorField,
			args:     []any{"DEPT OF DEFENSE", "541330", 0},
		},
		{
			name:     "recompete pipeline with quoted agency and explicit window",
			question: `show me the recompete pipeline for "DEPT OF THE NAVY" in the next 18 months`,
			intent:   IntentRecompetePipeline,
			args:     []any{"DEPT OF THE NAVY", 18},
		},
		{
			name:     "recompete pipeline with quoted agency and default window",
			question: `what is coming up for recompete at "DEPT OF THE ARMY"`,
			intent:   IntentRecompetePipeline,
			args:     []any{"DEPT OF THE ARMY", 0},
		},
		{
			name:     "agency buying pattern with quoted agency",
			question: `what does the agency buy, the buying pattern for "GSA"`,
			intent:   IntentAgencyBuyingPattern,
			args:     []any{"GSA", ""},
		},
		{
			name:     "agency buying pattern with quoted agency and optional naics",
			question: `where should we position on the buying pattern for "GSA" 541519`,
			intent:   IntentAgencyBuyingPattern,
			args:     []any{"GSA", "541519"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gr := graphClassify(tc.question)
			if gr == nil {
				t.Fatalf("graphClassify(%q) must route to a graph intent", tc.question)
			}
			if gr.intent != tc.intent {
				t.Errorf("intent: got %q, want %q", gr.intent, tc.intent)
			}
			if len(gr.args) != len(tc.args) {
				t.Fatalf("args length: got %d %v, want %d %v", len(gr.args), gr.args, len(tc.args), tc.args)
			}
			for i := range tc.args {
				if gr.args[i] != tc.args[i] {
					t.Errorf("arg %d: got %v, want %v", i, gr.args[i], tc.args[i])
				}
			}
		})
	}
}

// TestGraphClassifyRoutesLifecycleQuestions proves the conservative NL router maps a free form G9 lifecycle
// question that clearly names a traversal AND carries the params to bind it unambiguously to the right intent
// with the right args. The succession needs an explicit opportunity UUID, the recompete clock needs a quoted
// agency, and the forecast runway needs an explicit number of months; it never authors a param the user did
// not delimit.
func TestGraphClassifyRoutesLifecycleQuestions(t *testing.T) {
	oppID := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
	cases := []struct {
		name     string
		question string
		intent   string
		args     []any
	}{
		{
			name:     "procurement succession with an explicit opportunity uuid",
			question: `is this a recompete, walk the succession for opportunity ` + oppID,
			intent:   IntentProcurementSuccession,
			args:     []any{oppID},
		},
		{
			name:     "lifecycle question with an explicit opportunity uuid",
			question: `show me the lifecycle of ` + oppID,
			intent:   IntentProcurementSuccession,
			args:     []any{oppID},
		},
		{
			name:     "recompete clock with quoted agency and explicit window",
			question: `show me the recompete clock for "DEPT OF THE NAVY" over the next 18 months`,
			intent:   IntentRecompeteClock,
			args:     []any{"DEPT OF THE NAVY", 18},
		},
		{
			name:     "recompete clock with quoted agency and default window",
			question: `what is the recompete clock for "DEPT OF THE ARMY"`,
			intent:   IntentRecompeteClock,
			args:     []any{"DEPT OF THE ARMY", 0},
		},
		{
			name:     "forecast runway with an explicit window",
			question: `show me the forecast runway for the next 12 months`,
			intent:   IntentForecastRunway,
			args:     []any{12},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gr := graphClassify(tc.question)
			if gr == nil {
				t.Fatalf("graphClassify(%q) must route to a graph intent", tc.question)
			}
			if gr.intent != tc.intent {
				t.Errorf("intent: got %q, want %q", gr.intent, tc.intent)
			}
			if len(gr.args) != len(tc.args) {
				t.Fatalf("args length: got %d %v, want %d %v", len(gr.args), gr.args, len(tc.args), tc.args)
			}
			for i := range tc.args {
				if gr.args[i] != tc.args[i] {
					t.Errorf("arg %d: got %v, want %v", i, gr.args[i], tc.args[i])
				}
			}
		})
	}
}

// TestGraphClassifyLifecycleFallsBackWhenParamsMissing is the conservatism test for the G9 routes: a lifecycle
// question that clearly names a traversal but does NOT carry the params to bind it returns nil, so the caller
// falls back to RAG rather than guessing an opportunity, an agency, or a window. The model never authors
// params; an unbindable lifecycle question is a RAG question.
func TestGraphClassifyLifecycleFallsBackWhenParamsMissing(t *testing.T) {
	fallback := []string{
		// A succession or lifecycle question with no explicit opportunity UUID: nothing to bind.
		"is this a recompete or a new start",
		"walk the succession of this solicitation",
		"show me the lifecycle of the cloud contract",
		// A recompete clock question with no quoted agency: the agency is ambiguous.
		"show me the recompete clock for the next year",
		"what is the recompete window in general",
		// A forecast runway question with no explicit window: the horizon is ambiguous.
		"show me the forecast runway",
		"what is on the forecast pipeline",
	}
	for _, q := range fallback {
		if gr := graphClassify(q); gr != nil {
			t.Errorf("graphClassify(%q) must fall back (nil) when params cannot be bound, got intent %q args %v", q, gr.intent, gr.args)
		}
	}
}

// TestGraphClassifyBareRecompeteStillRoutesToPipeline pins the byte unchanged existing route: a bare recompete
// question with no clock phrase still routes to the existing G1 recompete pipeline intent (not the new G9
// clock), so the Phase 0 route is untouched. The distinct recompete clock phrase is what selects the new
// intent.
func TestGraphClassifyBareRecompeteStillRoutesToPipeline(t *testing.T) {
	gr := graphClassify(`what is coming up for recompete at "DEPT OF THE ARMY"`)
	if gr == nil {
		t.Fatal("a bare recompete question with a quoted agency must still route")
	}
	if gr.intent != IntentRecompetePipeline {
		t.Errorf("a bare recompete question must route to the existing pipeline, got %q", gr.intent)
	}
}

// TestGraphClassifyFallsBackWhenParamsMissing is the load bearing conservatism test: a question that clearly
// names a graph traversal but does NOT carry the params to bind it unambiguously returns nil, so the caller
// falls back to the existing RAG path rather than guessing a segment, an agency, or a skill. The model never
// authors params; an unbindable graph question is a RAG question.
func TestGraphClassifyFallsBackWhenParamsMissing(t *testing.T) {
	fallback := []string{
		// A competitor question with no quoted agency and no NAICS: nothing to bind.
		"who are the competitors in the cloud market",
		// A competitor question with a NAICS but no quoted agency: the segment key is incomplete.
		"who are the competitors for 541512",
		// A competitor question with a quoted agency but no NAICS: still incomplete.
		`who are the competitors in "DEPT OF THE AIR FORCE"`,
		// A recompete question with no quoted agency: the agency is ambiguous.
		"what is coming up for recompete next year",
		// A buying pattern question with no quoted agency.
		"what does the agency buy in general",
		// A skill adjacency question with no explicit skill id (a user never types a UUID): unbindable.
		"what substitutes for kubernetes",
		"show me the skill adjacency for cloud security",
		// A plain question that names no graph traversal at all.
		"who are the best cloud security candidates",
		"how many candidates hold a secret clearance",
		"what is the clearance hierarchy",
	}
	for _, q := range fallback {
		if gr := graphClassify(q); gr != nil {
			t.Errorf("graphClassify(%q) must fall back (nil) when params cannot be bound, got intent %q args %v", q, gr.intent, gr.args)
		}
	}
}

// TestGraphClassifySkillWithExplicitID proves the only unambiguous skill adjacency binding: a question that
// carries a canonical UUID skill id routes to the substitution closure with that id bound and the default
// depth (a bound zero, so the intent clamps it). Absent a UUID the prior test proves the fallback.
func TestGraphClassifySkillWithExplicitID(t *testing.T) {
	id := "11111111-2222-3333-4444-555555555555"
	gr := graphClassify(`what substitutes for skill ` + id)
	if gr == nil {
		t.Fatalf("a skill question carrying an explicit UUID must route to the substitution closure")
	}
	if gr.intent != IntentSkillSubstitutionClosure {
		t.Errorf("intent: got %q, want %q", gr.intent, IntentSkillSubstitutionClosure)
	}
	if len(gr.args) != 2 || gr.args[0] != id || gr.args[1] != 0 {
		t.Errorf("args: got %v, want [%q 0]", gr.args, id)
	}
}

// TestClassifyUnchanged is the regression guard for default path invariance: the existing Classify cases for
// Refuse, TAG, RAG, and Direct route exactly as before. Classify is untouched by the graph layer (the graph
// NL routing lives in graphClassify, run BEFORE Classify in Answer), so a non graph question classifies byte
// for byte as it did, and an EEO laced question still refuses.
func TestClassifyUnchanged(t *testing.T) {
	cases := map[string]Route{
		// Refuse: an eeo schema name or a protected attribute in plain words.
		"list candidates by race and gender":           RouteRefuse,
		"show me candidates grouped by veteran status": RouteRefuse,
		"select from eeo.candidate_demographics":       RouteRefuse,
		// TAG: an aggregation question.
		"how many candidates hold a secret clearance":   RouteTAG,
		"what is the total number of open requirements": RouteTAG,
		"give me the breakdown of the slate decisions":  RouteTAG,
		// RAG: a retrieval question.
		"who are the best cloud security candidates":  RouteRAG,
		"find me a candidate with a TS SCI clearance": RouteRAG,
		"show me the top engineers":                   RouteRAG,
		// Direct: a stable rules question with no aggregation or retrieval keyword.
		"what is the clearance hierarchy": RouteDirect,
		"explain the scoring rules":       RouteDirect,
	}
	for q, want := range cases {
		if got := Classify(q); got != want {
			t.Errorf("Classify(%q): got %q, want %q", q, got, want)
		}
	}
}

// TestAnswerDispatchesGraphIntentToAnswerGraph proves a named GRAPH intent dispatches to the structured graph
// path: with a nil client and a nil db the dispatch reaches AnswerGraph (which runs ExecuteIntent), so the
// resulting error is the no database error from ExecuteIntent, NOT the unknown intent error AnswerTAG would
// return for a graph name it does not own. This pins the dispatch fork without needing a database.
func TestAnswerDispatchesGraphIntentToAnswerGraph(t *testing.T) {
	o := NewOrchestrator(nil, nil, nil)
	_, err := o.Answer(context.Background(), "", IntentSegmentCompetitorField, []any{"DEPT OF THE AIR FORCE", "541512", 0})
	if err == nil {
		t.Fatal("a graph intent with a nil db must error from ExecuteIntent, not silently succeed")
	}
	// ExecuteIntent returns "ai: no database" when db is nil; AnswerTAG would have returned ErrUnknownIntent
	// for a name it does not recognize, which it would not since the name is registered, but the point is the
	// graph fork ran AnswerGraph. The no database error confirms ExecuteIntent was reached through AnswerGraph.
	if !strings.Contains(err.Error(), "no database") {
		t.Errorf("expected the no database error from ExecuteIntent via AnswerGraph, got %v", err)
	}
}

// TestAnswerGraphRefusesEEO proves the structured graph path runs the EEO and protected attribute guards
// FIRST and refuses on either, the orchestrator precedent. The refusal carries route graph, is marked
// Refused, and ships no answer, so the graph path can never group, filter, or rank by a protected attribute,
// even when a graph intent is named.
func TestAnswerGraphRefusesEEO(t *testing.T) {
	o := NewOrchestrator(nil, nil, nil)
	eeoQuestions := []string{
		"who are the competitors broken down by the veteran status of their staff",
		"the recompete pipeline grouped by race",
		"select from eeo.candidate_demographics for the incumbent",
	}
	for _, q := range eeoQuestions {
		resp, err := o.AnswerGraph(context.Background(), q, IntentSegmentCompetitorField, []any{"x", "541512", 0})
		if err != nil {
			t.Fatalf("an eeo laced graph question must refuse cleanly, got error %v", err)
		}
		if !resp.Refused {
			t.Errorf("AnswerGraph(%q) must refuse a protected attribute question", q)
		}
		if resp.Route != RouteGraph {
			t.Errorf("the refusal must carry route graph, got %q", resp.Route)
		}
		if resp.Answer != "" {
			t.Errorf("a refusal must ship no answer, got %q", resp.Answer)
		}
	}
}

// TestAnswerGraphUnknownIntent proves a graph dispatch for a name that is not a registered intent returns the
// typed ErrUnknownIntent (mapped to a 422 by the handler), never a raw error, mirroring AnswerTAG.
func TestAnswerGraphUnknownIntent(t *testing.T) {
	o := NewOrchestrator(nil, nil, nil)
	_, err := o.AnswerGraph(context.Background(), "a safe capture question", "no_such_graph_intent", nil)
	if err == nil {
		t.Fatal("an unknown graph intent must return ErrUnknownIntent")
	}
	if !strings.Contains(err.Error(), "unknown query intent") {
		t.Errorf("expected the unknown intent error, got %v", err)
	}
}

// TestRenderGraphEvidenceIsDeterministic proves the deterministic evidence rendering is the faithful answer
// floor for the graph path: it renders the cited rows verbatim in the byte stable order the SQL returned,
// recomputes no number, and produces the same string for the same rows. A plane outage degrades to exactly
// this rendering (proven by finalize), never a hang.
func TestRenderGraphEvidenceIsDeterministic(t *testing.T) {
	columns := []string{"recipient_uei", "dollar_share", "award_count"}
	rows := [][]any{
		{"UEIDOMINANT01", "0.80", 3},
		{"UEICHALLENG02", "0.20", 1},
	}
	first := renderGraphEvidence("the competitor field", columns, rows)
	second := renderGraphEvidence("the competitor field", columns, rows)
	if first != second {
		t.Errorf("the graph evidence rendering must be deterministic.\nfirst:  %s\nsecond: %s", first, second)
	}
	// The cited evidence (the UEI and the share) must appear verbatim; nothing is recomputed.
	for _, want := range []string{"UEIDOMINANT01", "dollar_share=0.80", "award_count=3"} {
		if !strings.Contains(first, want) {
			t.Errorf("the rendering must cite %q verbatim, got %s", want, first)
		}
	}
	// An empty result renders as a clean no rows statement, never a panic or an invented row.
	if got := renderGraphEvidence("the competitor field", columns, nil); !strings.Contains(got, "no rows") {
		t.Errorf("an empty graph result must render a no rows statement, got %s", got)
	}
}

// TestAnswerGraphNilClientFallsBackDeterministic proves the no plane posture for the structured graph path on
// the finalize step itself: with a nil client, generate returns ErrNoModel and finalize returns the
// deterministic rendering of the evidence with rationale source deterministic and no hang. It exercises
// finalize directly (the same step AnswerGraph calls) with the graph route, so the test needs no database.
func TestAnswerGraphNilClientFallsBackDeterministic(t *testing.T) {
	o := NewOrchestrator(nil, nil, nil) // a nil client is the no plane posture
	deterministic := renderGraphEvidence("the competitor field", []string{"recipient_uei"}, [][]any{{"UEIDOMINANT01"}})
	model, genErr := o.generate(context.Background(), "system", "prompt")
	if genErr == nil {
		t.Fatal("a nil client must make generate return an error so finalize takes the deterministic path")
	}
	// An orchestrator with no audit writer and no database audits nothing by design, so finalize returns
	// no error here; the fail closed audit path is proved separately in audit_failclosed_test.go.
	resp, err := o.finalize(context.Background(), RouteGraph, model, genErr, deterministic, []string{"evidence"})
	if err != nil {
		t.Fatalf("an unconfigured audit writer must not fail an answer: %v", err)
	}
	if resp.Route != RouteGraph {
		t.Errorf("the response must carry route graph, got %q", resp.Route)
	}
	if resp.RationaleSource != SourceDeterministic {
		t.Errorf("a nil client must fall back to the deterministic rendering, got rationale source %q", resp.RationaleSource)
	}
	if resp.Answer != deterministic {
		t.Errorf("the answer must be the deterministic rendering, got %q", resp.Answer)
	}
	if resp.FallbackReason != "no_model" {
		t.Errorf("the fallback reason must be no_model for a nil client, got %q", resp.FallbackReason)
	}
}