Janady13/missionmatch / internal/modules/ai/graph_agency_test.go

1package ai 2 3import ( 4 "strings" 5 "testing" 6) 7 8// TestGraphAgencyIntentsAreRegistered proves the three G7 agency buying pattern traversals are in the closed 9// Registry, so ValidateRegistry asserts them at boot and ExecuteIntent can run them. A missing intent means 10// the registration regressed. 11func TestGraphAgencyIntentsAreRegistered(t *testing.T) { 12 cases := map[string]int{ 13 IntentAgencyBuyingPatternDetail: 2, // agency_prefix, naics 14 IntentAgencyVendorConcentration: 2, // agency, naics 15 IntentAgencySpendTrend: 2, // agency, naics 16 } 17 for name, wantParams := range cases { 18 it, ok := IntentByName(name) 19 if !ok { 20 t.Fatalf("intent %q must be registered in the closed Registry", name) 21 } 22 if len(it.Params) != wantParams { 23 t.Errorf("intent %q must bind exactly %d params, got %d", name, wantParams, len(it.Params)) 24 } 25 } 26} 27 28// TestGraphAgencyIntentsPassTheReadOnlyGuard is the load bearing security assertion: each registered agency 29// intent passes GuardReadOnlyQuery against the real ReadAllowlist. A query that referenced a non allowlisted 30// table, hid a mutation, or named an eeo marker would fail here, the same gate ValidateRegistry runs at boot 31// and ExecuteIntent re runs at execution time. 32func TestGraphAgencyIntentsPassTheReadOnlyGuard(t *testing.T) { 33 for _, sql := range []string{ 34 agencyBuyingPatternDetailSQL, 35 agencyVendorConcentrationSQL, 36 agencySpendTrendSQL, 37 } { 38 if err := GuardReadOnlyQuery(sql, ReadAllowlist); err != nil { 39 t.Errorf("a registered agency intent must pass the read only guard: %v", err) 40 } 41 } 42} 43 44// TestValidateRegistryWithAgencyIntents proves the whole closed Registry, now carrying the three G7 intents 45// alongside the TAG, G3, G1, and G9 intents, validates at boot. This is the assertion the api boot runs. 46func TestValidateRegistryWithAgencyIntents(t *testing.T) { 47 if err := ValidateRegistry(); err != nil { 48 t.Fatalf("ValidateRegistry must pass with the three G7 agency intents registered: %v", err) 49 } 50} 51 52// TestAgencyRollupUsesSplitPartNotRecursion confirms the agency rollup is the sam_opportunities 53// full_parent_path string split plus a GROUP BY, NOT a recursive CTE: the SAM hierarchy is already a 54// materialized dotted path, so recursion would add a determinism risk and no value. The honesty mandate is 55// explicit that the agency rollup is a split_part, never a WITH RECURSIVE. 56func TestAgencyRollupUsesSplitPartNotRecursion(t *testing.T) { 57 // The detail intent is the only one that rolls up the agency hierarchy; the vendor concentration and the 58 // spend trend key directly on the award awarding_agency. 59 lower := strings.ToLower(agencyBuyingPatternDetailSQL) 60 if !strings.Contains(lower, "split_part(s.full_parent_path, '.', 1)") { 61 t.Error("the agency rollup must use split_part(full_parent_path, '.', 1), the materialized path split") 62 } 63 for name, sql := range map[string]string{ 64 "agency_buying_pattern_detail": agencyBuyingPatternDetailSQL, 65 "agency_vendor_concentration": agencyVendorConcentrationSQL, 66 "agency_spend_trend": agencySpendTrendSQL, 67 } { 68 if strings.Contains(strings.ToLower(sql), "with recursive") { 69 t.Errorf("%s: a G7 agency read must not use a recursive CTE; the hierarchy is a materialized path", name) 70 } 71 } 72} 73 74// TestAgencyAwardsArmsKeyOnlyOnAgencyAndNaics is the honesty mandate, pinned statically: usaspending_awards 75// carries NO psc, NO set_aside, and NO pop_state column, so every G7 arm that reads the award table (the award 76// totals in the detail intent, the vendor concentration, and the spend trend) may not select or join on those 77// columns against the award alias. The award arms key ONLY on awarding_agency and naics_code. A regression that 78// reached for a phantom award column would fail here. 79func TestAgencyAwardsArmsKeyOnlyOnAgencyAndNaics(t *testing.T) { 80 for name, sql := range map[string]string{ 81 "agency_buying_pattern_detail": agencyBuyingPatternDetailSQL, 82 "agency_vendor_concentration": agencyVendorConcentrationSQL, 83 "agency_spend_trend": agencySpendTrendSQL, 84 } { 85 lower := strings.ToLower(sql) 86 // The phantom columns the award table does NOT have. None may appear qualified to the award alias a. 87 for _, phantom := range []string{"a.psc", "a.set_aside", "a.pop_state", "a.award_date"} { 88 if strings.Contains(lower, phantom) { 89 t.Errorf("%s: usaspending_awards has no %s column; an award arm must not reference it", name, phantom) 90 } 91 } 92 } 93 // The detail award totals and the vendor concentration and the spend trend all key on awarding_agency and 94 // naics_code, the only keys usaspending_awards carries. 95 for name, sql := range map[string]string{ 96 "agency_vendor_concentration": agencyVendorConcentrationSQL, 97 "agency_spend_trend": agencySpendTrendSQL, 98 } { 99 lower := strings.ToLower(sql) 100 if !strings.Contains(lower, "a.awarding_agency = $1") { 101 t.Errorf("%s: the award arm must key on awarding_agency", name) 102 } 103 if !strings.Contains(lower, "a.naics_code = $2") { 104 t.Errorf("%s: the award arm must key on naics_code", name) 105 } 106 } 107} 108 109// TestAgencySpendTrendKeysFiscalPeriodOnPopStart pins the honesty detail that usaspending_awards has no 110// award_date column, so the obligation trend keys the fiscal period on the only date the award carries, 111// pop_start_date, and a null pop_start_date is excluded rather than bucketed under a fabricated year. A 112// regression that invented an award_date or bucketed a null year would fail here. 113func TestAgencySpendTrendKeysFiscalPeriodOnPopStart(t *testing.T) { 114 lower := strings.ToLower(agencySpendTrendSQL) 115 if !strings.Contains(lower, "date_part('year', a.pop_start_date)") { 116 t.Error("the spend trend must key the fiscal period on pop_start_date, the only award date the table carries") 117 } 118 if !strings.Contains(lower, "a.pop_start_date is not null") { 119 t.Error("the spend trend must exclude a null pop_start_date rather than bucket it under a fabricated year") 120 } 121} 122 123// TestAgencyNoFloatTouchesADollarShareOrBand is the decimal mandate, pinned statically across all three G7 124// intents: no ::float8, no ::float, no ::double precision, and no ::real cast may touch a dollar figure, a 125// share, an average, a median, or a band. Every dollar, average, median (percentile_cont, a numeric), and 126// share stays Postgres numeric end to end and is scanned into shopspring/decimal in Go. A cast to a float type 127// would silently lose the bit stable precision the AEDT compliance asset requires. 128func TestAgencyNoFloatTouchesADollarShareOrBand(t *testing.T) { 129 intents := map[string]string{ 130 "agency_buying_pattern_detail": agencyBuyingPatternDetailSQL, 131 "agency_vendor_concentration": agencyVendorConcentrationSQL, 132 "agency_spend_trend": agencySpendTrendSQL, 133 } 134 for name, sql := range intents { 135 lower := strings.ToLower(sql) 136 for _, floatCast := range []string{"::float8", "::float", "::double precision", "::real"} { 137 if strings.Contains(lower, floatCast) { 138 t.Errorf("%s: a float cast %q must never touch a dollar, a share, or a band (decimal mandate)", name, floatCast) 139 } 140 } 141 } 142 // The median is computed with percentile_cont, a numeric, never a float histogram; pin its presence so the 143 // value band stays exact decimal. 144 if !strings.Contains(strings.ToLower(agencyBuyingPatternDetailSQL), "percentile_cont(0.5) within group") { 145 t.Error("the value band median must be percentile_cont, a numeric, scanned into decimal") 146 } 147} 148 149// TestAgencyEveryListOrderEndsWithAUniqueTiebreaker pins the byte stable mandate: every G7 intent that returns 150// a list ends its ORDER BY with a unique tiebreaker, so the same inputs yield byte identical rows. The detail 151// intent orders by the record count then the grouped slice keys (a total order over the grouped rows), the 152// vendor concentration orders by the share then the recipient UEI, and the spend trend orders by the fiscal 153// period (a unique grouping key, one row per year, a total order on its own). 154func TestAgencyEveryListOrderEndsWithAUniqueTiebreaker(t *testing.T) { 155 listCases := []struct { 156 name string 157 sql string 158 mustContain string 159 }{ 160 // The detail intent orders by the record count then the full grouped slice key, a total order. 161 {"agency_buying_pattern_detail", agencyBuyingPatternDetailSQL, "s.naics, s.psc, s.set_aside, s.contract_vehicle"}, 162 // The vendor concentration orders by the share then the recipient UEI tiebreaker. 163 {"agency_vendor_concentration", agencyVendorConcentrationSQL, "w.recipient_uei"}, 164 // The spend trend orders by the fiscal period, the unique grouping key. 165 {"agency_spend_trend", agencySpendTrendSQL, "fiscal_period"}, 166 } 167 for _, tc := range listCases { 168 lower := strings.ToLower(tc.sql) 169 idx := strings.LastIndex(lower, "order by") 170 if idx < 0 { 171 t.Errorf("%s: the list must be ordered for byte stability", tc.name) 172 continue 173 } 174 orderClause := lower[idx:] 175 if !strings.Contains(orderClause, strings.ToLower(tc.mustContain)) { 176 t.Errorf("%s: the ORDER BY must carry the unique tiebreaker %q, got %q", tc.name, tc.mustContain, orderClause) 177 } 178 } 179} 180 181// TestAgencyNullableSlicesAreCoalescedOrNullableTargets pins the G9 NULL discipline carried into G7: a nullable 182// text slice column (naics, psc, set_aside, contract_vehicle) is COALESCEd to the empty string so a record with 183// a missing slice still groups cleanly, and the nullable numeric value band (the average and the median) is left 184// as a numeric so a slice with no priced record scans into a decimal pointer rather than failing the scan. A 185// live staging run will hit these NULLs, so the discipline is load bearing. 186func TestAgencyNullableSlicesAreCoalescedOrNullableTargets(t *testing.T) { 187 lower := strings.ToLower(agencyBuyingPatternDetailSQL) 188 for _, coalesced := range []string{ 189 "coalesce(o.naics, '')", 190 "coalesce(o.psc, '')", 191 "coalesce(o.set_aside, '')", 192 "coalesce(o.contract_vehicle, '')", 193 } { 194 if !strings.Contains(lower, coalesced) { 195 t.Errorf("the detail intent must COALESCE the nullable slice column %q so a missing slice groups cleanly", coalesced) 196 } 197 } 198 // The average and median are aggregate over a nullable column, so they are nullable numerics; the Go scan 199 // targets are *decimal.Decimal. Pin the row struct fields are decimal pointers (the nullable target shape). 200 r := BuyingPatternDetailRow{} 201 if r.AvgValue != nil || r.MedianValue != nil { 202 t.Error("the value band average and median must be nullable decimal pointers so a slice with no priced record scans cleanly") 203 } 204} 205 206// TestAgencyIntentsArePublicTablesOnly pins the public data mandate: every G7 intent reads ONLY the global 207// public market tables (opportunity_records, usaspending_awards, sam_opportunities) and never a tenant table. 208// A regression that joined a tenant table (candidates, requirements, the bench, the eeo schema) would fail 209// here, the same boundary the read allowlist enforces. 210func TestAgencyIntentsArePublicTablesOnly(t *testing.T) { 211 publicTables := map[string]bool{ 212 "opportunity_records": true, 213 "usaspending_awards": true, 214 "sam_opportunities": true, 215 } 216 tenantTables := []string{ 217 "candidates", "candidate_skills", "candidate_mission_tags", "requirements", 218 "required_skills", "requirement_mission_tags", "slates", "slate_members", 219 "candidate_evaluations", "candidate_demographics", "eeo", 220 } 221 for name, sql := range map[string]string{ 222 "agency_buying_pattern_detail": agencyBuyingPatternDetailSQL, 223 "agency_vendor_concentration": agencyVendorConcentrationSQL, 224 "agency_spend_trend": agencySpendTrendSQL, 225 } { 226 lower := strings.ToLower(sql) 227 for _, tt := range tenantTables { 228 // A whole word boundary check so a public table that contains a tenant word as a substring is not a 229 // false positive (none here, but the boundary keeps the assertion honest). 230 if strings.Contains(lower, " "+tt+" ") || strings.Contains(lower, " "+tt+"\n") || strings.Contains(lower, "from "+tt) || strings.Contains(lower, "join "+tt) { 231 t.Errorf("%s: must not read the tenant or eeo table %q (public data only)", name, tt) 232 } 233 } 234 // Positively confirm the intent reads at least one of the public market tables. 235 var sawPublic bool 236 for pt := range publicTables { 237 if strings.Contains(lower, pt) { 238 sawPublic = true 239 } 240 } 241 if !sawPublic { 242 t.Errorf("%s: must read a public market table", name) 243 } 244 } 245} 246 247// TestAgencyIntentsAreGraphIntents proves the three G7 intents are classified as GRAPH intents (dispatched to 248// AnswerGraph, not AnswerTAG), so the graph evidence rendering and the graph route audit apply, and each is 249// also a real registered intent in the closed Registry. 250func TestAgencyIntentsAreGraphIntents(t *testing.T) { 251 for _, name := range []string{IntentAgencyBuyingPatternDetail, IntentAgencyVendorConcentration, IntentAgencySpendTrend} { 252 if !IsGraphIntent(name) { 253 t.Errorf("%q must be a graph intent", name) 254 } 255 if _, ok := IntentByName(name); !ok { 256 t.Errorf("graph intent %q must be registered in the closed Registry", name) 257 } 258 } 259} 260 261// TestGraphClassifyRoutesAgencyQuestions proves the conservative NL router maps a free form G7 agency question 262// that clearly names a traversal AND carries the params to bind it unambiguously to the right intent with the 263// right args. The vendor concentration and the spend trend need BOTH a quoted agency and a bare NAICS; the 264// buying pattern detail needs a quoted agency with an optional NAICS. It never authors a param the user did not 265// delimit. 266func TestGraphClassifyRoutesAgencyQuestions(t *testing.T) { 267 cases := []struct { 268 name string 269 question string 270 intent string 271 args []any 272 }{ 273 { 274 name: "vendor concentration with quoted agency and bare naics", 275 question: `show me the vendor concentration in "DEPT OF THE AIR FORCE" for 541512`, 276 intent: IntentAgencyVendorConcentration, 277 args: []any{"DEPT OF THE AIR FORCE", "541512"}, 278 }, 279 { 280 name: "spend trend with quoted agency and bare naics", 281 question: `what is the spend trend for "DEPT OF DEFENSE" 541330, concentrating or fragmenting`, 282 intent: IntentAgencySpendTrend, 283 args: []any{"DEPT OF DEFENSE", "541330"}, 284 }, 285 { 286 name: "buying pattern detail with quoted agency and optional naics", 287 question: `give me the vehicle mix and value band, the buying pattern detail for "GSA" 541519`, 288 intent: IntentAgencyBuyingPatternDetail, 289 args: []any{"GSA", "541519"}, 290 }, 291 { 292 name: "buying pattern detail with quoted agency and no naics", 293 question: `how does the agency buy, the vehicle mix for "GSA"`, 294 intent: IntentAgencyBuyingPatternDetail, 295 args: []any{"GSA", ""}, 296 }, 297 } 298 for _, tc := range cases { 299 t.Run(tc.name, func(t *testing.T) { 300 gr := graphClassify(tc.question) 301 if gr == nil { 302 t.Fatalf("graphClassify(%q) must route to a graph intent", tc.question) 303 } 304 if gr.intent != tc.intent { 305 t.Errorf("intent: got %q, want %q", gr.intent, tc.intent) 306 } 307 if len(gr.args) != len(tc.args) { 308 t.Fatalf("args length: got %d %v, want %d %v", len(gr.args), gr.args, len(tc.args), tc.args) 309 } 310 for i := range tc.args { 311 if gr.args[i] != tc.args[i] { 312 t.Errorf("arg %d: got %v, want %v", i, gr.args[i], tc.args[i]) 313 } 314 } 315 }) 316 } 317} 318 319// TestGraphClassifyAgencyFallsBackWhenParamsMissing is the conservatism test for the G7 routes: an agency 320// question that clearly names a traversal but does NOT carry the params to bind it returns nil, so the caller 321// falls back to RAG rather than guessing an agency or a NAICS. The model never authors params; an unbindable 322// agency question is a RAG question. 323func TestGraphClassifyAgencyFallsBackWhenParamsMissing(t *testing.T) { 324 fallback := []string{ 325 // A vendor concentration question with no quoted agency and no NAICS: nothing to bind. 326 "who has the vendor concentration in the cloud market", 327 // A vendor concentration question with a NAICS but no quoted agency: the segment key is incomplete. 328 "what is the vendor concentration for 541512", 329 // A vendor concentration question with a quoted agency but no NAICS: still incomplete. 330 `what is the vendor concentration in "DEPT OF THE AIR FORCE"`, 331 // A spend trend question with no quoted agency. 332 "is the spend trend concentrating or fragmenting in general", 333 // A spend trend question with a quoted agency but no NAICS. 334 `what is the spend trend for "DEPT OF DEFENSE"`, 335 // A buying pattern detail question with no quoted agency. 336 "what is the vehicle mix in general", 337 } 338 for _, q := range fallback { 339 if gr := graphClassify(q); gr != nil { 340 t.Errorf("graphClassify(%q) must fall back (nil) when params cannot be bound, got intent %q args %v", q, gr.intent, gr.args) 341 } 342 } 343} 344 345// TestGraphClassifyExistingBuyingPatternRouteUnchanged pins the Phase 0 route is byte unchanged: the existing 346// G1 buying pattern keywords (agency buys, buying pattern, where to position, set aside mix) still route to the 347// existing G1 IntentAgencyBuyingPattern, not the new G7 detail intent. The new detail intent is selected only by 348// its distinct detail keywords (vehicle mix, value band, buying pattern detail), so the existing route is 349// untouched. 350func TestGraphClassifyExistingBuyingPatternRouteUnchanged(t *testing.T) { 351 gr := graphClassify(`what does the agency buy, the buying pattern for "GSA"`) 352 if gr == nil { 353 t.Fatal("the existing G1 buying pattern question with a quoted agency must still route") 354 } 355 if gr.intent != IntentAgencyBuyingPattern { 356 t.Errorf("the existing buying pattern question must route to the Phase 0 G1 intent, got %q", gr.intent) 357 } 358} 359