Janady13/missionmatch / internal/modules/ai/safety_test.go

1package ai 2 3import ( 4 "errors" 5 "os" 6 "path/filepath" 7 "strings" 8 "testing" 9) 10 11func TestForbidEEORejectsDemographics(t *testing.T) { 12 bad := []string{ 13 "SELECT * FROM eeo.candidate_demographics", 14 "select gender, race_ethnicity from eeo.candidate_demographics", 15 "join EEO.candidate_demographics on true", 16 "the candidate's veteran_status is", 17 } 18 for _, s := range bad { 19 if err := ForbidEEO(s); !errors.Is(err, ErrEEOForbidden) { 20 t.Errorf("ForbidEEO(%q) must refuse the eeo schema, got %v", s, err) 21 } 22 } 23 if err := ForbidEEO("SELECT full_name FROM candidates WHERE clearance_level='secret'"); err != nil { 24 t.Errorf("a clean query must pass: %v", err) 25 } 26} 27 28func TestGuardReadOnlyQuery(t *testing.T) { 29 bad := map[string]string{ 30 "insert": "INSERT INTO candidates (full_name) VALUES ('x')", 31 "update": "UPDATE candidates SET available=true", 32 "delete": "DELETE FROM candidates", 33 "stacked": "SELECT 1 FROM candidates; DROP TABLE candidates", 34 "eeo read": "SELECT gender FROM eeo.candidate_demographics", 35 "hidden write in cte": "WITH x AS (DELETE FROM candidates RETURNING id) SELECT * FROM x", 36 "non allowlisted": "SELECT * FROM auth_credentials", 37 } 38 for name, sql := range bad { 39 if err := GuardReadOnlyQuery(sql, ReadAllowlist); err == nil { 40 t.Errorf("GuardReadOnlyQuery must refuse %q", name) 41 } 42 } 43 ok := []string{ 44 "SELECT full_name, clearance_level FROM candidates WHERE available = true", 45 "WITH ranked AS (SELECT id FROM slate_members) SELECT count(*) FROM ranked", 46 "SELECT c.full_name FROM candidates c JOIN candidate_skills cs ON cs.candidate_id = c.id", 47 } 48 for _, sql := range ok { 49 if err := GuardReadOnlyQuery(sql, ReadAllowlist); err != nil { 50 t.Errorf("a clean allowlisted read must pass: %q: %v", sql, err) 51 } 52 } 53} 54 55// TestGuardReadOnlyQueryRecursiveCTE is the load bearing security test for the WITH RECURSIVE guard fix. 56// It proves the guard now ACCEPTS a legitimate recursive traversal whose base tables are all allowlisted 57// (the recursive self reference is recognized as a query local CTE, not a missing table) and STILL REJECTS 58// every dangerous shape: a non allowlisted base table, a hidden mutation, an eeo marker, and a stray 59// stacked statement. A regression here would either break every future graph traversal or, worse, open a 60// hole in the read only and eeo guarantees. 61func TestGuardReadOnlyQueryRecursiveCTE(t *testing.T) { 62 // A recursive closure whose only base table is allowlisted (skill_relationships) PASSES. The closure 63 // name is recognized as a CTE, so the self reference in the recursive arm is not flagged as a table. 64 okRecursive := []string{ 65 "WITH RECURSIVE closure AS (" + 66 "SELECT source_id, target_id, 1 AS depth FROM skill_relationships WHERE source_id = $1 " + 67 "UNION " + 68 "SELECT sr.source_id, sr.target_id, c.depth + 1 FROM skill_relationships sr JOIN closure c ON sr.source_id = c.target_id WHERE c.depth < 3" + 69 ") SELECT target_id, depth FROM closure ORDER BY depth, target_id", 70 // A multi CTE recursive query: BOTH a and b must register as CTEs, neither treated as a table. 71 "WITH RECURSIVE a AS (" + 72 "SELECT source_id, target_id FROM skill_relationships WHERE source_id = $1 " + 73 "UNION " + 74 "SELECT sr.source_id, sr.target_id FROM skill_relationships sr JOIN a ON sr.source_id = a.target_id" + 75 "), b AS (SELECT target_id FROM skill_aliases) " + 76 "SELECT a.target_id FROM a JOIN b ON a.target_id = b.target_id ORDER BY a.target_id", 77 } 78 for _, sql := range okRecursive { 79 if err := GuardReadOnlyQuery(sql, ReadAllowlist); err != nil { 80 t.Errorf("a clean recursive allowlisted read must pass: %q: %v", sql, err) 81 } 82 } 83 84 // Every dangerous recursive shape STILL FAILS. 85 badRecursive := map[string]string{ 86 // Base table not in the allowlist: candidate_demographics is also an eeo marker, so this is caught 87 // twice over, but the table not in allowlist path must independently reject an unlisted name too. 88 "recursive eeo base table": "WITH RECURSIVE closure AS (" + 89 "SELECT id FROM candidate_demographics WHERE id = $1 " + 90 "UNION SELECT cd.id FROM candidate_demographics cd JOIN closure c ON cd.id = c.id) SELECT id FROM closure", 91 "recursive unlisted base table": "WITH RECURSIVE closure AS (" + 92 "SELECT id FROM some_unlisted_table WHERE id = $1 " + 93 "UNION SELECT t.id FROM some_unlisted_table t JOIN closure c ON t.id = c.id) SELECT id FROM closure", 94 // A mutation hidden inside the recursive CTE body must be caught by mutatingRe. 95 "recursive hidden delete": "WITH RECURSIVE x AS (DELETE FROM candidates RETURNING id) SELECT id FROM x", 96 "recursive hidden insert": "WITH RECURSIVE x AS (INSERT INTO candidates (full_name) VALUES ('z') RETURNING id) SELECT id FROM x", 97 "recursive hidden update": "WITH RECURSIVE x AS (UPDATE candidates SET available = true RETURNING id) SELECT id FROM x", 98 // An eeo marker anywhere in the recursive body must be caught by ForbidEEO. 99 "recursive eeo marker": "WITH RECURSIVE closure AS (" + 100 "SELECT id, veteran_status FROM candidates WHERE id = $1 " + 101 "UNION SELECT c.id, c.veteran_status FROM candidates c JOIN closure cl ON c.id = cl.id) SELECT id FROM closure", 102 // A stray semicolon (stacked statements) must still fail even with a recursive lead. 103 "recursive stacked": "WITH RECURSIVE closure AS (SELECT id FROM skill_relationships WHERE id = $1) SELECT id FROM closure; DROP TABLE candidates", 104 } 105 for name, sql := range badRecursive { 106 if err := GuardReadOnlyQuery(sql, ReadAllowlist); err == nil { 107 t.Errorf("GuardReadOnlyQuery must refuse %q", name) 108 } 109 } 110 111 // The multi CTE registration is asserted directly against the helper too: both a and b are recognized. 112 ctes := cteNames(strings.ToLower( 113 "with recursive a as (select 1), b as (select 2) select * from a join b on true")) 114 if !ctes["a"] || !ctes["b"] { 115 t.Errorf("a multi CTE WITH RECURSIVE must register both a and b as CTEs, got %v", ctes) 116 } 117} 118 119// TestReadAllowlistComposition pins the security boundary of the read allowlist: the six tables added for 120// the graph layer must be present, and no eeo, auth secret, or audit table may ever appear. A future edit 121// that widens the allowlist into a protected surface fails here. 122func TestReadAllowlistComposition(t *testing.T) { 123 mustHave := []string{ 124 "skill_relationships", "skill_aliases", "ontology_crosswalk", "mission_areas", 125 "federal_mission_tags", "award_incumbent_roles", 126 } 127 for _, tbl := range mustHave { 128 if !ReadAllowlist[tbl] { 129 t.Errorf("ReadAllowlist must contain the graph reference table %q", tbl) 130 } 131 } 132 mustNotHave := []string{ 133 "eeo", "candidate_demographics", "auth_credentials", "auth_sessions", 134 "mfa_secrets", "audit_logs", "*", 135 } 136 for _, tbl := range mustNotHave { 137 if ReadAllowlist[tbl] { 138 t.Errorf("ReadAllowlist must NEVER contain the protected table %q", tbl) 139 } 140 } 141} 142 143// TestRegistryIsSafe asserts every curated TAG intent is read only, allowlisted, and eeo free, so an 144// unsafe intent cannot ship even if added by mistake. 145func TestRegistryIsSafe(t *testing.T) { 146 if err := ValidateRegistry(); err != nil { 147 t.Fatalf("a TAG intent is unsafe: %v", err) 148 } 149 if len(Registry) == 0 { 150 t.Fatal("the TAG registry is empty; the safety check is a no op") 151 } 152} 153 154// TestNoAIFileReferencesEEO is the required tripwire: it walks every Go file in the AI package, the W3 155// staffing recommendations package, and the WS9 bench ingestion surface (the benchimport mapping, the 156// sourcing ingest path, and the import-bench CLI), and fails if any of them, other than the prohibition 157// guard (safety.go) and the tests, names the eeo schema or a demographic column. No embedding, retrieval, 158// graph, memory, generated query, prompt builder, classifier, optimizer, value function, confidence 159// function, bench record, embed profile composer, CSV or JSONL column mapping, IngestBenchRecord, or the 160// CLI may reach demographics; this test makes that hold by construction, not by review. 161func TestNoAIFileReferencesEEO(t *testing.T) { 162 markers := []string{"eeo.", "candidate_demographics", "race_ethnicity", "veteran_status", "disability_status", "from eeo", "join eeo"} 163 // The AI layer, the W3 staffing recommendations layer, and the WS9 bench ingestion surface. The WS9 dirs 164 // are relative to this test's location (internal/modules/ai): benchimport and sourcing are two levels up 165 // under platform, the CLI is three levels up under cmd. INVARIANT 4 requires the tripwire to cover them. 166 dirs := []string{".", "../staffing", "../../platform/benchimport", "../../platform/sourcing", "../../../cmd/import-bench"} 167 total := 0 168 for _, dir := range dirs { 169 total += scanDirForEEO(t, dir, markers) 170 } 171 if total == 0 { 172 t.Fatal("no source files were scanned; the tripwire is not actually running") 173 } 174 t.Logf("scanned %d AI, staffing, and bench ingestion source files, none reach eeo", total) 175} 176 177func scanDirForEEO(t *testing.T, dir string, markers []string) int { 178 entries, err := os.ReadDir(dir) 179 if err != nil { 180 t.Fatalf("read dir %s: %v", dir, err) 181 } 182 checked := 0 183 for _, e := range entries { 184 name := e.Name() 185 if !strings.HasSuffix(name, ".go") { 186 continue 187 } 188 // safety.go IS the prohibition guard (it must name eeo to forbid it); tests describe the rule. 189 if name == "safety.go" || strings.HasSuffix(name, "_test.go") { 190 continue 191 } 192 body, err := os.ReadFile(filepath.Join(dir, name)) 193 if err != nil { 194 t.Fatalf("read %s/%s: %v", dir, name, err) 195 } 196 // A prohibition COMMENT may name eeo (the rule being documented); only a CODE reference is a reach. 197 // Drop comment lines, then scan the executable content. 198 var code strings.Builder 199 for _, line := range strings.Split(string(body), "\n") { 200 if strings.HasPrefix(strings.TrimSpace(line), "//") { 201 continue 202 } 203 code.WriteString(strings.ToLower(line)) 204 code.WriteByte('\n') 205 } 206 lower := code.String() 207 for _, m := range markers { 208 if strings.Contains(lower, m) { 209 t.Errorf("EEO REACH: %s/%s contains %q in executable code; no AI or staffing path may name the eeo schema", dir, name, m) 210 } 211 } 212 checked++ 213 } 214 return checked 215} 216