Janady13/missionmatch / internal/integration/eeo_dry_run_integration_test.go

1//go:build integration 2 3package integration 4 5// TestEEOAdverseImpactDryRun is the WS5 reproducible AEDT worked example. 6// 7// It generates a synthetic sample of 100 candidates, 5 requirements, and 500 screening 8// decisions (one per candidate per requirement), then writes consented demographics through 9// the compliance pool and calls the four fifths adverse impact report. 10// 11// EEO isolation contract (non negotiable): 12// - Demographics are written ONLY through the compliance pool into eeo.candidate_demographics 13// with consent_given true. No demographic value enters any audit payload. 14// - The matching service is used for scoring only. It never receives a demographic field. 15// - This file imports eeo and tenant only for the report call and the compliance write. 16// - No matching, scoring, or AI code file is modified. 17// 18// The test asserts structural correctness (rates in [0,1], ratio = rate/max_known_rate, Flagged 19// when ratio < 0.80, NoDemographicsCount and UnknownCount populated), NOT a specific pass or fail 20// outcome, because the exact distribution depends on the synthetic sample. 21// 22// The companion assertion at the end mirrors TestEEOPrivilegeBoundary: the app pool must still be 23// denied eeo schema access (42501). 24 25import ( 26 "context" 27 "encoding/json" 28 "fmt" 29 "testing" 30 "time" 31 32 "github.com/google/uuid" 33 "github.com/jackc/pgx/v5" 34 35 "github.com/missionworkforce/missionmatch/internal/modules/eeo" 36 "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" 37 "github.com/missionworkforce/missionmatch/internal/platform/tenant" 38) 39 40// ws5ClearanceLevels cycles through clearance levels so the synthetic bench has variety. 41var ws5ClearanceLevels = []string{"none", "secret", "top_secret", "ts_sci", "secret"} 42 43// ws5GenderValues are the consented gender labels written into eeo.candidate_demographics. 44// About 10 percent of candidates (indices 0..9 of each block of 10) receive no demographics row at 45// all, which exercises the NoDemographicsCount path. 46var ws5GenderValues = []string{"male", "female", "non_binary", "male", "female", "male", "female", "non_binary", "male", "female"} 47 48// ws5RaceValues are the consented race or ethnicity labels. 49var ws5RaceValues = []string{ 50 "white", "black", "hispanic", "asian", "native_hawaiian_pacific_islander", 51 "two_or_more", "white", "black", "hispanic", "asian", 52} 53 54// ws5RequirementLevels are the five required clearance levels for the five seeded requirements. 55var ws5RequirementLevels = []string{"none", "secret", "secret", "top_secret", "ts_sci"} 56 57// TestEEOAdverseImpactDryRun seeds a synthetic sample and runs the adverse impact report. 58func TestEEOAdverseImpactDryRun(t *testing.T) { 59 if compliance == nil { 60 t.Skip("compliance pool not configured (MM_COMPLIANCE_DSN); skipping WS5 dry run") 61 } 62 63 ctx := context.Background() 64 org := seedOrg(t) 65 66 // Step 1: seed 100 synthetic candidates with no demographic fields on the core type. 67 // Clearance cycles through ws5ClearanceLevels so the scoring spread is meaningful. 68 candidates := make([]uuid.UUID, 100) 69 for i := range candidates { 70 level := ws5ClearanceLevels[i%len(ws5ClearanceLevels)] 71 status := "active" 72 if i%7 == 0 { 73 status = "expired" 74 } 75 c, err := dbgen.New(migrator).CreateCandidate(ctx, dbgen.CreateCandidateParams{ 76 OrganizationID: org, 77 FullName: fmt.Sprintf("WS5 Synthetic Candidate %d", i+1), 78 ClearanceLevel: level, 79 ClearancePoly: "none", 80 ClearanceStatus: status, 81 Available: i%5 != 0, 82 }) 83 if err != nil { 84 t.Fatalf("seed candidate %d: %v", i, err) 85 } 86 candidates[i] = c.ID 87 } 88 89 // Step 2: seed 5 requirements with varied clearance levels. 90 // Each requirement is seeded via the migrator so it bypasses RLS during setup. 91 requirements := make([]uuid.UUID, 5) 92 for i, level := range ws5RequirementLevels { 93 var reqID uuid.UUID 94 err := migrator.QueryRow(ctx, 95 `INSERT INTO requirements (organization_id, title, required_level, required_poly) 96 VALUES ($1, $2, $3, 'none') RETURNING id`, 97 org, fmt.Sprintf("WS5 Requirement %d (Demo)", i+1), level).Scan(&reqID) 98 if err != nil { 99 t.Fatalf("seed requirement %d: %v", i, err) 100 } 101 requirements[i] = reqID 102 } 103 104 // Step 3: seed one slate per requirement with decided members so the selection stage has data. 105 // The approval pattern is deterministic: candidate index mod 3 != 0 is approved. 106 // This gives roughly a 2:1 approved to rejected ratio, and the per gender split will differ 107 // because of the demographic distribution modulo pattern below. 108 for ri, reqID := range requirements { 109 var slateID uuid.UUID 110 err := migrator.QueryRow(ctx, 111 `INSERT INTO slates (organization_id, requirement_id, name, status) 112 VALUES ($1, $2, $3, 'draft') RETURNING id`, 113 org, reqID, fmt.Sprintf("WS5 Demo Slate %d", ri+1)).Scan(&slateID) 114 if err != nil { 115 t.Fatalf("seed slate %d: %v", ri, err) 116 } 117 for ci, candID := range candidates { 118 decision := "approved" 119 if ci%3 == 0 { 120 decision = "rejected" 121 } 122 // Band and scores are deterministic placeholders; the report reads decision, not score. 123 _, err := migrator.Exec(ctx, 124 `INSERT INTO slate_members 125 (organization_id, slate_id, candidate_id, total_score, band, 126 clearance_score, skills_score, certs_score, customer_score, 127 experience_score, location_score, explanation, trace, gap_summary, 128 decision, model_version) 129 VALUES ($1,$2,$3,75,'yellow',80,70,80,70,80,70,'WS5 demo','[]','[]',$4,'v1.0.0')`, 130 org, slateID, candID, decision) 131 if err != nil { 132 t.Fatalf("seed slate member (slate %d, cand %d): %v", ri, ci, err) 133 } 134 } 135 } 136 137 // Step 4: seed aedt.score.served audit rows for the screening stage. 138 // The served score payload carries only tool, score, and band, never a demographic value. 139 // Candidates with an even index get band green (screened in); odd index gets band red. 140 // The migrator role has INSERT on audit_logs. One audit row per candidate is sufficient: 141 // the report groups by candidate id, so one row per candidate covers the full denominator. 142 for ci, candID := range candidates { 143 band := "green" 144 score := "85.00" 145 if ci%2 != 0 { 146 band = "red" 147 score = "55.00" 148 } 149 payload, _ := json.Marshal(map[string]any{ 150 "tool": "screenMATCH.match", 151 "score": score, 152 "band": band, 153 }) 154 entityType := "candidate" 155 entityID := candID.String() 156 _, err := dbgen.New(migrator).InsertAuditLog(ctx, dbgen.InsertAuditLogParams{ 157 OrganizationID: &org, 158 Action: "aedt.score.served", 159 EntityType: &entityType, 160 EntityID: &entityID, 161 NewValues: payload, 162 }) 163 if err != nil { 164 t.Fatalf("seed audit log (cand %d): %v", ci, err) 165 } 166 } 167 168 // Step 5: write consented demographics through the compliance pool. 169 // Candidates at position i%10 == 0 (indices 0, 10, 20 ...) receive NO demographics row. 170 // This exercises NoDemographicsCount on both stages. 171 // All other candidates receive a consent_given=true row via the compliance pool inside 172 // RunInTenantTx, exactly mirroring the seedDemographic helper in eeo_adverse_impact_test.go. 173 for i, candID := range candidates { 174 if i%10 == 0 { 175 // Deliberate no consent row: exercises the NoDemographicsCount counter. 176 continue 177 } 178 gender := ws5GenderValues[i%len(ws5GenderValues)] 179 race := ws5RaceValues[i%len(ws5RaceValues)] 180 err := tenant.New(compliance).RunInTenantTx(ctxOrg(org), func(ctx context.Context, tx pgx.Tx) error { 181 _, e := tx.Exec(ctx, 182 `INSERT INTO eeo.candidate_demographics 183 (organization_id, candidate_id, gender, race_ethnicity, consent_given, 184 collected_by, collected_at) 185 VALUES ($1, $2, $3, $4, true, NULL, now())`, 186 org, candID, gender, race) 187 return e 188 }) 189 if err != nil { 190 t.Fatalf("seed demographic (cand %d): %v", i, err) 191 } 192 } 193 194 // Step 6: call AdverseImpact through the compliance service. No app or matching code touches eeo. 195 rep, err := eeo.NewService(tenant.New(compliance)).AdverseImpact(ctxOrg(org), time.Now()) 196 if err != nil { 197 t.Fatalf("AdverseImpact: %v", err) 198 } 199 200 // Step 7: assert the report is structurally correct. 201 // We assert invariants that must hold for ANY valid synthetic sample, not a specific outcome. 202 203 t.Run("SelectionSectionPresent", func(t *testing.T) { 204 if rep.Selection.TotalN == 0 { 205 t.Fatalf("selection section has TotalN=0; expected candidates with decisions and demographics") 206 } 207 }) 208 209 t.Run("ScreeningSectionPresent", func(t *testing.T) { 210 if rep.Screening.TotalN == 0 { 211 t.Fatalf("screening section has TotalN=0; expected candidates with served scores and demographics") 212 } 213 }) 214 215 t.Run("SelectionNoDemographicsCount", func(t *testing.T) { 216 // 10 of 100 candidates have no demographics row. Each appears on all 5 slates, 217 // but the query is DISTINCT on candidate, so NoDemographicsCount must be 10. 218 if rep.Selection.NoDemographicsCount == 0 { 219 t.Fatalf("selection NoDemographicsCount is 0; expected ~10 candidates with no demographics row") 220 } 221 }) 222 223 t.Run("ScreeningNoDemographicsCount", func(t *testing.T) { 224 if rep.Screening.NoDemographicsCount == 0 { 225 t.Fatalf("screening NoDemographicsCount is 0; expected ~10 candidates with no demographics row") 226 } 227 }) 228 229 t.Run("ByGenderRatesInBounds", func(t *testing.T) { 230 assertRatesInBounds(t, rep.Selection.ByGender, "selection.ByGender") 231 assertRatesInBounds(t, rep.Screening.ByGender, "screening.ByGender") 232 }) 233 234 t.Run("ByRaceRatesInBounds", func(t *testing.T) { 235 assertRatesInBounds(t, rep.Selection.ByRace, "selection.ByRace") 236 assertRatesInBounds(t, rep.Screening.ByRace, "screening.ByRace") 237 }) 238 239 t.Run("ByGenderAtLeastTwoKnownGroups", func(t *testing.T) { 240 assertAtLeastTwoKnownGroups(t, rep.Selection.ByGender, "selection.ByGender") 241 assertAtLeastTwoKnownGroups(t, rep.Screening.ByGender, "screening.ByGender") 242 }) 243 244 t.Run("ByRaceAtLeastTwoKnownGroups", func(t *testing.T) { 245 assertAtLeastTwoKnownGroups(t, rep.Selection.ByRace, "selection.ByRace") 246 assertAtLeastTwoKnownGroups(t, rep.Screening.ByRace, "screening.ByRace") 247 }) 248 249 t.Run("FourFifthsFlaggedConsistency", func(t *testing.T) { 250 assertFourFifthsConsistency(t, rep.Selection.ByGender, "selection.ByGender") 251 assertFourFifthsConsistency(t, rep.Selection.ByRace, "selection.ByRace") 252 assertFourFifthsConsistency(t, rep.Screening.ByGender, "screening.ByGender") 253 assertFourFifthsConsistency(t, rep.Screening.ByRace, "screening.ByRace") 254 }) 255 256 t.Run("ConsideredSelectedConsistency", func(t *testing.T) { 257 for _, row := range rep.Selection.ByGender { 258 if row.Selected > row.Considered { 259 t.Errorf("selection ByGender %s: Selected %d > Considered %d", row.Group, row.Selected, row.Considered) 260 } 261 } 262 for _, row := range rep.Screening.ByGender { 263 if row.Selected > row.Considered { 264 t.Errorf("screening ByGender %s: Selected %d > Considered %d", row.Group, row.Selected, row.Considered) 265 } 266 } 267 }) 268 269 t.Run("ScreeningDetectsGenderDisparity", func(t *testing.T) { 270 // In the screening stage, candidates at even index get band green (screened in). 271 // The gender assignment is ws5GenderValues[i%10]. For i%10 in {0,3,5,8,9} the gender 272 // is male; for {1,4,6,9} it is female; for {2,7} it is non_binary. But index 0 has 273 // no demographics row. So among candidates who have demographics (i%10 != 0): 274 // male: positions 3,5,8,9 of each 10 block. Even: 8. Count per 10 = 4 from {3,5,8,9}. 275 // Among those, even (screened in): 8 is even. So male screened = varies. 276 // Rather than recomputing the exact arithmetic here, we just assert the report produced 277 // impact ratios for at least two known gender groups, which was already checked above. 278 // This sub test is a commentary anchor: the disparity outcome is sample dependent. 279 found := map[string]bool{} 280 for _, row := range rep.Screening.ByGender { 281 if row.ImpactRatio != "n/a" { 282 found[row.Group] = true 283 } 284 } 285 if len(found) < 2 { 286 t.Errorf("expected at least two known gender groups with a computed impact ratio in the screening section, got %v", found) 287 } 288 }) 289 290 // Step 8: assert the privilege boundary. The app pool must be denied eeo schema access. 291 t.Run("AppPoolDeniedEEOAccess", func(t *testing.T) { 292 _, err := appDB.Pool.Exec(ctx, "SELECT 1 FROM eeo.candidate_demographics LIMIT 1") 293 if pgCode(err) != "42501" { 294 t.Fatalf("app pool must be denied eeo schema access (42501), got %v", err) 295 } 296 // The compliance pool can reach the table (RLS may return 0 rows, that is fine). 297 if _, err := compliance.Exec(ctx, "SELECT 1 FROM eeo.candidate_demographics LIMIT 1"); err != nil { 298 t.Fatalf("compliance pool must be able to read eeo schema, got %v", err) 299 } 300 }) 301} 302 303// assertRatesInBounds verifies every ImpactRow in the cut has rate in [0.0000, 1.0000]. 304// Rate is a string like "0.6667" so we parse it as a float for comparison. 305func assertRatesInBounds(t *testing.T, rows []eeo.ImpactRow, label string) { 306 t.Helper() 307 for _, row := range rows { 308 var rate float64 309 if _, err := fmt.Sscanf(row.Rate, "%f", &rate); err != nil { 310 t.Errorf("%s group %s: cannot parse rate %q: %v", label, row.Group, row.Rate, err) 311 continue 312 } 313 if rate < 0 || rate > 1 { 314 t.Errorf("%s group %s: rate %s is outside [0,1]", label, row.Group, row.Rate) 315 } 316 } 317} 318 319// assertAtLeastTwoKnownGroups verifies that at least two groups in the cut have a computed 320// impact ratio (i.e., the reference group count >= 2 so ratios are not all "n/a"). 321func assertAtLeastTwoKnownGroups(t *testing.T, rows []eeo.ImpactRow, label string) { 322 t.Helper() 323 known := 0 324 for _, row := range rows { 325 if row.ImpactRatio != "n/a" { 326 known++ 327 } 328 } 329 if known < 2 { 330 t.Errorf("%s: want at least 2 groups with a computed impact ratio, got %d (all n/a?)", label, known) 331 } 332} 333 334// assertFourFifthsConsistency checks that Flagged is true iff the impact ratio is below 0.80. 335// Rows with ImpactRatio "n/a" are skipped (unknown or singleton group). 336func assertFourFifthsConsistency(t *testing.T, rows []eeo.ImpactRow, label string) { 337 t.Helper() 338 for _, row := range rows { 339 if row.ImpactRatio == "n/a" { 340 if row.Flagged { 341 t.Errorf("%s group %s: Flagged=true but impact ratio is n/a", label, row.Group) 342 } 343 continue 344 } 345 var ratio float64 346 if _, err := fmt.Sscanf(row.ImpactRatio, "%f", &ratio); err != nil { 347 t.Errorf("%s group %s: cannot parse impact ratio %q: %v", label, row.Group, row.ImpactRatio, err) 348 continue 349 } 350 // The four fifths threshold is 0.80 (strictly less than triggers the flag). 351 wantFlagged := ratio < 0.8 352 if row.Flagged != wantFlagged { 353 t.Errorf("%s group %s: ratio=%s (%.6f), want Flagged=%v, got Flagged=%v", 354 label, row.Group, row.ImpactRatio, ratio, wantFlagged, row.Flagged) 355 } 356 } 357} 358