Janady13/missionmatch / internal/integration/vendor_integration_test.go

1//go:build integration 2 3// Phase 2 Vendor Intelligence integration. Proves at runtime against a REAL Postgres and the migrated 0042 4// tables what the unit tests cannot: the AI brain measures a staffing vendor's performance WITHOUT ever 5// rescoring a person. The vendor_performance_scorecard TAG intent computes the exact outcome ratios (fill 6// rate, submit to interview rate) and floors the quality band average at ten evaluated submissions; the 7// vendor_quality_distribution BENCH intent partitions the vendor's submitted candidates into the frozen 8// readiness bands and small cell suppresses each vendor's sub k band. Both are RLS scoped, and neither 9// recomputes a score: the frozen engine stays the only per candidate scorer. 10// 11// Run: go test -tags=integration ./internal/integration/ -run Vendor -v 12// Env: MM_APP_DSN, MM_MIGRATOR_DSN. 13package integration 14 15import ( 16 "context" 17 "testing" 18 19 "github.com/google/uuid" 20 "github.com/jackc/pgx/v5" 21 "github.com/shopspring/decimal" 22 23 "github.com/missionworkforce/missionmatch/internal/modules/ai" 24 "github.com/missionworkforce/missionmatch/internal/platform/audit" 25) 26 27// ---------- Phase 2 vendor helpers (seed via the BYPASSRLS migrator so we can write across tenants) ---------- 28 29// seedVendor inserts a staffing vendor for one org and returns its id. It uses the migrator so the seed 30// crosses tenants for the RLS isolation proof. 31func seedVendor(t *testing.T, org uuid.UUID, name, vendorType string) uuid.UUID { 32 t.Helper() 33 var id uuid.UUID 34 if err := migrator.QueryRow(context.Background(), 35 `INSERT INTO staffing_vendors (organization_id, name, vendor_type) VALUES ($1,$2,$3) RETURNING id`, 36 org, name, vendorType).Scan(&id); err != nil { 37 t.Fatalf("seed vendor: %v", err) 38 } 39 return id 40} 41 42// seedVendorSubmission inserts one vendor submission (org, vendor, candidate, requirement) at a lifecycle 43// status. submitted_at and the requirement's created_at both default to now(), so the derived time to submit 44// is a tiny positive value (the speed component of the score is effectively one). It uses the migrator. 45func seedVendorSubmission(t *testing.T, org, vendor, candidate, requirement uuid.UUID, status string) { 46 t.Helper() 47 if _, err := migrator.Exec(context.Background(), 48 `INSERT INTO vendor_submissions (organization_id, vendor_id, candidate_id, requirement_id, status) 49 VALUES ($1,$2,$3,$4,$5)`, 50 org, vendor, candidate, requirement, status); err != nil { 51 t.Fatalf("seed vendor submission: %v", err) 52 } 53} 54 55// scorecardRow is one row of the vendor_performance_scorecard read. quality is a nullable decimal (NULL when 56// the vendor has fewer than ten evaluated submissions, the small cell floor). 57type scorecardRow struct { 58 vendorID string 59 vendorName string 60 vendorType string 61 submissions int64 62 fillRate decimal.Decimal 63 interviewRate decimal.Decimal 64 avgTimeToSubmit decimal.Decimal 65 quality *decimal.Decimal 66 performanceScore decimal.Decimal 67} 68 69// readScorecard runs the registered vendor_performance_scorecard intent SQL under the org's RLS session and 70// scans each row, proving every ratio scans into shopspring/decimal (never float64). It uses the SQL straight 71// from the closed AI Registry so the test exercises the exact reviewed read. Keyed by vendor name. 72func readScorecard(t *testing.T, org uuid.UUID) map[string]scorecardRow { 73 t.Helper() 74 intent, ok := ai.IntentByName(ai.IntentVendorPerformanceScorecard) 75 if !ok { 76 t.Fatal("vendor_performance_scorecard intent is not registered") 77 } 78 out := map[string]scorecardRow{} 79 if err := appDB.RunInTenantTx(ctxOrg(org), func(ctx context.Context, tx pgx.Tx) error { 80 rows, e := tx.Query(ctx, intent.SQL) 81 if e != nil { 82 return e 83 } 84 defer rows.Close() 85 for rows.Next() { 86 var r scorecardRow 87 if e := rows.Scan(&r.vendorID, &r.vendorName, &r.vendorType, &r.submissions, 88 &r.fillRate, &r.interviewRate, &r.avgTimeToSubmit, &r.quality, &r.performanceScore); e != nil { 89 return e 90 } 91 out[r.vendorName] = r 92 } 93 return rows.Err() 94 }); err != nil { 95 t.Fatalf("scorecard read for org: %v", err) 96 } 97 return out 98} 99 100// vendorBandCount returns the published headcount for a (vendor, band) cell, or minus one when suppressed or 101// absent (so a caller asserting an exact published count fails loudly). 102func vendorBandCount(rows []ai.VendorQualityRow, vendor, band string) int { 103 for _, r := range rows { 104 if r.VendorName == vendor && r.Band == band { 105 if r.Cell.Suppressed { 106 return -1 107 } 108 return r.Cell.Count 109 } 110 } 111 return -1 112} 113 114// ---------- vendor_performance_scorecard ---------- 115 116// TestVendorScorecardComputesOutcomeRatios proves the scorecard computes the exact fill and interview ratios 117// over a vendor's submissions and withholds the quality band average below the ten evaluation floor, RLS 118// scoped. It seeds one vendor with four submissions (two placed, one interviewed, one rejected) and three of 119// them evaluated (fewer than the floor), so fill_rate is one half, interview_rate is three quarters, and 120// quality is NULL. It never recomputes a score: the band it reads is the frozen engine's own decision. 121func TestVendorScorecardComputesOutcomeRatios(t *testing.T) { 122 org := seedOrg(t) 123 vendor := seedVendor(t, org, "Acme Cleared Staffing", "staffing_firm") 124 req := seedRequirementWithLevel(t, org, "secret") 125 126 // Four submissions: two placed, one interviewed, one rejected. Three of the four are evaluated green 127 // (fewer than the ten evaluation floor), so the quality band average is withheld. 128 statuses := []string{"placed", "placed", "interviewed", "rejected"} 129 for i, status := range statuses { 130 c := seedCandidate(t, org, "secret", "active", true) 131 seedVendorSubmission(t, org, vendor, c.ID, req, status) 132 if i < 3 { // three evaluated green, below the ten floor 133 seedEvaluation(t, org, c.ID, req, "green") 134 } 135 } 136 137 got := readScorecard(t, org) 138 row, ok := got["Acme Cleared Staffing"] 139 if !ok { 140 t.Fatal("the scorecard must carry the seeded vendor") 141 } 142 if row.submissions != 4 { 143 t.Fatalf("submissions = %d, want 4", row.submissions) 144 } 145 if !row.fillRate.Equal(decimal.RequireFromString("0.5")) { 146 t.Fatalf("fill_rate = %s, want 0.5 (2 placed of 4)", row.fillRate) 147 } 148 if !row.interviewRate.Equal(decimal.RequireFromString("0.75")) { 149 t.Fatalf("interview_rate = %s, want 0.75 (1 interviewed + 2 placed of 4)", row.interviewRate) 150 } 151 if row.quality != nil { 152 t.Fatalf("quality must be withheld (NULL) below the ten evaluation floor, got %s", row.quality) 153 } 154 // performance_score = fill 0.5*0.35 + quality(neutral 0.5)*0.30 + speed(~1.0)*0.20 + interview 0.75*0.15 155 // = 0.175 + 0.15 + 0.20 + 0.1125 = 0.6375 (speed ~1.0 since submitted ~ requirement created). 156 if row.performanceScore.Sub(decimal.RequireFromString("0.6375")).Abs().GreaterThan(decimal.RequireFromString("0.01")) { 157 t.Fatalf("performance_score = %s, want ~0.6375", row.performanceScore) 158 } 159} 160 161// TestVendorScorecardQualityFloorReleasesAtTen proves the quality band average is withheld at nine evaluated 162// submissions and released (published) at ten, the small cell floor. It seeds one vendor with exactly ten 163// green evaluated submissions (quality one) and a second vendor with nine (quality withheld), so the floor is 164// exercised on both sides. The quality it reports is the average of the frozen engine's own bands, never a 165// recomputed score. 166func TestVendorScorecardQualityFloorReleasesAtTen(t *testing.T) { 167 org := seedOrg(t) 168 atFloor := seedVendor(t, org, "Ten Eval Vendor", "subcontractor") 169 belowFloor := seedVendor(t, org, "Nine Eval Vendor", "subcontractor") 170 req := seedRequirementWithLevel(t, org, "secret") 171 172 for i := 0; i < 10; i++ { 173 c := seedCandidate(t, org, "secret", "active", true) 174 seedVendorSubmission(t, org, atFloor, c.ID, req, "placed") 175 seedEvaluation(t, org, c.ID, req, "green") 176 } 177 for i := 0; i < 9; i++ { 178 c := seedCandidate(t, org, "secret", "active", true) 179 seedVendorSubmission(t, org, belowFloor, c.ID, req, "placed") 180 seedEvaluation(t, org, c.ID, req, "green") 181 } 182 183 got := readScorecard(t, org) 184 185 at, ok := got["Ten Eval Vendor"] 186 if !ok { 187 t.Fatal("the scorecard must carry the ten evaluation vendor") 188 } 189 if at.quality == nil { 190 t.Fatal("quality must be PUBLISHED at ten evaluations (the floor), got NULL") 191 } 192 if !at.quality.Equal(decimal.RequireFromString("1")) { 193 t.Fatalf("quality = %s, want 1 (ten green evaluations)", at.quality) 194 } 195 196 below, ok := got["Nine Eval Vendor"] 197 if !ok { 198 t.Fatal("the scorecard must carry the nine evaluation vendor") 199 } 200 if below.quality != nil { 201 t.Fatalf("quality must be withheld (NULL) at nine evaluations (below the floor), got %s", below.quality) 202 } 203} 204 205// TestVendorScorecardRLSIsolation proves the scorecard is RLS scoped: a second org's vendors never enter the 206// first org's scorecard. Each org has a distinctly named vendor with submissions; org A's scorecard must 207// carry only org A's vendor. 208func TestVendorScorecardRLSIsolation(t *testing.T) { 209 orgA, orgB := seedOrg(t), seedOrg(t) 210 vendorA := seedVendor(t, orgA, "Org A Vendor", "prime") 211 vendorB := seedVendor(t, orgB, "Org B Vendor", "prime") 212 reqA := seedRequirementWithLevel(t, orgA, "secret") 213 reqB := seedRequirementWithLevel(t, orgB, "secret") 214 215 cA := seedCandidate(t, orgA, "secret", "active", true) 216 seedVendorSubmission(t, orgA, vendorA, cA.ID, reqA, "placed") 217 cB := seedCandidate(t, orgB, "secret", "active", true) 218 seedVendorSubmission(t, orgB, vendorB, cB.ID, reqB, "placed") 219 220 got := readScorecard(t, orgA) 221 if _, ok := got["Org A Vendor"]; !ok { 222 t.Fatal("org A scorecard must carry org A's own vendor") 223 } 224 if _, ok := got["Org B Vendor"]; ok { 225 t.Fatal("ISOLATION BREACH: org A scorecard carries org B's vendor") 226 } 227} 228 229// ---------- vendor_quality_distribution (bench, suppressed) ---------- 230 231// TestVendorQualityDistributionSuppressesSubK proves the distribution live under RLS, reading the already 232// audited candidate_evaluations band (never recomputing a score): it seeds one vendor with at least k green 233// submitted candidates (published) and fewer than k yellow (suppressed), then asserts the green band is 234// published with its exact count, the sub k yellow and the true zero red both render the sentinel, and the 235// complete three band partition sums to the seeded evaluated count. 236func TestVendorQualityDistributionSuppressesSubK(t *testing.T) { 237 org := seedOrg(t) 238 k := ai.ConfiguredSuppressionThreshold() 239 vendor := seedVendor(t, org, "Suppress Vendor", "staffing_firm") 240 req := seedRequirementWithLevel(t, org, "secret") 241 242 // green: k plus 2 distinct submitted candidates evaluated green (published). yellow: 3 (sub k, suppressed). 243 // red: 0 (a true zero, seeded by the executor, also suppressed). 244 green := k + 2 245 for i := 0; i < green; i++ { 246 c := seedCandidate(t, org, "secret", "active", true) 247 seedVendorSubmission(t, org, vendor, c.ID, req, "submitted") 248 seedEvaluation(t, org, c.ID, req, "green") 249 } 250 yellow := 3 251 for i := 0; i < yellow; i++ { 252 c := seedCandidate(t, org, "secret", "active", true) 253 seedVendorSubmission(t, org, vendor, c.ID, req, "submitted") 254 seedEvaluation(t, org, c.ID, req, "yellow") 255 } 256 evaluatedTotal := green + yellow 257 258 rows, err := ai.VendorQualityDistribution(ctxOrg(org), appDB, k) 259 if err != nil { 260 t.Fatalf("vendor quality distribution: %v", err) 261 } 262 // The executor seeds the complete {green, yellow, red} partition per vendor, so the seeded vendor has 263 // exactly three cells. 264 var vendorCells []ai.VendorQualityRow 265 for _, r := range rows { 266 if r.VendorName == "Suppress Vendor" { 267 vendorCells = append(vendorCells, r) 268 } 269 } 270 if len(vendorCells) != 3 { 271 t.Fatalf("the distribution must return the green, yellow, red partition per vendor, got %d cells", len(vendorCells)) 272 } 273 274 if got := vendorBandCount(rows, "Suppress Vendor", "green"); got != green { 275 t.Errorf("the green band of %d (at or above k=%d) must publish its exact count, got %d", green, k, got) 276 } 277 var hidden, partitionSum int 278 for _, r := range vendorCells { 279 partitionSum += r.Cell.Count 280 if r.Cell.Suppressed { 281 hidden++ 282 if ai.RenderCount(r.Cell) != ai.SuppressionSentinel { 283 t.Errorf("a suppressed band cell must render the sentinel, got %q", ai.RenderCount(r.Cell)) 284 } 285 } 286 } 287 // The sub k yellow and the true zero red are both below k, so primary suppression hides both (two cells), 288 // and the green band publishes. 289 if hidden != 2 { 290 t.Errorf("the sub k yellow and the zero red must both suppress, got %d hidden cells", hidden) 291 } 292 if partitionSum != evaluatedTotal { 293 t.Errorf("the three band partition counts must sum to the seeded evaluated total %d, got %d", evaluatedTotal, partitionSum) 294 } 295} 296 297// TestVendorQualityDistributionRLSIsolation proves the distribution is RLS scoped: a second org's vendor 298// never enters the first org's distribution. Each org has a distinctly named vendor with at least k green 299// evaluated candidates; org A's distribution must count only org A's vendor. 300func TestVendorQualityDistributionRLSIsolation(t *testing.T) { 301 orgA, orgB := seedOrg(t), seedOrg(t) 302 k := ai.ConfiguredSuppressionThreshold() 303 vendorA := seedVendor(t, orgA, "Iso A Vendor", "staffing_firm") 304 vendorB := seedVendor(t, orgB, "Iso B Vendor", "staffing_firm") 305 reqA := seedRequirementWithLevel(t, orgA, "secret") 306 reqB := seedRequirementWithLevel(t, orgB, "secret") 307 308 countA := k + 4 309 for i := 0; i < countA; i++ { 310 c := seedCandidate(t, orgA, "secret", "active", true) 311 seedVendorSubmission(t, orgA, vendorA, c.ID, reqA, "submitted") 312 seedEvaluation(t, orgA, c.ID, reqA, "green") 313 } 314 countB := k + 1 315 for i := 0; i < countB; i++ { 316 c := seedCandidate(t, orgB, "secret", "active", true) 317 seedVendorSubmission(t, orgB, vendorB, c.ID, reqB, "submitted") 318 seedEvaluation(t, orgB, c.ID, reqB, "green") 319 } 320 321 rowsA, err := ai.VendorQualityDistribution(ctxOrg(orgA), appDB, k) 322 if err != nil { 323 t.Fatalf("vendor quality distribution org A: %v", err) 324 } 325 if got := vendorBandCount(rowsA, "Iso A Vendor", "green"); got != countA { 326 t.Fatalf("org A green count = %d, want %d (a leak of org B's vendor?)", got, countA) 327 } 328 for _, r := range rowsA { 329 if r.VendorName == "Iso B Vendor" { 330 t.Fatal("ISOLATION BREACH: org A distribution carries org B's vendor") 331 } 332 } 333} 334 335// seedEvaluationAt writes one candidate_evaluations row with an EXPLICIT evaluated_at, so a test can control 336// which of a candidate's evaluations is the most recent (the representative band the dedup selects). It uses 337// the migrator. 338func seedEvaluationAt(t *testing.T, org, candidate, requirement uuid.UUID, band, evaluatedAt string) { 339 t.Helper() 340 if _, err := migrator.Exec(context.Background(), 341 `INSERT INTO candidate_evaluations ( 342 organization_id, candidate_id, requirement_id, total_score, band, 343 clearance_score, skills_score, certs_score, customer_score, experience_score, location_score, 344 explanation, model_version, evaluated_at 345 ) VALUES ($1,$2,$3,80,$4,100,100,100,100,100,100,'seeded for the dedup test','v1.4.0',$5) 346 ON CONFLICT (organization_id, candidate_id, requirement_id) DO UPDATE SET band = EXCLUDED.band, evaluated_at = EXCLUDED.evaluated_at`, 347 org, candidate, requirement, band, evaluatedAt); err != nil { 348 t.Fatalf("seed evaluation at: %v", err) 349 } 350} 351 352// TestVendorQualityDistributionDedupesOverlap proves the partition fix live: a candidate the same vendor 353// submitted for TWO requirements with DIFFERENT bands (an older green, a newer red) counts EXACTLY ONCE, in 354// the most recent band (red), never in both cells. Without the dedup the candidate would inflate two cells and 355// break the partition the k suppression relies on. The vendor's three bands must sum to the DISTINCT candidate 356// count (here one), never to the submission count (two). 357func TestVendorQualityDistributionDedupesOverlap(t *testing.T) { 358 org := seedOrg(t) 359 k := ai.ConfiguredSuppressionThreshold() 360 vendor := seedVendor(t, org, "Overlap Vendor", "staffing_firm") 361 reqOld := seedRequirementWithLevel(t, org, "secret") 362 reqNew := seedRequirementWithLevel(t, org, "secret") 363 364 // One candidate submitted for both requirements: an older green evaluation and a newer red evaluation. 365 cand := seedCandidate(t, org, "secret", "active", true) 366 seedVendorSubmission(t, org, vendor, cand.ID, reqOld, "submitted") 367 seedVendorSubmission(t, org, vendor, cand.ID, reqNew, "submitted") 368 seedEvaluationAt(t, org, cand.ID, reqOld, "green", "2026-01-01T00:00:00Z") 369 seedEvaluationAt(t, org, cand.ID, reqNew, "red", "2026-06-01T00:00:00Z") 370 371 rows, err := ai.VendorQualityDistribution(ctxOrg(org), appDB, k) 372 if err != nil { 373 t.Fatalf("vendor quality distribution: %v", err) 374 } 375 376 // The candidate must contribute to exactly one band (red, the most recent), never green, and the three 377 // bands must sum to exactly one distinct candidate. 378 var total int 379 for _, r := range rows { 380 if r.VendorName != "Overlap Vendor" { 381 continue 382 } 383 total += r.Cell.Count 384 if r.Band == "green" && r.Cell.Count != 0 { 385 t.Errorf("the overlapping candidate must NOT count in the older green band, got %d", r.Cell.Count) 386 } 387 if r.Band == "red" && r.Cell.Count != 1 { 388 t.Errorf("the overlapping candidate must count once in the most recent red band, got %d", r.Cell.Count) 389 } 390 } 391 if total != 1 { 392 t.Fatalf("the vendor's bands must sum to the DISTINCT candidate count 1 (not the submission count 2), got %d", total) 393 } 394} 395 396// TestVendorScorecardQualityUsesDistinctPeopleFloor proves two properties of the fixed scorecard quality 397// live: (a) an unevaluated submission never dilutes the average (a vendor with ten green evaluated candidates 398// plus two unevaluated submissions still scores quality exactly 1.0, not a diluted value), and (b) the k floor 399// counts DISTINCT PEOPLE, not submissions (a vendor with nine distinct candidates each submitted twice, so 400// eighteen evaluated submissions, still withholds quality because only nine distinct people are behind it). 401func TestVendorScorecardQualityUsesDistinctPeopleFloor(t *testing.T) { 402 org := seedOrg(t) 403 clean := seedVendor(t, org, "Clean Ten Vendor", "staffing_firm") 404 repeat := seedVendor(t, org, "Repeat Nine Vendor", "staffing_firm") 405 req1 := seedRequirementWithLevel(t, org, "secret") 406 req2 := seedRequirementWithLevel(t, org, "secret") 407 408 // clean: ten DISTINCT candidates evaluated green, plus two more submitted but NOT evaluated. quality must 409 // be exactly 1.0 (the two unevaluated contribute nothing), published at the ten distinct people floor. 410 for i := 0; i < 10; i++ { 411 c := seedCandidate(t, org, "secret", "active", true) 412 seedVendorSubmission(t, org, clean, c.ID, req1, "placed") 413 seedEvaluation(t, org, c.ID, req1, "green") 414 } 415 for i := 0; i < 2; i++ { 416 c := seedCandidate(t, org, "secret", "active", true) 417 seedVendorSubmission(t, org, clean, c.ID, req1, "submitted") // no evaluation seeded 418 } 419 420 // repeat: nine DISTINCT candidates each submitted for TWO requirements and evaluated green on both, so 421 // eighteen evaluated submissions but only nine distinct people. The floor counts people, so quality is 422 // withheld (NULL). 423 for i := 0; i < 9; i++ { 424 c := seedCandidate(t, org, "secret", "active", true) 425 seedVendorSubmission(t, org, repeat, c.ID, req1, "placed") 426 seedVendorSubmission(t, org, repeat, c.ID, req2, "placed") 427 seedEvaluation(t, org, c.ID, req1, "green") 428 seedEvaluation(t, org, c.ID, req2, "green") 429 } 430 431 got := readScorecard(t, org) 432 433 cleanRow, ok := got["Clean Ten Vendor"] 434 if !ok { 435 t.Fatal("the scorecard must carry the clean ten vendor") 436 } 437 if cleanRow.quality == nil { 438 t.Fatal("quality must be published at ten distinct evaluated candidates, got NULL") 439 } 440 if !cleanRow.quality.Equal(decimal.RequireFromString("1")) { 441 t.Fatalf("quality = %s, want exactly 1 (two unevaluated submissions must NOT dilute the average)", cleanRow.quality) 442 } 443 444 repeatRow, ok := got["Repeat Nine Vendor"] 445 if !ok { 446 t.Fatal("the scorecard must carry the repeat nine vendor") 447 } 448 if repeatRow.quality != nil { 449 t.Fatalf("quality must be withheld: nine distinct people (eighteen submissions) is below the ten PEOPLE floor, got %s", repeatRow.quality) 450 } 451} 452 453// TestVendorQualityDistributionRequiresAuth proves the bench route gate: the vendor quality distribution is a 454// person touching bench intent, so answering it through the orchestrator WITHOUT an authenticated tenant 455// context is refused, never served as an unsuppressed read. This mirrors the other bench intents' auth gate. 456func TestVendorQualityDistributionRequiresAuth(t *testing.T) { 457 orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) 458 // context.Background() carries no resolved org, so the bench fork must refuse. 459 resp, err := orch.Answer(context.Background(), "", ai.IntentVendorQualityDistribution, nil) 460 if err != nil { 461 t.Fatalf("answer: %v", err) 462 } 463 if !resp.Refused { 464 t.Fatalf("a vendor quality distribution with no authenticated tenant must be refused, got %+v", resp) 465 } 466} 467