//go:build integration // Phase 2 Vendor Intelligence integration. Proves at runtime against a REAL Postgres and the migrated 0042 // tables what the unit tests cannot: the AI brain measures a staffing vendor's performance WITHOUT ever // rescoring a person. The vendor_performance_scorecard TAG intent computes the exact outcome ratios (fill // rate, submit to interview rate) and floors the quality band average at ten evaluated submissions; the // vendor_quality_distribution BENCH intent partitions the vendor's submitted candidates into the frozen // readiness bands and small cell suppresses each vendor's sub k band. Both are RLS scoped, and neither // recomputes a score: the frozen engine stays the only per candidate scorer. // // Run: go test -tags=integration ./internal/integration/ -run Vendor -v // Env: MM_APP_DSN, MM_MIGRATOR_DSN. package integration import ( "context" "testing" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/shopspring/decimal" "github.com/missionworkforce/missionmatch/internal/modules/ai" "github.com/missionworkforce/missionmatch/internal/platform/audit" ) // ---------- Phase 2 vendor helpers (seed via the BYPASSRLS migrator so we can write across tenants) ---------- // seedVendor inserts a staffing vendor for one org and returns its id. It uses the migrator so the seed // crosses tenants for the RLS isolation proof. func seedVendor(t *testing.T, org uuid.UUID, name, vendorType string) uuid.UUID { t.Helper() var id uuid.UUID if err := migrator.QueryRow(context.Background(), `INSERT INTO staffing_vendors (organization_id, name, vendor_type) VALUES ($1,$2,$3) RETURNING id`, org, name, vendorType).Scan(&id); err != nil { t.Fatalf("seed vendor: %v", err) } return id } // seedVendorSubmission inserts one vendor submission (org, vendor, candidate, requirement) at a lifecycle // status. submitted_at and the requirement's created_at both default to now(), so the derived time to submit // is a tiny positive value (the speed component of the score is effectively one). It uses the migrator. func seedVendorSubmission(t *testing.T, org, vendor, candidate, requirement uuid.UUID, status string) { t.Helper() if _, err := migrator.Exec(context.Background(), `INSERT INTO vendor_submissions (organization_id, vendor_id, candidate_id, requirement_id, status) VALUES ($1,$2,$3,$4,$5)`, org, vendor, candidate, requirement, status); err != nil { t.Fatalf("seed vendor submission: %v", err) } } // scorecardRow is one row of the vendor_performance_scorecard read. quality is a nullable decimal (NULL when // the vendor has fewer than ten evaluated submissions, the small cell floor). type scorecardRow struct { vendorID string vendorName string vendorType string submissions int64 fillRate decimal.Decimal interviewRate decimal.Decimal avgTimeToSubmit decimal.Decimal quality *decimal.Decimal performanceScore decimal.Decimal } // readScorecard runs the registered vendor_performance_scorecard intent SQL under the org's RLS session and // scans each row, proving every ratio scans into shopspring/decimal (never float64). It uses the SQL straight // from the closed AI Registry so the test exercises the exact reviewed read. Keyed by vendor name. func readScorecard(t *testing.T, org uuid.UUID) map[string]scorecardRow { t.Helper() intent, ok := ai.IntentByName(ai.IntentVendorPerformanceScorecard) if !ok { t.Fatal("vendor_performance_scorecard intent is not registered") } out := map[string]scorecardRow{} if err := appDB.RunInTenantTx(ctxOrg(org), func(ctx context.Context, tx pgx.Tx) error { rows, e := tx.Query(ctx, intent.SQL) if e != nil { return e } defer rows.Close() for rows.Next() { var r scorecardRow if e := rows.Scan(&r.vendorID, &r.vendorName, &r.vendorType, &r.submissions, &r.fillRate, &r.interviewRate, &r.avgTimeToSubmit, &r.quality, &r.performanceScore); e != nil { return e } out[r.vendorName] = r } return rows.Err() }); err != nil { t.Fatalf("scorecard read for org: %v", err) } return out } // vendorBandCount returns the published headcount for a (vendor, band) cell, or minus one when suppressed or // absent (so a caller asserting an exact published count fails loudly). func vendorBandCount(rows []ai.VendorQualityRow, vendor, band string) int { for _, r := range rows { if r.VendorName == vendor && r.Band == band { if r.Cell.Suppressed { return -1 } return r.Cell.Count } } return -1 } // ---------- vendor_performance_scorecard ---------- // TestVendorScorecardComputesOutcomeRatios proves the scorecard computes the exact fill and interview ratios // over a vendor's submissions and withholds the quality band average below the ten evaluation floor, RLS // scoped. It seeds one vendor with four submissions (two placed, one interviewed, one rejected) and three of // them evaluated (fewer than the floor), so fill_rate is one half, interview_rate is three quarters, and // quality is NULL. It never recomputes a score: the band it reads is the frozen engine's own decision. func TestVendorScorecardComputesOutcomeRatios(t *testing.T) { org := seedOrg(t) vendor := seedVendor(t, org, "Acme Cleared Staffing", "staffing_firm") req := seedRequirementWithLevel(t, org, "secret") // Four submissions: two placed, one interviewed, one rejected. Three of the four are evaluated green // (fewer than the ten evaluation floor), so the quality band average is withheld. statuses := []string{"placed", "placed", "interviewed", "rejected"} for i, status := range statuses { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, vendor, c.ID, req, status) if i < 3 { // three evaluated green, below the ten floor seedEvaluation(t, org, c.ID, req, "green") } } got := readScorecard(t, org) row, ok := got["Acme Cleared Staffing"] if !ok { t.Fatal("the scorecard must carry the seeded vendor") } if row.submissions != 4 { t.Fatalf("submissions = %d, want 4", row.submissions) } if !row.fillRate.Equal(decimal.RequireFromString("0.5")) { t.Fatalf("fill_rate = %s, want 0.5 (2 placed of 4)", row.fillRate) } if !row.interviewRate.Equal(decimal.RequireFromString("0.75")) { t.Fatalf("interview_rate = %s, want 0.75 (1 interviewed + 2 placed of 4)", row.interviewRate) } if row.quality != nil { t.Fatalf("quality must be withheld (NULL) below the ten evaluation floor, got %s", row.quality) } // performance_score = fill 0.5*0.35 + quality(neutral 0.5)*0.30 + speed(~1.0)*0.20 + interview 0.75*0.15 // = 0.175 + 0.15 + 0.20 + 0.1125 = 0.6375 (speed ~1.0 since submitted ~ requirement created). if row.performanceScore.Sub(decimal.RequireFromString("0.6375")).Abs().GreaterThan(decimal.RequireFromString("0.01")) { t.Fatalf("performance_score = %s, want ~0.6375", row.performanceScore) } } // TestVendorScorecardQualityFloorReleasesAtTen proves the quality band average is withheld at nine evaluated // submissions and released (published) at ten, the small cell floor. It seeds one vendor with exactly ten // green evaluated submissions (quality one) and a second vendor with nine (quality withheld), so the floor is // exercised on both sides. The quality it reports is the average of the frozen engine's own bands, never a // recomputed score. func TestVendorScorecardQualityFloorReleasesAtTen(t *testing.T) { org := seedOrg(t) atFloor := seedVendor(t, org, "Ten Eval Vendor", "subcontractor") belowFloor := seedVendor(t, org, "Nine Eval Vendor", "subcontractor") req := seedRequirementWithLevel(t, org, "secret") for i := 0; i < 10; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, atFloor, c.ID, req, "placed") seedEvaluation(t, org, c.ID, req, "green") } for i := 0; i < 9; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, belowFloor, c.ID, req, "placed") seedEvaluation(t, org, c.ID, req, "green") } got := readScorecard(t, org) at, ok := got["Ten Eval Vendor"] if !ok { t.Fatal("the scorecard must carry the ten evaluation vendor") } if at.quality == nil { t.Fatal("quality must be PUBLISHED at ten evaluations (the floor), got NULL") } if !at.quality.Equal(decimal.RequireFromString("1")) { t.Fatalf("quality = %s, want 1 (ten green evaluations)", at.quality) } below, ok := got["Nine Eval Vendor"] if !ok { t.Fatal("the scorecard must carry the nine evaluation vendor") } if below.quality != nil { t.Fatalf("quality must be withheld (NULL) at nine evaluations (below the floor), got %s", below.quality) } } // TestVendorScorecardRLSIsolation proves the scorecard is RLS scoped: a second org's vendors never enter the // first org's scorecard. Each org has a distinctly named vendor with submissions; org A's scorecard must // carry only org A's vendor. func TestVendorScorecardRLSIsolation(t *testing.T) { orgA, orgB := seedOrg(t), seedOrg(t) vendorA := seedVendor(t, orgA, "Org A Vendor", "prime") vendorB := seedVendor(t, orgB, "Org B Vendor", "prime") reqA := seedRequirementWithLevel(t, orgA, "secret") reqB := seedRequirementWithLevel(t, orgB, "secret") cA := seedCandidate(t, orgA, "secret", "active", true) seedVendorSubmission(t, orgA, vendorA, cA.ID, reqA, "placed") cB := seedCandidate(t, orgB, "secret", "active", true) seedVendorSubmission(t, orgB, vendorB, cB.ID, reqB, "placed") got := readScorecard(t, orgA) if _, ok := got["Org A Vendor"]; !ok { t.Fatal("org A scorecard must carry org A's own vendor") } if _, ok := got["Org B Vendor"]; ok { t.Fatal("ISOLATION BREACH: org A scorecard carries org B's vendor") } } // ---------- vendor_quality_distribution (bench, suppressed) ---------- // TestVendorQualityDistributionSuppressesSubK proves the distribution live under RLS, reading the already // audited candidate_evaluations band (never recomputing a score): it seeds one vendor with at least k green // submitted candidates (published) and fewer than k yellow (suppressed), then asserts the green band is // published with its exact count, the sub k yellow and the true zero red both render the sentinel, and the // complete three band partition sums to the seeded evaluated count. func TestVendorQualityDistributionSuppressesSubK(t *testing.T) { org := seedOrg(t) k := ai.ConfiguredSuppressionThreshold() vendor := seedVendor(t, org, "Suppress Vendor", "staffing_firm") req := seedRequirementWithLevel(t, org, "secret") // green: k plus 2 distinct submitted candidates evaluated green (published). yellow: 3 (sub k, suppressed). // red: 0 (a true zero, seeded by the executor, also suppressed). green := k + 2 for i := 0; i < green; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, vendor, c.ID, req, "submitted") seedEvaluation(t, org, c.ID, req, "green") } yellow := 3 for i := 0; i < yellow; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, vendor, c.ID, req, "submitted") seedEvaluation(t, org, c.ID, req, "yellow") } evaluatedTotal := green + yellow rows, err := ai.VendorQualityDistribution(ctxOrg(org), appDB, k) if err != nil { t.Fatalf("vendor quality distribution: %v", err) } // The executor seeds the complete {green, yellow, red} partition per vendor, so the seeded vendor has // exactly three cells. var vendorCells []ai.VendorQualityRow for _, r := range rows { if r.VendorName == "Suppress Vendor" { vendorCells = append(vendorCells, r) } } if len(vendorCells) != 3 { t.Fatalf("the distribution must return the green, yellow, red partition per vendor, got %d cells", len(vendorCells)) } if got := vendorBandCount(rows, "Suppress Vendor", "green"); got != green { t.Errorf("the green band of %d (at or above k=%d) must publish its exact count, got %d", green, k, got) } var hidden, partitionSum int for _, r := range vendorCells { partitionSum += r.Cell.Count if r.Cell.Suppressed { hidden++ if ai.RenderCount(r.Cell) != ai.SuppressionSentinel { t.Errorf("a suppressed band cell must render the sentinel, got %q", ai.RenderCount(r.Cell)) } } } // The sub k yellow and the true zero red are both below k, so primary suppression hides both (two cells), // and the green band publishes. if hidden != 2 { t.Errorf("the sub k yellow and the zero red must both suppress, got %d hidden cells", hidden) } if partitionSum != evaluatedTotal { t.Errorf("the three band partition counts must sum to the seeded evaluated total %d, got %d", evaluatedTotal, partitionSum) } } // TestVendorQualityDistributionRLSIsolation proves the distribution is RLS scoped: a second org's vendor // never enters the first org's distribution. Each org has a distinctly named vendor with at least k green // evaluated candidates; org A's distribution must count only org A's vendor. func TestVendorQualityDistributionRLSIsolation(t *testing.T) { orgA, orgB := seedOrg(t), seedOrg(t) k := ai.ConfiguredSuppressionThreshold() vendorA := seedVendor(t, orgA, "Iso A Vendor", "staffing_firm") vendorB := seedVendor(t, orgB, "Iso B Vendor", "staffing_firm") reqA := seedRequirementWithLevel(t, orgA, "secret") reqB := seedRequirementWithLevel(t, orgB, "secret") countA := k + 4 for i := 0; i < countA; i++ { c := seedCandidate(t, orgA, "secret", "active", true) seedVendorSubmission(t, orgA, vendorA, c.ID, reqA, "submitted") seedEvaluation(t, orgA, c.ID, reqA, "green") } countB := k + 1 for i := 0; i < countB; i++ { c := seedCandidate(t, orgB, "secret", "active", true) seedVendorSubmission(t, orgB, vendorB, c.ID, reqB, "submitted") seedEvaluation(t, orgB, c.ID, reqB, "green") } rowsA, err := ai.VendorQualityDistribution(ctxOrg(orgA), appDB, k) if err != nil { t.Fatalf("vendor quality distribution org A: %v", err) } if got := vendorBandCount(rowsA, "Iso A Vendor", "green"); got != countA { t.Fatalf("org A green count = %d, want %d (a leak of org B's vendor?)", got, countA) } for _, r := range rowsA { if r.VendorName == "Iso B Vendor" { t.Fatal("ISOLATION BREACH: org A distribution carries org B's vendor") } } } // seedEvaluationAt writes one candidate_evaluations row with an EXPLICIT evaluated_at, so a test can control // which of a candidate's evaluations is the most recent (the representative band the dedup selects). It uses // the migrator. func seedEvaluationAt(t *testing.T, org, candidate, requirement uuid.UUID, band, evaluatedAt string) { t.Helper() if _, err := migrator.Exec(context.Background(), `INSERT INTO candidate_evaluations ( organization_id, candidate_id, requirement_id, total_score, band, clearance_score, skills_score, certs_score, customer_score, experience_score, location_score, explanation, model_version, evaluated_at ) VALUES ($1,$2,$3,80,$4,100,100,100,100,100,100,'seeded for the dedup test','v1.4.0',$5) ON CONFLICT (organization_id, candidate_id, requirement_id) DO UPDATE SET band = EXCLUDED.band, evaluated_at = EXCLUDED.evaluated_at`, org, candidate, requirement, band, evaluatedAt); err != nil { t.Fatalf("seed evaluation at: %v", err) } } // TestVendorQualityDistributionDedupesOverlap proves the partition fix live: a candidate the same vendor // submitted for TWO requirements with DIFFERENT bands (an older green, a newer red) counts EXACTLY ONCE, in // the most recent band (red), never in both cells. Without the dedup the candidate would inflate two cells and // break the partition the k suppression relies on. The vendor's three bands must sum to the DISTINCT candidate // count (here one), never to the submission count (two). func TestVendorQualityDistributionDedupesOverlap(t *testing.T) { org := seedOrg(t) k := ai.ConfiguredSuppressionThreshold() vendor := seedVendor(t, org, "Overlap Vendor", "staffing_firm") reqOld := seedRequirementWithLevel(t, org, "secret") reqNew := seedRequirementWithLevel(t, org, "secret") // One candidate submitted for both requirements: an older green evaluation and a newer red evaluation. cand := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, vendor, cand.ID, reqOld, "submitted") seedVendorSubmission(t, org, vendor, cand.ID, reqNew, "submitted") seedEvaluationAt(t, org, cand.ID, reqOld, "green", "2026-01-01T00:00:00Z") seedEvaluationAt(t, org, cand.ID, reqNew, "red", "2026-06-01T00:00:00Z") rows, err := ai.VendorQualityDistribution(ctxOrg(org), appDB, k) if err != nil { t.Fatalf("vendor quality distribution: %v", err) } // The candidate must contribute to exactly one band (red, the most recent), never green, and the three // bands must sum to exactly one distinct candidate. var total int for _, r := range rows { if r.VendorName != "Overlap Vendor" { continue } total += r.Cell.Count if r.Band == "green" && r.Cell.Count != 0 { t.Errorf("the overlapping candidate must NOT count in the older green band, got %d", r.Cell.Count) } if r.Band == "red" && r.Cell.Count != 1 { t.Errorf("the overlapping candidate must count once in the most recent red band, got %d", r.Cell.Count) } } if total != 1 { t.Fatalf("the vendor's bands must sum to the DISTINCT candidate count 1 (not the submission count 2), got %d", total) } } // TestVendorScorecardQualityUsesDistinctPeopleFloor proves two properties of the fixed scorecard quality // live: (a) an unevaluated submission never dilutes the average (a vendor with ten green evaluated candidates // plus two unevaluated submissions still scores quality exactly 1.0, not a diluted value), and (b) the k floor // counts DISTINCT PEOPLE, not submissions (a vendor with nine distinct candidates each submitted twice, so // eighteen evaluated submissions, still withholds quality because only nine distinct people are behind it). func TestVendorScorecardQualityUsesDistinctPeopleFloor(t *testing.T) { org := seedOrg(t) clean := seedVendor(t, org, "Clean Ten Vendor", "staffing_firm") repeat := seedVendor(t, org, "Repeat Nine Vendor", "staffing_firm") req1 := seedRequirementWithLevel(t, org, "secret") req2 := seedRequirementWithLevel(t, org, "secret") // clean: ten DISTINCT candidates evaluated green, plus two more submitted but NOT evaluated. quality must // be exactly 1.0 (the two unevaluated contribute nothing), published at the ten distinct people floor. for i := 0; i < 10; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, clean, c.ID, req1, "placed") seedEvaluation(t, org, c.ID, req1, "green") } for i := 0; i < 2; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, clean, c.ID, req1, "submitted") // no evaluation seeded } // repeat: nine DISTINCT candidates each submitted for TWO requirements and evaluated green on both, so // eighteen evaluated submissions but only nine distinct people. The floor counts people, so quality is // withheld (NULL). for i := 0; i < 9; i++ { c := seedCandidate(t, org, "secret", "active", true) seedVendorSubmission(t, org, repeat, c.ID, req1, "placed") seedVendorSubmission(t, org, repeat, c.ID, req2, "placed") seedEvaluation(t, org, c.ID, req1, "green") seedEvaluation(t, org, c.ID, req2, "green") } got := readScorecard(t, org) cleanRow, ok := got["Clean Ten Vendor"] if !ok { t.Fatal("the scorecard must carry the clean ten vendor") } if cleanRow.quality == nil { t.Fatal("quality must be published at ten distinct evaluated candidates, got NULL") } if !cleanRow.quality.Equal(decimal.RequireFromString("1")) { t.Fatalf("quality = %s, want exactly 1 (two unevaluated submissions must NOT dilute the average)", cleanRow.quality) } repeatRow, ok := got["Repeat Nine Vendor"] if !ok { t.Fatal("the scorecard must carry the repeat nine vendor") } if repeatRow.quality != nil { t.Fatalf("quality must be withheld: nine distinct people (eighteen submissions) is below the ten PEOPLE floor, got %s", repeatRow.quality) } } // TestVendorQualityDistributionRequiresAuth proves the bench route gate: the vendor quality distribution is a // person touching bench intent, so answering it through the orchestrator WITHOUT an authenticated tenant // context is refused, never served as an unsuppressed read. This mirrors the other bench intents' auth gate. func TestVendorQualityDistributionRequiresAuth(t *testing.T) { orch := ai.NewOrchestrator(appDB, nil, audit.New(appDB)) // context.Background() carries no resolved org, so the bench fork must refuse. resp, err := orch.Answer(context.Background(), "", ai.IntentVendorQualityDistribution, nil) if err != nil { t.Fatalf("answer: %v", err) } if !resp.Refused { t.Fatalf("a vendor quality distribution with no authenticated tenant must be refused, got %+v", resp) } }