Janady13/missionmatch / internal/integration/mag_reranker_integration_test.go

1//go:build integration 2 3// MAG display order re ranker loop integration. Proves at runtime against a REAL Postgres and the migrated 0037 4// prior only re ranker what the unit tests cannot: the offline TRAINER fits a pairwise learning to rank 5// challenger from decided slate_members (approve/reject) over the frozen engine sub scores, the EVALUATOR 6// compares its held out ranking accuracy to the prior only engine order champion AND runs the four fifths no 7// adverse impact check recomputed under the challenger ranking, a human promotion flips the champion, and the 8// learned weights then REORDER a slate that the engine order and the human decisions genuinely disagree on, 9// with the frozen engine score untouched (the re ranker emits an ordering, never a candidate score, and a human 10// still approves every member). 11// 12// The seeded signal is honest: approved candidates are high location and low clearance, rejected candidates are 13// the reverse. The engine weights clearance (0.25) above location (0.10), so the engine order puts the rejected 14// candidates first; the humans approved the opposite, so a correct re ranker learns to weight location above 15// clearance and flips the order. NONE of the features is a protected attribute. 16// 17// Run: go test -tags=integration ./internal/integration/ -run MAGReranker -v 18// Env: MM_APP_DSN, MM_MIGRATOR_DSN, MM_COMPLIANCE_DSN. 19package integration 20 21import ( 22 "context" 23 "testing" 24 25 "github.com/google/uuid" 26 27 "github.com/missionworkforce/missionmatch/internal/modules/eeo" 28 "github.com/missionworkforce/missionmatch/internal/platform/magstore" 29 "github.com/missionworkforce/missionmatch/internal/platform/tenant" 30) 31 32// memberSpec is one seeded slate member: its approval and its six frozen sub score features in RerankFeatureOrder 33// (clearance, skills, certs, customer, experience, location). 34type memberSpec struct { 35 approved bool 36 features [6]float64 37} 38 39// seedDecidedSlate seeds one slate with decided members carrying the six frozen sub scores and the approve or 40// reject decision, via the BYPASSRLS migrator (cross tenant seed). total_score and band are the engine weighted 41// sum of the sub scores, so the prior only re ranker (the engine weights) orders by total exactly. 42func seedDecidedSlate(t *testing.T, org uuid.UUID, specs []memberSpec) { 43 t.Helper() 44 ctx := context.Background() 45 req := seedRequirementWithAgency(t, org, "RERANK "+uuid.NewString()[:8]) 46 slateID := uuid.New() 47 if _, err := migrator.Exec(ctx, 48 `INSERT INTO slates (id, organization_id, requirement_id, name) VALUES ($1,$2,$3,$4)`, 49 slateID, org, req, "reranker seed slate"); err != nil { 50 t.Fatalf("seed slate: %v", err) 51 } 52 for _, s := range specs { 53 cand := seedCandidate(t, org, "secret", "active", true) 54 total := 0.25*s.features[0] + 0.25*s.features[1] + 0.15*s.features[2] + 0.15*s.features[3] + 0.10*s.features[4] + 0.10*s.features[5] 55 band := "red" 56 if total >= 90 { 57 band = "green" 58 } else if total >= 75 { 59 band = "yellow" 60 } 61 decision := "rejected" 62 if s.approved { 63 decision = "approved" 64 } 65 if _, err := migrator.Exec(ctx, 66 `INSERT INTO slate_members 67 (organization_id, slate_id, candidate_id, total_score, band, 68 clearance_score, skills_score, certs_score, customer_score, experience_score, location_score, 69 explanation, trace, decision, decided_at) 70 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'seed','[]',$12, now())`, 71 org, slateID, cand.ID, total, band, 72 s.features[0], s.features[1], s.features[2], s.features[3], s.features[4], s.features[5], 73 decision); err != nil { 74 t.Fatalf("seed slate member: %v", err) 75 } 76 } 77} 78 79// TestMAGRerankerTrainsEvaluatesPromotesAndReorders drives the whole re ranker loop end to end on staging. 80func TestMAGRerankerTrainsEvaluatesPromotesAndReorders(t *testing.T) { 81 modelKey := magstore.RerankModelKey 82 const priorOnlyVersion = 1 83 ctx := context.Background() 84 85 // Force the champion back to the prior only engine order v1 so the before and after comparison is 86 // deterministic, and restore it on cleanup so it never leaks a learned re ranker into other tests. 87 if _, err := magstore.PromoteChampion(ctx, migrator, modelKey, priorOnlyVersion); err != nil { 88 t.Fatalf("reset reranker champion to v%d: %v", priorOnlyVersion, err) 89 } 90 t.Cleanup(func() { 91 if _, cerr := magstore.PromoteChampion(ctx, migrator, modelKey, priorOnlyVersion); cerr != nil { 92 t.Logf("cleanup: could not restore prior only reranker champion v%d: %v", priorOnlyVersion, cerr) 93 } 94 }) 95 96 // Seed slates with the honest signal: approved are high location (100) low clearance (0), rejected are the 97 // reverse; the middle four sub scores are fixed at 50. The engine weights clearance above location, so the 98 // engine order puts rejected first; a correct re ranker flips it. 99 approved := [6]float64{0, 50, 50, 50, 50, 100} // clearance 0, location 100 100 rejected := [6]float64{100, 50, 50, 50, 50, 0} // clearance 100, location 0 101 org := seedOrg(t) 102 for i := 0; i < 10; i++ { 103 seedDecidedSlate(t, org, []memberSpec{ 104 {approved: true, features: approved}, 105 {approved: true, features: approved}, 106 {approved: false, features: rejected}, 107 {approved: false, features: rejected}, 108 }) 109 } 110 111 // The prior only champion (the engine weights) orders a test slate by total: the rejected profile (higher 112 // total, because clearance outweighs location) ranks ABOVE the approved profile. 113 champV1, err := magstore.LoadModel(ctx, migrator, modelKey, priorOnlyVersion) 114 if err != nil { 115 t.Fatalf("load prior only champion: %v", err) 116 } 117 v1Coeffs := magstore.RerankCoeffs(champV1) 118 testSlate := []magstore.SlateMember{ 119 {CandidateID: "approved", Features: approved[:], Approved: true}, 120 {CandidateID: "rejected", Features: rejected[:], Approved: false}, 121 } 122 engineOrder := magstore.OrderMembers(v1Coeffs, testSlate) 123 if engineOrder[0].CandidateID != "rejected" { 124 t.Fatalf("the engine order must rank the higher total (rejected profile) first, got %s", engineOrder[0].CandidateID) 125 } 126 127 // TRAIN: fit a pairwise re ranker challenger. min-pairs 20 so the seeded pairs clear the floor. 128 res, err := magstore.TrainReranker(ctx, migrator, modelKey, 1.0, 20, false) 129 if err != nil { 130 t.Fatalf("train reranker: %v", err) 131 } 132 if res.Skipped { 133 t.Fatalf("the fit must not be skipped with %d pairs: %s", res.Pairs, res.SkipReason) 134 } 135 if res.Version <= priorOnlyVersion { 136 t.Fatalf("the challenger version %d must be newer than the prior champion", res.Version) 137 } 138 // The learned location weight (index 5) must exceed the learned clearance weight (index 0): the human signal 139 // preferred location over clearance, the opposite of the engine. 140 if res.Coeffs[5] <= res.Coeffs[0] { 141 t.Fatalf("the learned re ranker must weight location above clearance, got location %.4f clearance %.4f", res.Coeffs[5], res.Coeffs[0]) 142 } 143 t.Logf("challenger v%d learned weights: clearance %.4f location %.4f over %d pairs", res.Version, res.Coeffs[0], res.Coeffs[5], res.Pairs) 144 145 // The trainer must NOT flip champion. 146 stillV1, err := magstore.ChampionVersion(ctx, migrator, modelKey) 147 if err != nil { 148 t.Fatalf("champion after train: %v", err) 149 } 150 if stillV1 != priorOnlyVersion { 151 t.Fatalf("the trainer must not flip champion, moved to v%d", stillV1) 152 } 153 154 // EVALUATE ranking quality: the challenger must order approved above rejected far better than the engine 155 // order champion (which orders them backwards, accuracy near 0). 156 chal, err := magstore.LoadModel(ctx, migrator, modelKey, res.Version) 157 if err != nil { 158 t.Fatalf("load challenger: %v", err) 159 } 160 quality, err := magstore.EvaluateRerankerQuality(ctx, migrator, v1Coeffs, magstore.RerankCoeffs(chal), 0.30) 161 if err != nil { 162 t.Fatalf("evaluate reranker: %v", err) 163 } 164 // The learned challenger must clear the ranking quality gate (at least as good as the engine order within 165 // tolerance). Aggregate held out accuracy is dominated by the large pooled slate history on staging, so the 166 // robust proofs of learning are the weight direction (above) and the reorder (below), not the aggregate 167 // delta; the gate guards against a genuine regression. 168 t.Logf("held out %d slates, pairwise accuracy champion %.4f vs challenger %.4f", quality.HeldOutGroups, quality.ChampAccuracy, quality.ChalAccuracy) 169 if !quality.Pass { 170 t.Errorf("the learned challenger must clear the ranking quality gate, got %+v", quality) 171 } 172 173 // The FAIRNESS gate must be callable through the compliance pool. With no seeded demographics it recomputes 174 // clean (no group flagged), exercising the ranking recomputation path end to end. 175 if compliance != nil { 176 groups, gerr := magstore.ReadSlateGroups(ctx, migrator) 177 if gerr != nil { 178 t.Fatalf("read slate groups: %v", gerr) 179 } 180 orgGroups := magstore.GroupsByOrg(groups)[org.String()] 181 sel := magstore.CandidateSelection(magstore.RerankCoeffs(chal), orgGroups) 182 svc := eeo.NewService(tenant.New(compliance)) 183 section, ferr := svc.AdverseImpactForSelection(tenant.WithOrg(ctx, org.String()), sel) 184 if ferr != nil { 185 t.Fatalf("the ranking fairness recomputation must be callable: %v", ferr) 186 } 187 if r := section.MinImpactRatio(); r < 0.80 { 188 t.Errorf("with no seeded demographics the recomputation must be clean, got min ratio %v", r) 189 } 190 } 191 192 // PROMOTE: the human governance action flips the champion. 193 from, err := magstore.PromoteChampion(ctx, migrator, modelKey, res.Version) 194 if err != nil { 195 t.Fatalf("promote reranker: %v", err) 196 } 197 if from != priorOnlyVersion { 198 t.Errorf("the promotion must replace the prior champion v%d, replaced v%d", priorOnlyVersion, from) 199 } 200 201 // REORDER: the learned champion now orders the SAME test slate the other way, the approved profile first, 202 // proving the display order moved off the engine order toward the human signal. 203 champLearned, err := magstore.LoadModel(ctx, migrator, modelKey, res.Version) 204 if err != nil { 205 t.Fatalf("load learned champion: %v", err) 206 } 207 learnedOrder := magstore.OrderMembers(magstore.RerankCoeffs(champLearned), testSlate) 208 if learnedOrder[0].CandidateID != "approved" { 209 t.Fatalf("the learned re ranker must reorder the approved profile first, got %s", learnedOrder[0].CandidateID) 210 } 211 if learnedOrder[0].CandidateID == engineOrder[0].CandidateID { 212 t.Fatalf("the display order must change from the engine order, both put %s first", engineOrder[0].CandidateID) 213 } 214 t.Logf("engine order first=%s, learned order first=%s (the re ranker flipped the display order)", engineOrder[0].CandidateID, learnedOrder[0].CandidateID) 215} 216