Janady13/missionmatch / internal/platform/magstore/rerank.go

1package magstore 2 3import ( 4 "context" 5 "fmt" 6 "sort" 7 "strings" 8 9 "github.com/jackc/pgx/v5/pgxpool" 10 11 "github.com/missionworkforce/missionmatch/internal/platform/learn" 12) 13 14// RankTolerance is the slack the ranking quality gate allows so a tiny held out set or a floating point tie does 15// not fail a genuinely equal challenger. The challenger passes when its pairwise accuracy is within this of the 16// champion (better or equal). 17const RankTolerance = 0.02 18 19// RerankModelKey is the model_key of the learned slate display order re ranker (migration 0037 seeds the prior 20// only v1 whose coefficients are the frozen engine component weights, so it reproduces the engine order until a 21// challenger is fitted and promoted). 22const RerankModelKey = "slate_display_order" 23 24// RerankFeatureOrder is the FIXED feature order the display order re ranker trains and serves on, the frozen 25// engine's six sub scores. Training, ordering, and the coefficient store all use this exact order so a fitted 26// coefficient pairs with its sub score. NONE is a protected attribute or a proxy. 27var RerankFeatureOrder = []string{"clearance_score", "skills_score", "certs_score", "customer_score", "experience_score", "location_score"} 28 29// SlateMember is one decided slate member for the re ranker: its slate and candidate ids, its six frozen engine 30// sub score features (aligned to RerankFeatureOrder), and whether the human approved it. The ids carry NO 31// demographic data; they are the only thing that crosses to the eeo compliance path for the fairness gate. 32type SlateMember struct { 33 SlateID string 34 CandidateID string 35 Features []float64 36 Approved bool 37} 38 39// SlateGroup is one slate's decided members, the ranking group the pairwise re ranker learns within. OrgID 40// carries the owning tenant so the fairness gate can recompute the four fifths ratio per org (a tenant scoped 41// EEO measure) under a challenger ranking. 42type SlateGroup struct { 43 SlateID string 44 OrgID string 45 Members []SlateMember 46} 47 48// ReadSlateGroups reads every decided slate member (approved or rejected) POOLED across tenants, grouped by 49// slate, with the six frozen engine sub scores as float features. It reads ONLY the frozen sub scores and the 50// decision, never a demographic attribute (the eeo schema is unreachable from here); the demographic join for 51// the fairness gate happens separately through the compliance pool keyed only by candidate id. Ordered 52// deterministically by slate then candidate. 53func ReadSlateGroups(ctx context.Context, pool *pgxpool.Pool) ([]SlateGroup, error) { 54 const q = ` 55SELECT slate_id::text, organization_id::text, candidate_id::text, decision, 56 clearance_score::float8, skills_score::float8, certs_score::float8, 57 customer_score::float8, experience_score::float8, location_score::float8 58 FROM slate_members 59 WHERE decision IN ('approved', 'rejected') 60 ORDER BY slate_id, candidate_id` 61 rows, err := pool.Query(ctx, q) 62 if err != nil { 63 return nil, err 64 } 65 defer rows.Close() 66 var groups []SlateGroup 67 var cur *SlateGroup 68 for rows.Next() { 69 var slateID, orgID, candidateID, decision string 70 f := make([]float64, len(RerankFeatureOrder)) 71 if err := rows.Scan(&slateID, &orgID, &candidateID, &decision, &f[0], &f[1], &f[2], &f[3], &f[4], &f[5]); err != nil { 72 return nil, err 73 } 74 if cur == nil || cur.SlateID != slateID { 75 groups = append(groups, SlateGroup{SlateID: slateID, OrgID: orgID}) 76 cur = &groups[len(groups)-1] 77 } 78 cur.Members = append(cur.Members, SlateMember{ 79 SlateID: slateID, 80 CandidateID: candidateID, 81 Features: f, 82 Approved: decision == "approved", 83 }) 84 } 85 return groups, rows.Err() 86} 87 88// toRankGroups projects the slate groups into the pure learn ranking groups (features plus the approved label), 89// dropping the ids the numerical core does not need. 90func toRankGroups(groups []SlateGroup) []learn.RankGroup { 91 out := make([]learn.RankGroup, 0, len(groups)) 92 for _, g := range groups { 93 items := make([]learn.RankItem, 0, len(g.Members)) 94 for _, m := range g.Members { 95 items = append(items, learn.RankItem{Features: m.Features, Approved: m.Approved}) 96 } 97 out = append(out, learn.RankGroup{Items: items}) 98 } 99 return out 100} 101 102// RerankTrainResult summarizes a re ranker training run. 103type RerankTrainResult struct { 104 Skipped bool 105 SkipReason string 106 Version int 107 Groups int 108 Pairs int 109 FeatureNames []string 110 Coeffs []float64 // raw utility weights, aligned to RerankFeatureOrder 111} 112 113// TrainReranker reads the decided slates, fits a pairwise learning to rank model over the frozen sub score 114// features (approved should rank above rejected within a slate), and writes a CHALLENGER version of the display 115// order re ranker. It refuses to fit below the pair floor, leaving the prior only engine order champion in 116// force. It NEVER flips champion. The learned utility weights order candidates; a display order model emits an 117// ORDERING, never a per candidate score, and stores no calibration bins. 118func TrainReranker(ctx context.Context, pool *pgxpool.Pool, modelKey string, l2 float64, minPairs int, dryRun bool) (RerankTrainResult, error) { 119 groups, err := ReadSlateGroups(ctx, pool) 120 if err != nil { 121 return RerankTrainResult{}, err 122 } 123 x, y := learn.PairwiseDataset(toRankGroups(groups)) 124 res := RerankTrainResult{Groups: len(groups), Pairs: len(x), FeatureNames: RerankFeatureOrder} 125 126 if minPairs <= 0 { 127 minPairs = 20 * len(RerankFeatureOrder) // twenty pairs per feature, the ranking analogue of the events floor 128 } 129 if len(x) < minPairs { 130 res.Skipped = true 131 res.SkipReason = fmt.Sprintf("%d approved over rejected pairs is below the floor of %d", len(x), minPairs) 132 return res, nil 133 } 134 135 model, err := learn.FitLogistic(x, y, l2) 136 if err != nil { 137 return RerankTrainResult{}, err 138 } 139 // The utility weights: the raw space coefficients. The intercept is irrelevant to an ordering (it is a 140 // constant across candidates), so it is not stored as a weight; the display order is the utility descending. 141 res.Coeffs = make([]float64, len(RerankFeatureOrder)) 142 for j := range RerankFeatureOrder { 143 res.Coeffs[j] = model.Coeffs[j] / model.Scales[j] 144 } 145 146 if dryRun { 147 return res, nil 148 } 149 150 notes := fmt.Sprintf("Challenger pairwise learning to rank fit by cmd/train-model over %d decided slates (%d approved over rejected pairs) pooled across tenants, ridge l2=%v. OutputKind display_order, never a candidate score; the frozen v1.4.0 engine stays the only per candidate scorer and a human approves every member. champion=false until cmd/eval-model passes the ranking and four fifths gates and a human promotes it.", len(groups), len(x), l2) 151 version, err := WriteChallenger(ctx, pool, modelKey, Challenger{ 152 Family: "pairwise_logistic", 153 OutputKind: "display_order", 154 FeatureNames: RerankFeatureOrder, 155 Intercept: 0, // an ordering is invariant to the intercept 156 Coeffs: res.Coeffs, 157 Notes: notes, 158 }) 159 if err != nil { 160 return RerankTrainResult{}, err 161 } 162 res.Version = version 163 return res, nil 164} 165 166// RerankCoeffs extracts a loaded model's utility weights in RerankFeatureOrder, so a stored re ranker (champion 167// or challenger) can order a slate or be scored for ranking quality. 168func RerankCoeffs(m Model) []float64 { 169 out := make([]float64, len(RerankFeatureOrder)) 170 for j, name := range RerankFeatureOrder { 171 out[j] = m.Coeffs[name] 172 } 173 return out 174} 175 176// OrderMembers returns the slate members in the display order the utility weights imply: the learned utility 177// descending, with the frozen engine order (the incoming slice order) as the STABLE tiebreak, so a zero weight 178// or a tie preserves the engine order exactly. It emits an ORDERING (a permutation), never a score. 179func OrderMembers(coeffs []float64, members []SlateMember) []SlateMember { 180 out := make([]SlateMember, len(members)) 181 copy(out, members) 182 sort.SliceStable(out, func(i, j int) bool { 183 return learn.RankUtility(out[i].Features, coeffs) > learn.RankUtility(out[j].Features, coeffs) 184 }) 185 return out 186} 187 188// RerankQualityResult is the held out ranking comparison of the champion and a challenger re ranker. 189type RerankQualityResult struct { 190 HeldOutGroups int 191 ChampAccuracy float64 192 ChalAccuracy float64 193 Pass bool 194} 195 196// EvaluateRerankerQuality reads the decided slates, holds out a deterministic tail fraction as a test split, and 197// computes the pairwise ranking accuracy (approved ordered above rejected within a slate) under both the 198// champion and the challenger utility weights. The challenger passes when it ranks at least as well as the 199// champion (accuracy within RankTolerance). It is pure over the stored weights and the slate read; the fairness 200// gate is a separate check the caller runs through the compliance pool. 201func EvaluateRerankerQuality(ctx context.Context, pool *pgxpool.Pool, champCoeffs, chalCoeffs []float64, holdout float64) (RerankQualityResult, error) { 202 groups, err := ReadSlateGroups(ctx, pool) 203 if err != nil { 204 return RerankQualityResult{}, err 205 } 206 if len(groups) == 0 { 207 return RerankQualityResult{}, fmt.Errorf("magstore: no decided slates to evaluate") 208 } 209 if holdout <= 0 || holdout >= 1 { 210 holdout = 0.30 211 } 212 cut := len(groups) - int(float64(len(groups))*holdout) 213 if cut >= len(groups) { 214 cut = len(groups) - 1 215 } 216 if cut < 0 { 217 cut = 0 218 } 219 test := toRankGroups(groups[cut:]) 220 res := RerankQualityResult{ 221 HeldOutGroups: len(groups) - cut, 222 ChampAccuracy: learn.PairwiseAccuracy(test, champCoeffs), 223 ChalAccuracy: learn.PairwiseAccuracy(test, chalCoeffs), 224 } 225 res.Pass = res.ChalAccuracy >= res.ChampAccuracy-RankTolerance 226 return res, nil 227} 228 229// GroupsByOrg buckets the slate groups by owning tenant, so the fairness gate can recompute the four fifths 230// ratio per org (a tenant scoped EEO measure) under a challenger ranking. 231func GroupsByOrg(groups []SlateGroup) map[string][]SlateGroup { 232 out := map[string][]SlateGroup{} 233 for _, g := range groups { 234 out[g.OrgID] = append(out[g.OrgID], g) 235 } 236 return out 237} 238 239// CandidateSelection reduces the per slate top k selection to a per candidate set (selected if the ranking 240// surfaces the candidate in the top of ANY of their slates), the shape the eeo four fifths recomputation 241// consumes. It carries only candidate ids, no demographic data. 242func CandidateSelection(coeffs []float64, groups []SlateGroup) map[string]bool { 243 perSlate := TopKSelection(coeffs, groups) 244 out := map[string]bool{} 245 for key := range perSlate { 246 if i := strings.LastIndex(key, "|"); i >= 0 { 247 out[key[i+1:]] = true 248 } 249 } 250 return out 251} 252 253// TopKSelection marks, per slate, the candidates the ranking would surface as the SELECTION it nudges: the top 254// N by the utility weights where N is the number the human actually approved on that slate. It returns a set 255// keyed by "slateID|candidateID". This is the selection proxy the fairness gate recomputes the four fifths 256// ratio over, so a display order change that would skew who surfaces first is caught. Ties preserve the engine 257// order (the stable incoming member order). 258func TopKSelection(coeffs []float64, groups []SlateGroup) map[string]bool { 259 selected := map[string]bool{} 260 for _, g := range groups { 261 n := 0 262 for _, m := range g.Members { 263 if m.Approved { 264 n++ 265 } 266 } 267 if n == 0 { 268 continue 269 } 270 ordered := OrderMembers(coeffs, g.Members) 271 for i := 0; i < n && i < len(ordered); i++ { 272 selected[ordered[i].SlateID+"|"+ordered[i].CandidateID] = true 273 } 274 } 275 return selected 276} 277