Janady13/missionmatch / internal/platform/magstore/survival.go

1package magstore 2 3import ( 4 "context" 5 "fmt" 6 7 "github.com/jackc/pgx/v5/pgxpool" 8 9 "github.com/missionworkforce/missionmatch/internal/platform/learn" 10) 11 12// TimeToFillModelKey is the model_key of the discrete time survival forecast of how long a staffing requirement 13// stays open before it is filled (migration 0039 seeds the prior only v1 constant baseline hazard). 14const TimeToFillModelKey = "requirement_time_to_fill" 15 16// HorizonMonths is the fixed hazard curve length (one coefficient per month). The survival is computed over 17// these months; a fill within horizon question beyond it uses the full curve. 18const HorizonMonths = 12 19 20// hazardFeatureName is the stored coefficient name for a month's hazard (hazard_month_1 .. hazard_month_12), the 21// same names migration 0039 seeds and the serve path reads. 22func hazardFeatureName(month int) string { 23 return fmt.Sprintf("hazard_month_%d", month) 24} 25 26// ReadRequirementDurations reads every requirement's time to fill observation POOLED across tenants: the whole 27// months from creation to the FIRST approved slate member (the fill event), or, when no approval exists, the 28// months from creation to now, right censored (still open). Months are ceil of the elapsed time, clamped to at 29// least 1 and at most maxPeriod. A soft deleted requirement is excluded. Requirements are joined to their 30// slates' approved members by the composite org key, so a cross tenant join cannot happen under the migrator. 31func ReadRequirementDurations(ctx context.Context, pool *pgxpool.Pool, maxPeriod int) ([]learn.Duration, error) { 32 if maxPeriod < 1 { 33 maxPeriod = HorizonMonths 34 } 35 const q = ` 36SELECT 37 GREATEST(1, LEAST($1::int, CEIL(EXTRACT(EPOCH FROM (COALESCE(f.filled_at, now()) - r.created_at)) / (30.0 * 86400))))::int AS periods, 38 (f.filled_at IS NOT NULL) AS filled 39 FROM requirements r 40 LEFT JOIN LATERAL ( 41 SELECT min(sm.decided_at) AS filled_at 42 FROM slate_members sm 43 JOIN slates s ON s.organization_id = sm.organization_id AND s.id = sm.slate_id 44 WHERE s.requirement_id = r.id AND sm.decision = 'approved' 45 ) f ON true 46 WHERE r.deleted_at IS NULL 47 ORDER BY r.id` 48 rows, err := pool.Query(ctx, q, maxPeriod) 49 if err != nil { 50 return nil, err 51 } 52 defer rows.Close() 53 var out []learn.Duration 54 for rows.Next() { 55 var d learn.Duration 56 if err := rows.Scan(&d.Periods, &d.Filled); err != nil { 57 return nil, err 58 } 59 out = append(out, d) 60 } 61 return out, rows.Err() 62} 63 64// TimeToFillTrainResult summarizes a time to fill training run. 65type TimeToFillTrainResult struct { 66 Skipped bool 67 SkipReason string 68 Version int 69 Requirements int 70 Filled int 71 Hazard []float64 // the fitted per month hazard curve 72} 73 74// TrainTimeToFill reads the pooled requirement durations, estimates the empirical discrete time hazard curve, 75// and writes a new CHALLENGER version of the time to fill model. It refuses to fit below the requirement floor 76// or when no requirement has ever been filled (no fill signal to learn from), leaving the prior only baseline 77// champion in force. It NEVER flips champion. The hazard is stored as one coefficient per month; a discrete 78// hazard model carries no calibration bins. 79func TrainTimeToFill(ctx context.Context, pool *pgxpool.Pool, modelKey string, minRequirements int, dryRun bool) (TimeToFillTrainResult, error) { 80 durations, err := ReadRequirementDurations(ctx, pool, HorizonMonths) 81 if err != nil { 82 return TimeToFillTrainResult{}, err 83 } 84 res := TimeToFillTrainResult{Requirements: len(durations)} 85 for _, d := range durations { 86 if d.Filled { 87 res.Filled++ 88 } 89 } 90 if minRequirements <= 0 { 91 minRequirements = 30 92 } 93 if res.Requirements < minRequirements { 94 res.Skipped = true 95 res.SkipReason = fmt.Sprintf("%d requirements is below the floor of %d", res.Requirements, minRequirements) 96 return res, nil 97 } 98 if res.Filled == 0 { 99 res.Skipped = true 100 res.SkipReason = "no requirement has been filled yet, so there is no fill hazard to learn" 101 return res, nil 102 } 103 104 res.Hazard = learn.EmpiricalHazard(durations, HorizonMonths) 105 106 if dryRun { 107 return res, nil 108 } 109 110 names := make([]string, HorizonMonths) 111 for m := 1; m <= HorizonMonths; m++ { 112 names[m-1] = hazardFeatureName(m) 113 } 114 notes := fmt.Sprintf("Challenger empirical discrete time hazard fit by cmd/train-model over %d requirements (%d filled) pooled across tenants. OutputKind opportunity_score expressing P(filled within a horizon), never a candidate score. champion=false until cmd/eval-model passes the calibration gate and a human promotes it.", res.Requirements, res.Filled) 115 version, werr := WriteChallenger(ctx, pool, modelKey, Challenger{ 116 Family: "discrete_hazard", 117 OutputKind: "opportunity_score", 118 FeatureNames: names, 119 Intercept: 0, 120 Coeffs: res.Hazard, 121 Notes: notes, 122 }) 123 if werr != nil { 124 return TimeToFillTrainResult{}, werr 125 } 126 res.Version = version 127 return res, nil 128} 129 130// HazardCurve extracts a loaded model's per month hazard vector in month order (hazard_month_1 .. _12), so a 131// stored time to fill model (champion or challenger) can form its survival curve and answer a fill within 132// horizon question. 133func HazardCurve(m Model) []float64 { 134 hz := make([]float64, HorizonMonths) 135 for month := 1; month <= HorizonMonths; month++ { 136 hz[month-1] = m.Coeffs[hazardFeatureName(month)] 137 } 138 return hz 139} 140 141// FillWithinHorizon returns the model's probability that a requirement is filled within `horizon` months: one 142// minus the cumulative survival at the horizon. It is the served opportunity level number; the per month hazard 143// curve is the interpretable evidence. 144func FillWithinHorizon(m Model, horizon int) float64 { 145 return learn.FillWithin(HazardCurve(m), horizon) 146} 147 148// TimeToFillEval is the held out CALIBRATION comparison of the champion and a challenger time to fill model at a 149// horizon. A baseline hazard is one curve for every requirement, so the meaningful quality metric is 150// calibration: does the predicted fill within horizon probability match the observed fraction filled within the 151// horizon? The gate blocks a challenger that calibrates materially worse. 152type TimeToFillEval struct { 153 Horizon int 154 Evaluated int 155 FillRate float64 156 ChampPred float64 157 ChalPred float64 158 ChampECE float64 159 ChalECE float64 160 Pass bool 161} 162 163// EvaluateTimeToFill reads the pooled requirement durations, holds out a deterministic tail, and over the held 164// out requirements whose fill within horizon outcome is KNOWN (filled at or before the horizon, or observed open 165// for at least the horizon) compares the calibration of the champion and the challenger predicted fill within 166// horizon probabilities against the observed fill rate. A requirement still open but observed for less than the 167// horizon has an unknown outcome and is excluded. The challenger passes when its calibration error is within 168// tolerance of the champion. 169func EvaluateTimeToFill(ctx context.Context, pool *pgxpool.Pool, champ, chal Model, horizon int, holdout float64) (TimeToFillEval, error) { 170 durations, err := ReadRequirementDurations(ctx, pool, HorizonMonths) 171 if err != nil { 172 return TimeToFillEval{}, err 173 } 174 if len(durations) == 0 { 175 return TimeToFillEval{}, fmt.Errorf("magstore: no requirements to evaluate") 176 } 177 if horizon <= 0 { 178 horizon = 3 179 } 180 if holdout <= 0 || holdout >= 1 { 181 holdout = 0.30 182 } 183 cut := len(durations) - int(float64(len(durations))*holdout) 184 if cut >= len(durations) { 185 cut = len(durations) - 1 186 } 187 if cut < 0 { 188 cut = 0 189 } 190 test := durations[cut:] 191 192 res := TimeToFillEval{Horizon: horizon} 193 res.ChampPred = FillWithinHorizon(champ, horizon) 194 res.ChalPred = FillWithinHorizon(chal, horizon) 195 196 var y []int 197 for _, d := range test { 198 known := (d.Filled && d.Periods <= horizon) || d.Periods >= horizon 199 if !known { 200 continue // still open, observed for less than the horizon: the outcome within the horizon is unknown 201 } 202 if d.Filled && d.Periods <= horizon { 203 y = append(y, 1) 204 } else { 205 y = append(y, 0) 206 } 207 } 208 res.Evaluated = len(y) 209 if len(y) == 0 { 210 res.Pass = true // nothing decidable on the held out split; do not block on an empty comparison 211 return res, nil 212 } 213 var sum int 214 for _, v := range y { 215 sum += v 216 } 217 res.FillRate = float64(sum) / float64(len(y)) 218 champProbs := make([]float64, len(y)) 219 chalProbs := make([]float64, len(y)) 220 for i := range y { 221 champProbs[i] = res.ChampPred 222 chalProbs[i] = res.ChalPred 223 } 224 res.ChampECE = learn.ECE(champProbs, y, 1) 225 res.ChalECE = learn.ECE(chalProbs, y, 1) 226 res.Pass = res.ChalECE <= res.ChampECE+ECETolerance 227 return res, nil 228} 229