//go:build integration // MAG time to fill survival loop integration. Proves at runtime against a REAL Postgres and the migrated 0039 // prior only baseline what the unit tests cannot: the offline TRAINER estimates the empirical discrete time // hazard from requirement fill durations (created to first approved slate member, or right censored when still // open), the EVALUATOR compares the challenger's fill within horizon calibration to the prior only constant // baseline, a human promotion flips the champion, and the served fill within horizon probability then reflects // the LEARNED hazard, with the frozen engine untouched (this is an opportunity level forecast, never a candidate // score). // // Run: go test -tags=integration ./internal/integration/ -run MAGTimeToFill -v // Env: MM_APP_DSN, MM_MIGRATOR_DSN, MM_COMPLIANCE_DSN. package integration import ( "context" "testing" "github.com/missionworkforce/missionmatch/internal/platform/magstore" ) func TestMAGTimeToFillTrainsEvaluatesPromotesAndServes(t *testing.T) { modelKey := magstore.TimeToFillModelKey const priorOnlyVersion = 1 ctx := context.Background() if _, err := magstore.PromoteChampion(ctx, migrator, modelKey, priorOnlyVersion); err != nil { t.Fatalf("reset time to fill champion: %v", err) } t.Cleanup(func() { if _, cerr := magstore.PromoteChampion(ctx, migrator, modelKey, priorOnlyVersion); cerr != nil { t.Logf("cleanup: could not restore prior only time to fill champion: %v", cerr) } }) // Seed a handful of FILLED requirements (an approved slate member is the fill event) so the pooled fit has a // fill signal to learn from. Each is created and decided in this run, so it fills at month 1. org := seedOrg(t) approved := [6]float64{50, 50, 50, 50, 50, 50} for i := 0; i < 6; i++ { seedDecidedSlate(t, org, []memberSpec{ {approved: true, features: approved}, {approved: false, features: approved}, }) } // The prior only champion forecasts P(fill within 3) from the constant 0.25 monthly hazard (about 0.578). champV1, err := magstore.LoadModel(ctx, migrator, modelKey, priorOnlyVersion) if err != nil { t.Fatalf("load prior only champion: %v", err) } priorFill3 := magstore.FillWithinHorizon(champV1, 3) t.Logf("prior only P(fill within 3 months) = %.4f", priorFill3) // TRAIN: estimate the empirical hazard challenger. min 5 requirements so the pooled staging set clears it. res, err := magstore.TrainTimeToFill(ctx, migrator, modelKey, 5, false) if err != nil { t.Fatalf("train time to fill: %v", err) } if res.Skipped { t.Fatalf("the fit must not be skipped with %d requirements (%d filled): %s", res.Requirements, res.Filled, res.SkipReason) } if res.Version <= priorOnlyVersion { t.Fatalf("the challenger version %d must be newer than the prior", res.Version) } if len(res.Hazard) != magstore.HorizonMonths { t.Fatalf("the hazard curve must have %d months, got %d", magstore.HorizonMonths, len(res.Hazard)) } anyHazard := false for _, h := range res.Hazard { if h > 0 { anyHazard = true } if h < 0 || h > 1 { t.Fatalf("every hazard must be a probability in [0,1], got %v", res.Hazard) } } if !anyHazard { t.Fatal("with filled requirements in the pool at least one month must carry a positive hazard") } t.Logf("challenger v%d learned hazard month 1 = %.4f over %d requirements (%d filled)", res.Version, res.Hazard[0], res.Requirements, res.Filled) // The trainer must NOT flip champion. stillV1, err := magstore.ChampionVersion(ctx, migrator, modelKey) if err != nil { t.Fatalf("champion after train: %v", err) } if stillV1 != priorOnlyVersion { t.Fatalf("the trainer must not flip champion, moved to v%d", stillV1) } chal, err := magstore.LoadModel(ctx, migrator, modelKey, res.Version) if err != nil { t.Fatalf("load challenger: %v", err) } chalFill3 := magstore.FillWithinHorizon(chal, 3) if chalFill3 < 0 || chalFill3 > 1 { t.Fatalf("the learned P(fill within 3) must be a probability, got %v", chalFill3) } if chalFill3 == priorFill3 { t.Fatalf("the learned forecast must differ from the constant prior, both %v", chalFill3) } // EVALUATE calibration at a 1 month horizon (robustly decidable regardless of the staging data age): the // challenger empirical hazard must calibrate at least as well as the arbitrary constant prior. quality, err := magstore.EvaluateTimeToFill(ctx, migrator, champV1, chal, 1, 0.30) if err != nil { t.Fatalf("evaluate time to fill: %v", err) } t.Logf("horizon 1: %d decidable, observed fill rate %.4f, P champion %.4f vs challenger %.4f, ECE champion %.4f vs challenger %.4f", quality.Evaluated, quality.FillRate, quality.ChampPred, quality.ChalPred, quality.ChampECE, quality.ChalECE) if quality.Evaluated == 0 { t.Fatal("the held out split must have decidable requirements at a 1 month horizon") } if !quality.Pass { t.Errorf("the empirical challenger must clear the calibration gate against the arbitrary prior, got %+v", quality) } // PROMOTE and serve: the champion forecast now reflects the learned hazard. from, err := magstore.PromoteChampion(ctx, migrator, modelKey, res.Version) if err != nil { t.Fatalf("promote: %v", err) } if from != priorOnlyVersion { t.Errorf("the promotion must replace the prior champion v%d, replaced v%d", priorOnlyVersion, from) } champNew, err := magstore.LoadModel(ctx, migrator, modelKey, res.Version) if err != nil { t.Fatalf("load new champion: %v", err) } if got := magstore.FillWithinHorizon(champNew, 3); got != chalFill3 { t.Fatalf("the served forecast must reflect the learned hazard %v, got %v", chalFill3, got) } t.Logf("promoted: served P(fill within 3) moved from prior %.4f to learned %.4f", priorFill3, chalFill3) }