Janady13/missionmatch / internal/modules/ai/teaching_test.go

1package ai 2 3// teaching_test.go unit tests the WS8 producer's folding and persist logic with NO database. fold is a pure 4// function of the stored decision evidence, so a fixture of DecisionRows fully exercises it: the four craft 5// patterns, the similarity banding, the decision to outcome mapping, the observation and success counts, the 6// engine model version stamp, and the scrub boundary acceptance of every produced signal. persist runs the 7// teach and write loop against a fake writer and a gated off inference client, so the live in tenant write 8// shape (tenant_id set, dp_applied false) is asserted off box. The full AggregateForOrg path (the RLS scoped 9// tenant tx read and the audit on that tx) needs a real Postgres and is covered by the integration test. 10 11import ( 12 "context" 13 "encoding/json" 14 "testing" 15 "time" 16 17 "github.com/google/uuid" 18 "github.com/shopspring/decimal" 19 20 "github.com/missionworkforce/missionmatch/internal/modules/matching/engine" 21 "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" 22 "github.com/missionworkforce/missionmatch/internal/platform/inference" 23 "github.com/missionworkforce/missionmatch/internal/platform/scrub" 24) 25 26func dec(s string) decimal.Decimal { return decimal.RequireFromString(s) } 27 28// matchedSkillsJSON builds the matched_skills jsonb the engine persists. The engine SkillMatch type has no 29// json tags, so the real column keys are the Go field names; the fixture uses those same keys so the test 30// exercises the exact bytes the producer reads in production. 31func matchedSkillsJSON(t *testing.T, skills ...map[string]any) []byte { 32 t.Helper() 33 b, err := json.Marshal(skills) 34 if err != nil { 35 t.Fatalf("marshal matched skills fixture: %v", err) 36 } 37 return b 38} 39 40func semanticSkill(similarity string) map[string]any { 41 return map[string]any{"Similarity": similarity, "Exact": false, "Matched": true} 42} 43 44func exactSkill() map[string]any { 45 return map[string]any{"Similarity": "1", "Exact": true, "Matched": true} 46} 47 48// findSignal returns the produced signal for a (pattern, band, outcome) bucket, or fails the test. 49func findSignal(t *testing.T, sigs []scrub.TeachingSignal, pattern string, band scrub.SimilarityBand, outcome scrub.Outcome) scrub.TeachingSignal { 50 t.Helper() 51 for _, s := range sigs { 52 if s.Pattern == pattern && s.Band == band && s.Outcome == outcome { 53 return s 54 } 55 } 56 t.Fatalf("no signal for pattern=%s band=%s outcome=%s in %+v", pattern, band, outcome, sigs) 57 return scrub.TeachingSignal{} 58} 59 60// TestFoldAdjacentSkillSubstitution proves a non exact matched skill folds into 61// adjacent_skill_substitution_outcome, banded by its cosine similarity, with the right observation and 62// success counts, and that an exact match contributes nothing to this pattern. 63func TestFoldAdjacentSkillSubstitution(t *testing.T) { 64 rows := []DecisionRow{ 65 // approved, one high band (0.92) semantic substitution and one exact (ignored here) 66 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100"), 67 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.92"), exactSkill())}, 68 // approved, one medium band (0.85) semantic substitution 69 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100"), 70 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.85"))}, 71 // rejected, one low band (0.70) semantic substitution 72 {Decision: "rejected", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100"), 73 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.70"))}, 74 // approved, another high band (0.95) substitution, so the high+approved bucket reaches two 75 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100"), 76 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.95"))}, 77 } 78 79 sigs := fold(rows, engine.ModelVersion) 80 81 high := findSignal(t, sigs, "adjacent_skill_substitution_outcome", scrub.BandHigh, scrub.OutcomeApproved) 82 if high.Observations != 2 || high.Successes != 2 { 83 t.Fatalf("high/approved = obs %d succ %d, want 2/2", high.Observations, high.Successes) 84 } 85 medium := findSignal(t, sigs, "adjacent_skill_substitution_outcome", scrub.BandMedium, scrub.OutcomeApproved) 86 if medium.Observations != 1 || medium.Successes != 1 { 87 t.Fatalf("medium/approved = obs %d succ %d, want 1/1", medium.Observations, medium.Successes) 88 } 89 low := findSignal(t, sigs, "adjacent_skill_substitution_outcome", scrub.BandLow, scrub.OutcomeRejected) 90 if low.Observations != 1 || low.Successes != 0 { 91 t.Fatalf("low/rejected = obs %d succ %d, want 1/0 (rejected is not a success)", low.Observations, low.Successes) 92 } 93 // The exact match must never appear as a substitution band. 94 for _, s := range sigs { 95 if s.Pattern == "adjacent_skill_substitution_outcome" && s.Band == scrub.BandExact { 96 t.Fatalf("an exact match was folded as a substitution: %+v", s) 97 } 98 } 99} 100 101// TestFoldClearanceGapOverride proves a required clearance the engine scored zero folds into 102// clearance_gap_override_outcome at the unknown band, and that a met clearance or a not required clearance 103// folds nothing. 104func TestFoldClearanceGapOverride(t *testing.T) { 105 rows := []DecisionRow{ 106 // required secret, clearance_score 0, approved => an override (success) 107 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("100")}, 108 // required secret, clearance_score 0, rejected => a gap observed but not overridden 109 {Decision: "rejected", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("100")}, 110 // required secret but met (score 100) => not a gap, folds nothing here 111 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("100"), ExperienceScore: dec("100")}, 112 // no clearance required (none) with score 0 => not a gap (engine passes everyone), folds nothing 113 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("0"), ExperienceScore: dec("100")}, 114 } 115 116 sigs := fold(rows, engine.ModelVersion) 117 118 app := findSignal(t, sigs, "clearance_gap_override_outcome", scrub.BandUnknown, scrub.OutcomeApproved) 119 if app.Observations != 1 || app.Successes != 1 { 120 t.Fatalf("clearance approved = obs %d succ %d, want 1/1", app.Observations, app.Successes) 121 } 122 rej := findSignal(t, sigs, "clearance_gap_override_outcome", scrub.BandUnknown, scrub.OutcomeRejected) 123 if rej.Observations != 1 || rej.Successes != 0 { 124 t.Fatalf("clearance rejected = obs %d succ %d, want 1/0", rej.Observations, rej.Successes) 125 } 126 // Exactly two clearance signals (the approved and the rejected bucket), never a spurious one. 127 clearanceSignals := 0 128 for _, s := range sigs { 129 if s.Pattern == "clearance_gap_override_outcome" { 130 clearanceSignals++ 131 } 132 } 133 if clearanceSignals != 2 { 134 t.Fatalf("got %d clearance signals, want 2 (the met and the not required members must not fold)", clearanceSignals) 135 } 136} 137 138// TestFoldExperienceShortfall proves an experience_score below 100 folds into experience_shortfall_outcome, 139// and a full experience score folds nothing. 140func TestFoldExperienceShortfall(t *testing.T) { 141 rows := []DecisionRow{ 142 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("60")}, 143 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100")}, // full, no shortfall 144 {Decision: "pending", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("0")}, 145 } 146 147 sigs := fold(rows, engine.ModelVersion) 148 149 app := findSignal(t, sigs, "experience_shortfall_outcome", scrub.BandUnknown, scrub.OutcomeApproved) 150 if app.Observations != 1 || app.Successes != 1 { 151 t.Fatalf("experience approved = obs %d succ %d, want 1/1", app.Observations, app.Successes) 152 } 153 pend := findSignal(t, sigs, "experience_shortfall_outcome", scrub.BandUnknown, scrub.OutcomePending) 154 if pend.Observations != 1 || pend.Successes != 0 { 155 t.Fatalf("experience pending = obs %d succ %d, want 1/0 (pending is not a success)", pend.Observations, pend.Successes) 156 } 157} 158 159// TestFoldRerankerLabelAgreementIsZeroToday documents and pins the honest gap: no stored evidence supports 160// reranker_label_agreement, so the producer derives nothing for it. If a future change starts emitting it, 161// this test fails and forces the author to wire real stored evidence first. 162func TestFoldRerankerLabelAgreementIsZeroToday(t *testing.T) { 163 rows := []DecisionRow{ 164 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("50"), 165 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.9"))}, 166 } 167 for _, s := range fold(rows, engine.ModelVersion) { 168 if s.Pattern == "reranker_label_agreement" { 169 t.Fatalf("reranker_label_agreement was emitted with no stored evidence to support it: %+v", s) 170 } 171 } 172} 173 174// TestFoldNoStoredEvaluationFoldsNoSubstitution proves a slate member with no candidate_evaluations row (the 175// LEFT join yields a NULL matched_skills) contributes nothing to the substitution pattern, so the producer 176// never fabricates a skill match it cannot see. 177func TestFoldNoStoredEvaluationFoldsNoSubstitution(t *testing.T) { 178 rows := []DecisionRow{ 179 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("100"), MatchedSkills: nil}, 180 } 181 for _, s := range fold(rows, engine.ModelVersion) { 182 if s.Pattern == "adjacent_skill_substitution_outcome" { 183 t.Fatalf("a substitution was folded from a member with no stored evaluation: %+v", s) 184 } 185 } 186} 187 188// TestFoldModelVersionIsEngineVersion proves every produced signal is stamped with engine.ModelVersion, never 189// a literal, so a drifted engine version is visible at the boundary. 190func TestFoldModelVersionIsEngineVersion(t *testing.T) { 191 rows := []DecisionRow{ 192 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("100")}, 193 } 194 sigs := fold(rows, engine.ModelVersion) 195 if len(sigs) == 0 { 196 t.Fatal("expected at least one signal") 197 } 198 for _, s := range sigs { 199 if s.ModelVersion != engine.ModelVersion { 200 t.Fatalf("signal ModelVersion = %q, want engine.ModelVersion %q", s.ModelVersion, engine.ModelVersion) 201 } 202 } 203} 204 205// TestEveryProducedSignalCrossesTheScrubBoundary proves the boundary holds: every signal the producer builds 206// from a rich fixture is accepted by scrub.Marshal (the same validation the inference plane runs on receive), 207// and its serialized form carries nothing but the six closed fields, no candidate id, no name, no raw score. 208func TestEveryProducedSignalCrossesTheScrubBoundary(t *testing.T) { 209 // register here so the test is order independent of the package init; RegisterModelVersion is idempotent. 210 scrub.RegisterModelVersion(engine.ModelVersion) 211 212 rows := []DecisionRow{ 213 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("50"), 214 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.93"), semanticSkill("0.82"))}, 215 {Decision: "rejected", RequiredLevel: "top_secret", ClearanceScore: dec("0"), ExperienceScore: dec("100"), 216 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.70"))}, 217 {Decision: "pending", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("20")}, 218 } 219 sigs := fold(rows, engine.ModelVersion) 220 if len(sigs) == 0 { 221 t.Fatal("expected produced signals to validate") 222 } 223 want := map[string]bool{"pattern": true, "band": true, "outcome": true, "observations": true, "successes": true, "model_version": true} 224 for _, s := range sigs { 225 raw, err := scrub.Marshal(s) 226 if err != nil { 227 t.Fatalf("scrub.Marshal refused a produced signal %+v: %v", s, err) 228 } 229 var m map[string]json.RawMessage 230 if err := json.Unmarshal(raw, &m); err != nil { 231 t.Fatalf("unmarshal produced signal: %v", err) 232 } 233 if len(m) != len(want) { 234 t.Fatalf("produced signal has %d fields, want %d: %s", len(m), len(want), raw) 235 } 236 for k := range m { 237 if !want[k] { 238 t.Fatalf("produced signal carries an unexpected field %q: %s", k, raw) 239 } 240 } 241 } 242} 243 244// recordingWriter records every aggregate upsert so the test asserts the live in tenant write shape. 245type recordingWriter struct { 246 upserts []dbgen.UpsertTeachingSignalAggregateParams 247} 248 249func (w *recordingWriter) UpsertTeachingSignalAggregate(_ context.Context, arg dbgen.UpsertTeachingSignalAggregateParams) error { 250 w.upserts = append(w.upserts, arg) 251 return nil 252} 253 254// TestPersistWritesLiveInTenantRows exercises the teach and write loop with a fake migrator writer and an 255// inference client whose TeachingEnabled is false. It proves every folded signal is persisted with tenant_id 256// set to the org, dp_applied false, the engine model version, valid window dates, and the folded counts, and 257// that Teach is exercised end to end (scrub.Marshal) without transmitting anything off box. 258func TestPersistWritesLiveInTenantRows(t *testing.T) { 259 scrub.RegisterModelVersion(engine.ModelVersion) 260 org := uuid.New() 261 ctx := context.Background() 262 263 rows := []DecisionRow{ 264 {Decision: "approved", RequiredLevel: "secret", ClearanceScore: dec("0"), ExperienceScore: dec("100")}, 265 {Decision: "approved", RequiredLevel: "none", ClearanceScore: dec("100"), ExperienceScore: dec("40"), 266 MatchedSkills: matchedSkillsJSON(t, semanticSkill("0.91"))}, 267 } 268 signals := fold(rows, engine.ModelVersion) 269 270 writer := &recordingWriter{} 271 // TeachingEnabled false: Teach validates the boundary but transmits nothing, so the placeholder BaseURL is 272 // never reached (a network call would fail the test). 273 teacher, err := inference.New(inference.Config{BaseURL: "http://127.0.0.1:0", TeachingEnabled: false}) 274 if err != nil { 275 t.Fatalf("inference client: %v", err) 276 } 277 prod := &TeachingProducer{ 278 writer: writer, 279 teacher: teacher, 280 now: func() time.Time { return time.Date(2026, 6, 30, 12, 0, 0, 0, time.UTC) }, 281 } 282 283 if err := prod.persist(ctx, org.String(), signals); err != nil { 284 t.Fatalf("persist: %v", err) 285 } 286 if len(writer.upserts) != len(signals) { 287 t.Fatalf("persisted %d aggregates, want %d (one per signal)", len(writer.upserts), len(signals)) 288 } 289 for _, u := range writer.upserts { 290 if u.TenantID == nil || *u.TenantID != org { 291 t.Fatalf("aggregate tenant_id = %v, want the org %v (live in tenant row)", u.TenantID, org) 292 } 293 if u.DpApplied { 294 t.Fatal("aggregate dp_applied = true, want false for a live in tenant row") 295 } 296 if u.ModelVersion != engine.ModelVersion { 297 t.Fatalf("aggregate model_version = %q, want %q", u.ModelVersion, engine.ModelVersion) 298 } 299 if !u.WindowStart.Valid || !u.WindowEnd.Valid { 300 t.Fatal("aggregate window dates must be valid, never NULL") 301 } 302 if u.Observations < u.Successes || u.Successes < 0 { 303 t.Fatalf("aggregate counts not a valid aggregate: obs %d succ %d", u.Observations, u.Successes) 304 } 305 } 306} 307