Janady13/missionmatch / internal/modules/ai/curation_test.go

1package ai 2 3// curation_test.go unit tests the WS8 Phase 4 proposal generator with NO database. The derivation 4// (derivePairs) and the selection (selectProposals) are pure functions of the stored evaluation evidence, the 5// clearance override count, and the contents free governance confidence, so a fixture of evaluation rows and 6// aggregates fully exercises them: a recurring approved non exact pair above the threshold yields one 7// substitutes_for_edge with the right from, to, strength, and evidence; a one off pair does not; an exact 8// match never proposes; the governance confidence double gate holds; and the idempotency check suppresses a 9// duplicate. The full GenerateProposalsForOrg path (the RLS scoped scan and the migrator pool write) needs a 10// real Postgres and is covered by the integration test. 11 12import ( 13 "context" 14 "encoding/json" 15 "testing" 16 17 "github.com/google/uuid" 18 19 "github.com/missionworkforce/missionmatch/internal/modules/matching/engine" 20 "github.com/missionworkforce/missionmatch/internal/platform/audit" 21 "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" 22 "github.com/missionworkforce/missionmatch/internal/platform/scrub" 23) 24 25// two fixed skill ids so the fixtures speak in ids the matcher already populated (the common case). When a 26// fixture wants to exercise the name resolution path it leaves the id empty and sets the name. 27var ( 28 skillReqA = uuid.MustParse("11111111-1111-1111-1111-111111111111") 29 skillCandA = uuid.MustParse("22222222-2222-2222-2222-222222222222") 30 skillReqB = uuid.MustParse("33333333-3333-3333-3333-333333333333") 31 skillCandB = uuid.MustParse("44444444-4444-4444-4444-444444444444") 32) 33 34// fullSkill builds a matched_skills entry with explicit ids (the populated id case). 35func fullSkill(reqID, candID uuid.UUID, similarity string, exact, matched bool) map[string]any { 36 return map[string]any{ 37 "RequiredSkillID": reqID.String(), "RequiredSkillName": "req", 38 "CandidateSkillID": candID.String(), "CandidateName": "cand", 39 "Similarity": similarity, "Exact": exact, "Matched": matched, 40 } 41} 42 43// namedSkill builds a matched_skills entry with NO ids, only names (the resolution case). 44func namedSkill(reqName, candName, similarity string, exact, matched bool) map[string]any { 45 return map[string]any{ 46 "RequiredSkillID": "", "RequiredSkillName": reqName, 47 "CandidateSkillID": "", "CandidateName": candName, 48 "Similarity": similarity, "Exact": exact, "Matched": matched, 49 } 50} 51 52func curationEvalRow(t *testing.T, decision string, skills ...map[string]any) evalRow { 53 t.Helper() 54 mj, err := json.Marshal(skills) 55 if err != nil { 56 t.Fatalf("marshal matched skills: %v", err) 57 } 58 return evalRow{ 59 Decision: decision, 60 ClearanceScore: dec("100"), 61 ExperienceScore: dec("100"), 62 RequiredLevel: "none", 63 MatchedSkills: mj, 64 } 65} 66 67// fakeScanner returns fixture rows and resolves a name to an id from a static map. 68type fakeScanner struct { 69 rows []evalRow 70 nameMap map[string]uuid.UUID 71} 72 73func (f fakeScanner) ScanEvaluations(ctx context.Context) ([]evalRow, error) { return f.rows, nil } 74func (f fakeScanner) ResolveSkillID(ctx context.Context, name string) (uuid.UUID, bool, error) { 75 id, ok := f.nameMap[name] 76 return id, ok, nil 77} 78 79// govHigh is a governance aggregate that confirms the substitution pattern is reliably approved at the high 80// band (more approved than not), so the double gate is satisfied for a high band pair. 81func govHigh() []dbgen.TeachingSignalAggregate { 82 return []dbgen.TeachingSignalAggregate{ 83 {Pattern: "adjacent_skill_substitution_outcome", Band: "high", Outcome: "approved", Observations: 5, Successes: 5}, 84 } 85} 86 87// TestDerivePairsCountsRecurringApprovedNonExact proves a recurring approved non exact pair is counted with 88// the right approval, total, and band, and that an exact match and a one off contribute correctly. 89func TestDerivePairsCountsRecurringApprovedNonExact(t *testing.T) { 90 rows := []evalRow{ 91 curationEvalRow(t, "approved", fullSkill(skillReqA, skillCandA, "0.92", false, true)), 92 curationEvalRow(t, "approved", fullSkill(skillReqA, skillCandA, "0.95", false, true)), 93 curationEvalRow(t, "approved", fullSkill(skillReqA, skillCandA, "1", true, true)), // exact, never a pair 94 curationEvalRow(t, "rejected", fullSkill(skillReqA, skillCandA, "0.91", false, true)), 95 } 96 pairs, err := derivePairs(context.Background(), rows, fakeScanner{rows: rows}) 97 if err != nil { 98 t.Fatalf("derivePairs: %v", err) 99 } 100 k := pairKey{fromSkillID: skillCandA.String(), toSkillID: skillReqA.String()} 101 ev := pairs[k] 102 if ev == nil { 103 t.Fatal("expected a derived pair for cand A to req A") 104 } 105 // Three non exact observations (two approved, one rejected); the exact one is never counted. 106 if ev.total != 3 { 107 t.Fatalf("pair total = %d, want 3 (the exact match is excluded)", ev.total) 108 } 109 if ev.approvals != 2 { 110 t.Fatalf("pair approvals = %d, want 2", ev.approvals) 111 } 112 if ev.band != scrub.BandHigh { 113 t.Fatalf("pair band = %q, want high (the strongest approved similarity 0.95)", ev.band) 114 } 115} 116 117// TestSelectProposalsRecurringApprovedPairProposesEdge proves a recurring approved high band pair, supported 118// by the governance confidence, yields exactly one substitutes_for_edge with the right from, to, strength, and 119// evidence counts. 120func TestSelectProposalsRecurringApprovedPairProposesEdge(t *testing.T) { 121 pairs := map[pairKey]*pairEvidence{ 122 {fromSkillID: skillCandA.String(), toSkillID: skillReqA.String()}: {approvals: 3, total: 4, band: scrub.BandHigh}, 123 } 124 plans := selectProposals(pairs, 0, govHigh()) 125 if len(plans) != 1 { 126 t.Fatalf("got %d plans, want exactly 1 substitutes_for_edge", len(plans)) 127 } 128 p := plans[0] 129 if p.kind != kindSubstitutesForEdge { 130 t.Fatalf("plan kind = %q, want substitutes_for_edge", p.kind) 131 } 132 if p.fromSkillID == nil || *p.fromSkillID != skillCandA { 133 t.Fatalf("from_skill_id = %v, want the candidate skill %v", p.fromSkillID, skillCandA) 134 } 135 if p.toSkillID == nil || *p.toSkillID != skillReqA { 136 t.Fatalf("to_skill_id = %v, want the required skill %v", p.toSkillID, skillReqA) 137 } 138 if p.relation != substitutionRelation { 139 t.Fatalf("relation = %q, want substitutes_for", p.relation) 140 } 141 if p.strength == nil || p.strength.String() != "0.9" { 142 t.Fatalf("strength = %v, want 0.90 for the high band", p.strength) 143 } 144 if p.successes != 3 || p.observations != 4 { 145 t.Fatalf("evidence = %d of %d, want 3 of 4", p.successes, p.observations) 146 } 147} 148 149// TestSelectProposalsOneOffDoesNotPropose proves a single approval (below the recurring threshold) never 150// proposes an edge, even with a supporting governance aggregate. 151func TestSelectProposalsOneOffDoesNotPropose(t *testing.T) { 152 pairs := map[pairKey]*pairEvidence{ 153 {fromSkillID: skillCandA.String(), toSkillID: skillReqA.String()}: {approvals: 1, total: 1, band: scrub.BandHigh}, 154 } 155 plans := selectProposals(pairs, 0, govHigh()) 156 for _, p := range plans { 157 if p.kind == kindSubstitutesForEdge { 158 t.Fatalf("a one off pair must not propose a substitutes_for_edge, got %+v", p) 159 } 160 } 161} 162 163// TestSelectProposalsRequiresGovernanceConfidence proves the double gate: a recurring approved pair does NOT 164// propose when the contents free governance aggregate does not support the pattern at that band, even though 165// the within tenant pair count clears the threshold. 166func TestSelectProposalsRequiresGovernanceConfidence(t *testing.T) { 167 pairs := map[pairKey]*pairEvidence{ 168 {fromSkillID: skillCandA.String(), toSkillID: skillReqA.String()}: {approvals: 5, total: 5, band: scrub.BandHigh}, 169 } 170 // Governance says the pattern was mostly NOT approved at the high band, so the gate fails. 171 govWeak := []dbgen.TeachingSignalAggregate{ 172 {Pattern: "adjacent_skill_substitution_outcome", Band: "high", Outcome: "approved", Observations: 1, Successes: 1}, 173 {Pattern: "adjacent_skill_substitution_outcome", Band: "high", Outcome: "rejected", Observations: 9, Successes: 0}, 174 } 175 plans := selectProposals(pairs, 0, govWeak) 176 for _, p := range plans { 177 if p.kind == kindSubstitutesForEdge { 178 t.Fatalf("without governance confidence the pair must not propose an edge, got %+v", p) 179 } 180 } 181} 182 183// TestSelectProposalsLowBandNeverProposes proves a low band similarity, even when recurring and approved, is 184// too weak to curate as a substitution. 185func TestSelectProposalsLowBandNeverProposes(t *testing.T) { 186 pairs := map[pairKey]*pairEvidence{ 187 {fromSkillID: skillCandA.String(), toSkillID: skillReqA.String()}: {approvals: 4, total: 4, band: scrub.BandLow}, 188 } 189 gov := []dbgen.TeachingSignalAggregate{ 190 {Pattern: "adjacent_skill_substitution_outcome", Band: "low", Outcome: "approved", Observations: 4, Successes: 4}, 191 } 192 plans := selectProposals(pairs, 0, gov) 193 for _, p := range plans { 194 if p.kind == kindSubstitutesForEdge { 195 t.Fatalf("a low band pair must not propose a substitutes_for_edge, got %+v", p) 196 } 197 } 198} 199 200// TestSelectProposalsReciprocityReview proves recurring approved clearance gap overrides raise a 201// reciprocity_review (evidence only, no skill ids) and that a single override does not. 202func TestSelectProposalsReciprocityReview(t *testing.T) { 203 plansTwo := selectProposals(nil, 2, nil) 204 sawReview := false 205 for _, p := range plansTwo { 206 if p.kind == kindReciprocityReview { 207 sawReview = true 208 if p.fromSkillID != nil || p.toSkillID != nil { 209 t.Fatal("a reciprocity_review carries no skill ids, it is evidence only") 210 } 211 if p.observations != 2 { 212 t.Fatalf("reciprocity evidence = %d, want 2", p.observations) 213 } 214 } 215 } 216 if !sawReview { 217 t.Fatal("two recurring clearance overrides must raise a reciprocity_review") 218 } 219 plansOne := selectProposals(nil, 1, nil) 220 for _, p := range plansOne { 221 if p.kind == kindReciprocityReview { 222 t.Fatal("a single clearance override must not raise a reciprocity_review") 223 } 224 } 225} 226 227// TestDerivePairsResolvesNamesWhenIdsAbsent proves a matched_skills entry that carries only names is resolved 228// to skill ids through the scanner, so the pair is still derivable when the matcher did not populate ids. 229func TestDerivePairsResolvesNamesWhenIdsAbsent(t *testing.T) { 230 rows := []evalRow{ 231 curationEvalRow(t, "approved", namedSkill("Splunk", "Elastic SIEM", "0.93", false, true)), 232 curationEvalRow(t, "approved", namedSkill("Splunk", "Elastic SIEM", "0.94", false, true)), 233 } 234 scanner := fakeScanner{ 235 rows: rows, 236 nameMap: map[string]uuid.UUID{"Splunk": skillReqB, "Elastic SIEM": skillCandB}, 237 } 238 pairs, err := derivePairs(context.Background(), rows, scanner) 239 if err != nil { 240 t.Fatalf("derivePairs: %v", err) 241 } 242 k := pairKey{fromSkillID: skillCandB.String(), toSkillID: skillReqB.String()} 243 ev := pairs[k] 244 if ev == nil { 245 t.Fatal("expected a derived pair resolved from names") 246 } 247 if ev.approvals != 2 || ev.total != 2 { 248 t.Fatalf("resolved pair = %d of %d, want 2 of 2", ev.approvals, ev.total) 249 } 250} 251 252// TestDerivePairsUnresolvedNameYieldsNoPair proves a name the ontology does not know yields NO pair, never a 253// fabricated id. 254func TestDerivePairsUnresolvedNameYieldsNoPair(t *testing.T) { 255 rows := []evalRow{ 256 curationEvalRow(t, "approved", namedSkill("UnknownReq", "UnknownCand", "0.93", false, true)), 257 curationEvalRow(t, "approved", namedSkill("UnknownReq", "UnknownCand", "0.94", false, true)), 258 } 259 pairs, err := derivePairs(context.Background(), rows, fakeScanner{rows: rows, nameMap: map[string]uuid.UUID{}}) 260 if err != nil { 261 t.Fatalf("derivePairs: %v", err) 262 } 263 if len(pairs) != 0 { 264 t.Fatalf("an unresolved name must yield no pair, got %d", len(pairs)) 265 } 266} 267 268// silentAuditor is a no op systemAuditor for the unit test: it records nothing and never errors, so the write 269// loop can be exercised with no database. The live path uses audit.Writer.System (covered by the integration 270// test, where the global event is asserted against real Postgres). 271type silentAuditor struct{} 272 273func (silentAuditor) System(ctx context.Context, e audit.Event) error { return nil } 274 275func newSilentAuditWriter() systemAuditor { return silentAuditor{} } 276 277// fakeProposalWriter records the inserts and answers the idempotency check from a set of existing pairs. 278type fakeProposalWriter struct { 279 existing map[string]bool // key: from|to|relation 280 inserted []dbgen.InsertCurationProposalParams 281} 282 283func pairExistKey(from, to uuid.UUID, relation string) string { 284 return from.String() + "|" + to.String() + "|" + relation 285} 286 287func (w *fakeProposalWriter) ActiveProposalExists(ctx context.Context, from, to uuid.UUID, relation string) (bool, error) { 288 return w.existing[pairExistKey(from, to, relation)], nil 289} 290 291func (w *fakeProposalWriter) InsertProposal(ctx context.Context, arg dbgen.InsertCurationProposalParams) (uuid.UUID, error) { 292 w.inserted = append(w.inserted, arg) 293 return uuid.New(), nil 294} 295 296// TestWriteProposalIdempotent proves the generator does not insert a duplicate proposal for a pair that is 297// already pending or approved, and the model_version stamp comes from engine.ModelVersion. 298func TestWriteProposalIdempotent(t *testing.T) { 299 strength := dec("0.90") 300 plan := proposalPlan{ 301 kind: kindSubstitutesForEdge, fromSkillID: &skillCandA, toSkillID: &skillReqA, 302 relation: substitutionRelation, strength: &strength, observations: 4, successes: 3, 303 } 304 305 // First, with no existing proposal: it inserts once and stamps the engine model version. 306 writer := &fakeProposalWriter{existing: map[string]bool{}} 307 gen := NewProposalGenerator(nil, nil, writer, newSilentAuditWriter()) 308 created, err := gen.writeProposal(context.Background(), plan) 309 if err != nil { 310 t.Fatalf("writeProposal: %v", err) 311 } 312 if !created || len(writer.inserted) != 1 { 313 t.Fatalf("expected one insert, created=%v inserts=%d", created, len(writer.inserted)) 314 } 315 if writer.inserted[0].ModelVersion != engine.ModelVersion { 316 t.Fatalf("model_version = %q, want engine.ModelVersion %q", writer.inserted[0].ModelVersion, engine.ModelVersion) 317 } 318 319 // Now with the pair already active: it suppresses the duplicate. 320 writer2 := &fakeProposalWriter{existing: map[string]bool{ 321 pairExistKey(skillCandA, skillReqA, substitutionRelation): true, 322 }} 323 gen2 := NewProposalGenerator(nil, nil, writer2, newSilentAuditWriter()) 324 created2, err := gen2.writeProposal(context.Background(), plan) 325 if err != nil { 326 t.Fatalf("writeProposal (dup): %v", err) 327 } 328 if created2 || len(writer2.inserted) != 0 { 329 t.Fatalf("a duplicate pair must not insert, created=%v inserts=%d", created2, len(writer2.inserted)) 330 } 331} 332