Janady13/missionmatch / internal/modules/matching/eval/eval_test.go

1package eval_test 2 3import ( 4 "testing" 5 6 "github.com/missionworkforce/missionmatch/internal/modules/matching/engine" 7 "github.com/missionworkforce/missionmatch/internal/modules/matching/eval" 8) 9 10// ---------- helpers ---------- 11 12func fixtureGreen() eval.Fixture { 13 // TS/SCI active, all required skills, cert, customer, experience, on site, available => Green. 14 // Clearance 100*0.25=25, Skills 100*0.25=25, Certs 100*0.15=15, Customer 100*0.15=15, 15 // Exp 100*0.10=10, Location (100+100)/2*0.10=10. Total=100.00 => Green. 16 return eval.Fixture{ 17 ID: "test-green", 18 Lane: "test_lane", 19 Candidate: eval.CandidateJSON{ 20 ClearanceLevel: "ts_sci", 21 ClearancePoly: "none", 22 ClearanceStatus: "active", 23 Skills: []string{"Python", "Machine Learning"}, 24 Certs: []string{"CISSP"}, 25 CustomerAgencies: []string{"DIA"}, 26 YearsExperience: 6, 27 Location: "Fort Meade, MD", 28 Remote: false, 29 Available: true, 30 }, 31 Requirement: eval.RequirementJSON{ 32 RequiredLevel: "ts_sci", 33 RequiredPoly: "none", 34 RequiredSkills: []eval.RequiredSkillJSON{ 35 {Name: "Python", Mandatory: true}, 36 {Name: "Machine Learning", Mandatory: true}, 37 }, 38 RequiredCerts: []string{"CISSP"}, 39 CustomerAgency: "DIA", 40 RequiredYears: 5, 41 WorkLocation: "Fort Meade, MD", 42 RemoteAllowed: false, 43 }, 44 Expected: eval.ExpectedJSON{Band: "Green", Rationale: "perfect match"}, 45 } 46} 47 48func fixtureYellowOneMandatorySkillShort() eval.Fixture { 49 // Missing mandatory Machine Learning. Skills 50%. Clearance 100*0.25=25, Skills 50*0.25=12.5, 50 // Certs 100*0.15=15, Customer 100*0.15=15, Exp 100*0.10=10, Location 100*0.10=10. Total=87.50 => Yellow. 51 return eval.Fixture{ 52 ID: "test-yellow-skill", 53 Lane: "test_lane", 54 Candidate: eval.CandidateJSON{ 55 ClearanceLevel: "ts_sci", 56 ClearancePoly: "none", 57 ClearanceStatus: "active", 58 Skills: []string{"Python"}, 59 Certs: []string{"CISSP"}, 60 CustomerAgencies: []string{"DIA"}, 61 YearsExperience: 5, 62 Location: "Fort Meade, MD", 63 Remote: false, 64 Available: true, 65 }, 66 Requirement: eval.RequirementJSON{ 67 RequiredLevel: "ts_sci", 68 RequiredPoly: "none", 69 RequiredSkills: []eval.RequiredSkillJSON{ 70 {Name: "Python", Mandatory: true}, 71 {Name: "Machine Learning", Mandatory: true}, 72 }, 73 RequiredCerts: []string{"CISSP"}, 74 CustomerAgency: "DIA", 75 RequiredYears: 5, 76 WorkLocation: "Fort Meade, MD", 77 RemoteAllowed: false, 78 }, 79 Expected: eval.ExpectedJSON{ 80 Band: "Yellow", 81 Rationale: "missing mandatory Machine Learning", 82 MandatoryGaps: []string{"Machine Learning"}, 83 }, 84 } 85} 86 87func fixtureRedExpiredClearance() eval.Fixture { 88 // Expired clearance => clearance 0. All other dimensions are perfect. 89 // Total = 0+25+15+15+10+10 = 75.00. bandFor(75.00) = Yellow, but v1.4.0 clearance gap cap 90 // fires (expired clearance zeroes the component; required level TS/SCI >= ClearanceConfidential). 91 // Band is capped to Red. Expert labels Red; engine now agrees. 92 return eval.Fixture{ 93 ID: "test-mismatch-red", 94 Lane: "test_lane", 95 Candidate: eval.CandidateJSON{ 96 ClearanceLevel: "ts_sci", 97 ClearancePoly: "none", 98 ClearanceStatus: "expired", 99 Skills: []string{"Python", "Machine Learning"}, 100 Certs: []string{"CISSP"}, 101 CustomerAgencies: []string{"DIA"}, 102 YearsExperience: 5, 103 Location: "Fort Meade, MD", 104 Remote: false, 105 Available: true, 106 }, 107 Requirement: eval.RequirementJSON{ 108 RequiredLevel: "ts_sci", 109 RequiredPoly: "none", 110 RequiredSkills: []eval.RequiredSkillJSON{ 111 {Name: "Python", Mandatory: true}, 112 {Name: "Machine Learning", Mandatory: true}, 113 }, 114 RequiredCerts: []string{"CISSP"}, 115 CustomerAgency: "DIA", 116 RequiredYears: 5, 117 WorkLocation: "Fort Meade, MD", 118 RemoteAllowed: false, 119 }, 120 Expected: eval.ExpectedJSON{Band: "Red", Rationale: "expired clearance is disqualifying"}, 121 } 122} 123 124// fixtureRedExpertOnlyNoCustomer is a fixture where the expert labels Red because the candidate has 125// no experience with the required customer agency. The engine scores: clearance 100*0.25=25, 126// skills 100*0.25=25 (both mandatory met), certs 100*0.15=15, customer 0*0.15=0, exp 100*0.10=10, 127// location 100*0.10=10. Total = 85.00 => Yellow by the engine. No clearance gap (clearance is fine). 128// No mandatory skill gap (both skills are met). Expert says Red; engine says Yellow: real mismatch. 129func fixtureRedExpertOnlyNoCustomer() eval.Fixture { 130 return eval.Fixture{ 131 ID: "test-mismatch-red-no-customer", 132 Lane: "test_lane", 133 Candidate: eval.CandidateJSON{ 134 ClearanceLevel: "ts_sci", 135 ClearancePoly: "none", 136 ClearanceStatus: "active", 137 Skills: []string{"Python", "Machine Learning"}, 138 Certs: []string{"CISSP"}, 139 CustomerAgencies: []string{}, // no DIA experience 140 YearsExperience: 5, 141 Location: "Fort Meade, MD", 142 Remote: false, 143 Available: true, 144 }, 145 Requirement: eval.RequirementJSON{ 146 RequiredLevel: "ts_sci", 147 RequiredPoly: "none", 148 RequiredSkills: []eval.RequiredSkillJSON{ 149 {Name: "Python", Mandatory: true}, 150 {Name: "Machine Learning", Mandatory: true}, 151 }, 152 RequiredCerts: []string{"CISSP"}, 153 CustomerAgency: "DIA", 154 RequiredYears: 5, 155 WorkLocation: "Fort Meade, MD", 156 RemoteAllowed: false, 157 }, 158 // Expert says Red because no DIA experience on this program. Engine says Yellow (85.00). 159 Expected: eval.ExpectedJSON{Band: "Red", Rationale: "no customer experience with DIA is disqualifying per expert"}, 160 } 161} 162 163// ---------- conversion round-trip tests ---------- 164 165func TestToEngineCandidateClearanceLevels(t *testing.T) { 166 cases := []struct { 167 code string 168 level engine.ClearanceLevel 169 }{ 170 {"unknown", engine.ClearanceUnknown}, 171 {"none", engine.ClearanceNone}, 172 {"confidential", engine.ClearanceConfidential}, 173 {"secret", engine.ClearanceSecret}, 174 {"top_secret", engine.ClearanceTopSecret}, 175 {"ts_sci", engine.ClearanceTSSCI}, 176 } 177 for _, tc := range cases { 178 cj := eval.CandidateJSON{ClearanceLevel: tc.code, ClearanceStatus: "active"} 179 c := eval.ToEngineCandidate("x", cj) 180 if c.Clearance.Level != tc.level { 181 t.Errorf("code %q: got level %v, want %v", tc.code, c.Clearance.Level, tc.level) 182 } 183 } 184} 185 186func TestToEngineCandidatePolyCodes(t *testing.T) { 187 cases := []struct { 188 code string 189 poly engine.Polygraph 190 }{ 191 {"none", engine.PolyNone}, 192 {"ci", engine.PolyCI}, 193 {"full_scope", engine.PolyFullScope}, 194 } 195 for _, tc := range cases { 196 cj := eval.CandidateJSON{ClearancePoly: tc.code, ClearanceStatus: "active"} 197 c := eval.ToEngineCandidate("x", cj) 198 if c.Clearance.Poly != tc.poly { 199 t.Errorf("code %q: got poly %v, want %v", tc.code, c.Clearance.Poly, tc.poly) 200 } 201 } 202} 203 204func TestToEngineCandidateSkillsCarryNames(t *testing.T) { 205 cj := eval.CandidateJSON{Skills: []string{"Python", "RMF", "Splunk"}, ClearanceStatus: "active"} 206 c := eval.ToEngineCandidate("id", cj) 207 if len(c.Skills) != 3 { 208 t.Fatalf("got %d skills, want 3", len(c.Skills)) 209 } 210 if c.Skills[1].Name != "RMF" { 211 t.Errorf("skill[1] name %q, want RMF", c.Skills[1].Name) 212 } 213} 214 215// ---------- Score function tests ---------- 216 217func TestScoreGreenFixtureMatchesEngine(t *testing.T) { 218 fx := fixtureGreen() 219 r := eval.Score([]eval.Fixture{fx}) 220 if r.Total != 1 { 221 t.Fatalf("total %d, want 1", r.Total) 222 } 223 if r.Correct != 1 { 224 t.Fatalf("correct %d, want 1; mismatches: %v", r.Correct, r.Mismatches) 225 } 226 if len(r.Mismatches) != 0 { 227 t.Fatalf("expected no mismatches, got %v", r.Mismatches) 228 } 229 // Confusion matrix: expected Green (2), scored Green (2). 230 if r.Matrix[engine.BandGreen][engine.BandGreen] != 1 { 231 t.Errorf("matrix[Green][Green] = %d, want 1", r.Matrix[engine.BandGreen][engine.BandGreen]) 232 } 233} 234 235func TestScoreYellowFixtureMatchesEngine(t *testing.T) { 236 fx := fixtureYellowOneMandatorySkillShort() 237 r := eval.Score([]eval.Fixture{fx}) 238 if r.Correct != 1 { 239 t.Fatalf("correct %d, want 1; mismatches: %v", r.Correct, r.Mismatches) 240 } 241 if r.Matrix[engine.BandYellow][engine.BandYellow] != 1 { 242 t.Errorf("matrix[Yellow][Yellow] = %d, want 1", r.Matrix[engine.BandYellow][engine.BandYellow]) 243 } 244} 245 246func TestScoreReportsMismatch(t *testing.T) { 247 // v1.4.0: the expired clearance fixture is labeled Red by the expert. The engine now also 248 // produces Red via the clearance gap cap (expired clearance => clearance 0, required level 249 // TS/SCI >= Confidential, so the cap fires). This is now a CORRECT classification, not a 250 // mismatch. To test mismatch surfacing we use a fixture where the expert says Red but the 251 // engine produces Yellow: a non-mandatory skills gap that lands the total at 85.00 Yellow 252 // while the expert judges it Red because of the customer experience miss. 253 fx := fixtureRedExpertOnlyNoCustomer() 254 r := eval.Score([]eval.Fixture{fx}) 255 if r.Total != 1 { 256 t.Fatalf("total %d, want 1", r.Total) 257 } 258 if r.Correct != 0 { 259 t.Fatalf("expected 0 correct (mismatch), got %d", r.Correct) 260 } 261 if len(r.Mismatches) != 1 { 262 t.Fatalf("expected 1 mismatch, got %d: %v", len(r.Mismatches), r.Mismatches) 263 } 264 mm := r.Mismatches[0] 265 if mm.Expected != engine.BandRed { 266 t.Errorf("mismatch.Expected = %v, want Red", mm.Expected) 267 } 268 if mm.Scored != engine.BandYellow { 269 t.Errorf("mismatch.Scored = %v, want Yellow", mm.Scored) 270 } 271 // Matrix: expected Red (0), scored Yellow (1). 272 if r.Matrix[engine.BandRed][engine.BandYellow] != 1 { 273 t.Errorf("matrix[Red][Yellow] = %d, want 1", r.Matrix[engine.BandRed][engine.BandYellow]) 274 } 275} 276 277func TestScoreMultipleFixturesAccumulatesMatrix(t *testing.T) { 278 // v1.4.0: fixtureRedExpiredClearance is now CORRECT (clearance gap cap fires, Red matches expert). 279 // fixtureRedExpertOnlyNoCustomer is the mismatch (expert Red, engine Yellow 85.00, no cap fires). 280 fixtures := []eval.Fixture{ 281 fixtureGreen(), 282 fixtureYellowOneMandatorySkillShort(), 283 fixtureRedExpiredClearance(), 284 fixtureRedExpertOnlyNoCustomer(), 285 } 286 r := eval.Score(fixtures) 287 if r.Total != 4 { 288 t.Fatalf("total %d, want 4", r.Total) 289 } 290 // Green, Yellow, and Red-expired all agree; only the no-customer fixture mismatches. 291 if r.Correct != 3 { 292 t.Fatalf("correct %d, want 3; mismatches: %v", r.Correct, r.Mismatches) 293 } 294 if len(r.Mismatches) != 1 { 295 t.Fatalf("mismatches %d, want 1", len(r.Mismatches)) 296 } 297} 298 299func TestScoreLaneResultsTracked(t *testing.T) { 300 fx1 := fixtureGreen() 301 fx1.Lane = "lane_a" 302 fx2 := fixtureYellowOneMandatorySkillShort() 303 fx2.Lane = "lane_b" 304 r := eval.Score([]eval.Fixture{fx1, fx2}) 305 la, ok := r.LaneResults["lane_a"] 306 if !ok || la.Total != 1 || la.Correct != 1 { 307 t.Errorf("lane_a: %+v", la) 308 } 309 lb, ok := r.LaneResults["lane_b"] 310 if !ok || lb.Total != 1 || lb.Correct != 1 { 311 t.Errorf("lane_b: %+v", lb) 312 } 313} 314 315func TestConfusionMatrixPrecisionAndRecall(t *testing.T) { 316 var m eval.ConfusionMatrix 317 // Two Green correct, one Red misclassified as Green. 318 m[engine.BandGreen][engine.BandGreen] = 2 319 m[engine.BandRed][engine.BandGreen] = 1 320 321 // Green precision = 2 / (2+1) = 0.666... 322 p := m.BandPrecision(engine.BandGreen) 323 if p < 0.666 || p > 0.667 { 324 t.Errorf("precision %v, want ~0.667", p) 325 } 326 327 // Green recall = 2 / 2 = 1.0 (all expected Green are scored Green). 328 rec := m.BandRecall(engine.BandGreen) 329 if rec != 1.0 { 330 t.Errorf("recall %v, want 1.0", rec) 331 } 332 333 // Red recall = 0 / 1 = 0.0 (the one expected Red was scored Green). 334 redRec := m.BandRecall(engine.BandRed) 335 if redRec != 0.0 { 336 t.Errorf("red recall %v, want 0.0", redRec) 337 } 338} 339