package eval_test import ( "testing" "github.com/missionworkforce/missionmatch/internal/modules/matching/engine" "github.com/missionworkforce/missionmatch/internal/modules/matching/eval" ) // ---------- helpers ---------- func fixtureGreen() eval.Fixture { // TS/SCI active, all required skills, cert, customer, experience, on site, available => Green. // Clearance 100*0.25=25, Skills 100*0.25=25, Certs 100*0.15=15, Customer 100*0.15=15, // Exp 100*0.10=10, Location (100+100)/2*0.10=10. Total=100.00 => Green. return eval.Fixture{ ID: "test-green", Lane: "test_lane", Candidate: eval.CandidateJSON{ ClearanceLevel: "ts_sci", ClearancePoly: "none", ClearanceStatus: "active", Skills: []string{"Python", "Machine Learning"}, Certs: []string{"CISSP"}, CustomerAgencies: []string{"DIA"}, YearsExperience: 6, Location: "Fort Meade, MD", Remote: false, Available: true, }, Requirement: eval.RequirementJSON{ RequiredLevel: "ts_sci", RequiredPoly: "none", RequiredSkills: []eval.RequiredSkillJSON{ {Name: "Python", Mandatory: true}, {Name: "Machine Learning", Mandatory: true}, }, RequiredCerts: []string{"CISSP"}, CustomerAgency: "DIA", RequiredYears: 5, WorkLocation: "Fort Meade, MD", RemoteAllowed: false, }, Expected: eval.ExpectedJSON{Band: "Green", Rationale: "perfect match"}, } } func fixtureYellowOneMandatorySkillShort() eval.Fixture { // Missing mandatory Machine Learning. Skills 50%. Clearance 100*0.25=25, Skills 50*0.25=12.5, // Certs 100*0.15=15, Customer 100*0.15=15, Exp 100*0.10=10, Location 100*0.10=10. Total=87.50 => Yellow. return eval.Fixture{ ID: "test-yellow-skill", Lane: "test_lane", Candidate: eval.CandidateJSON{ ClearanceLevel: "ts_sci", ClearancePoly: "none", ClearanceStatus: "active", Skills: []string{"Python"}, Certs: []string{"CISSP"}, CustomerAgencies: []string{"DIA"}, YearsExperience: 5, Location: "Fort Meade, MD", Remote: false, Available: true, }, Requirement: eval.RequirementJSON{ RequiredLevel: "ts_sci", RequiredPoly: "none", RequiredSkills: []eval.RequiredSkillJSON{ {Name: "Python", Mandatory: true}, {Name: "Machine Learning", Mandatory: true}, }, RequiredCerts: []string{"CISSP"}, CustomerAgency: "DIA", RequiredYears: 5, WorkLocation: "Fort Meade, MD", RemoteAllowed: false, }, Expected: eval.ExpectedJSON{ Band: "Yellow", Rationale: "missing mandatory Machine Learning", MandatoryGaps: []string{"Machine Learning"}, }, } } func fixtureRedExpiredClearance() eval.Fixture { // Expired clearance => clearance 0. All other dimensions are perfect. // Total = 0+25+15+15+10+10 = 75.00. bandFor(75.00) = Yellow, but v1.4.0 clearance gap cap // fires (expired clearance zeroes the component; required level TS/SCI >= ClearanceConfidential). // Band is capped to Red. Expert labels Red; engine now agrees. return eval.Fixture{ ID: "test-mismatch-red", Lane: "test_lane", Candidate: eval.CandidateJSON{ ClearanceLevel: "ts_sci", ClearancePoly: "none", ClearanceStatus: "expired", Skills: []string{"Python", "Machine Learning"}, Certs: []string{"CISSP"}, CustomerAgencies: []string{"DIA"}, YearsExperience: 5, Location: "Fort Meade, MD", Remote: false, Available: true, }, Requirement: eval.RequirementJSON{ RequiredLevel: "ts_sci", RequiredPoly: "none", RequiredSkills: []eval.RequiredSkillJSON{ {Name: "Python", Mandatory: true}, {Name: "Machine Learning", Mandatory: true}, }, RequiredCerts: []string{"CISSP"}, CustomerAgency: "DIA", RequiredYears: 5, WorkLocation: "Fort Meade, MD", RemoteAllowed: false, }, Expected: eval.ExpectedJSON{Band: "Red", Rationale: "expired clearance is disqualifying"}, } } // fixtureRedExpertOnlyNoCustomer is a fixture where the expert labels Red because the candidate has // no experience with the required customer agency. The engine scores: clearance 100*0.25=25, // skills 100*0.25=25 (both mandatory met), certs 100*0.15=15, customer 0*0.15=0, exp 100*0.10=10, // location 100*0.10=10. Total = 85.00 => Yellow by the engine. No clearance gap (clearance is fine). // No mandatory skill gap (both skills are met). Expert says Red; engine says Yellow: real mismatch. func fixtureRedExpertOnlyNoCustomer() eval.Fixture { return eval.Fixture{ ID: "test-mismatch-red-no-customer", Lane: "test_lane", Candidate: eval.CandidateJSON{ ClearanceLevel: "ts_sci", ClearancePoly: "none", ClearanceStatus: "active", Skills: []string{"Python", "Machine Learning"}, Certs: []string{"CISSP"}, CustomerAgencies: []string{}, // no DIA experience YearsExperience: 5, Location: "Fort Meade, MD", Remote: false, Available: true, }, Requirement: eval.RequirementJSON{ RequiredLevel: "ts_sci", RequiredPoly: "none", RequiredSkills: []eval.RequiredSkillJSON{ {Name: "Python", Mandatory: true}, {Name: "Machine Learning", Mandatory: true}, }, RequiredCerts: []string{"CISSP"}, CustomerAgency: "DIA", RequiredYears: 5, WorkLocation: "Fort Meade, MD", RemoteAllowed: false, }, // Expert says Red because no DIA experience on this program. Engine says Yellow (85.00). Expected: eval.ExpectedJSON{Band: "Red", Rationale: "no customer experience with DIA is disqualifying per expert"}, } } // ---------- conversion round-trip tests ---------- func TestToEngineCandidateClearanceLevels(t *testing.T) { cases := []struct { code string level engine.ClearanceLevel }{ {"unknown", engine.ClearanceUnknown}, {"none", engine.ClearanceNone}, {"confidential", engine.ClearanceConfidential}, {"secret", engine.ClearanceSecret}, {"top_secret", engine.ClearanceTopSecret}, {"ts_sci", engine.ClearanceTSSCI}, } for _, tc := range cases { cj := eval.CandidateJSON{ClearanceLevel: tc.code, ClearanceStatus: "active"} c := eval.ToEngineCandidate("x", cj) if c.Clearance.Level != tc.level { t.Errorf("code %q: got level %v, want %v", tc.code, c.Clearance.Level, tc.level) } } } func TestToEngineCandidatePolyCodes(t *testing.T) { cases := []struct { code string poly engine.Polygraph }{ {"none", engine.PolyNone}, {"ci", engine.PolyCI}, {"full_scope", engine.PolyFullScope}, } for _, tc := range cases { cj := eval.CandidateJSON{ClearancePoly: tc.code, ClearanceStatus: "active"} c := eval.ToEngineCandidate("x", cj) if c.Clearance.Poly != tc.poly { t.Errorf("code %q: got poly %v, want %v", tc.code, c.Clearance.Poly, tc.poly) } } } func TestToEngineCandidateSkillsCarryNames(t *testing.T) { cj := eval.CandidateJSON{Skills: []string{"Python", "RMF", "Splunk"}, ClearanceStatus: "active"} c := eval.ToEngineCandidate("id", cj) if len(c.Skills) != 3 { t.Fatalf("got %d skills, want 3", len(c.Skills)) } if c.Skills[1].Name != "RMF" { t.Errorf("skill[1] name %q, want RMF", c.Skills[1].Name) } } // ---------- Score function tests ---------- func TestScoreGreenFixtureMatchesEngine(t *testing.T) { fx := fixtureGreen() r := eval.Score([]eval.Fixture{fx}) if r.Total != 1 { t.Fatalf("total %d, want 1", r.Total) } if r.Correct != 1 { t.Fatalf("correct %d, want 1; mismatches: %v", r.Correct, r.Mismatches) } if len(r.Mismatches) != 0 { t.Fatalf("expected no mismatches, got %v", r.Mismatches) } // Confusion matrix: expected Green (2), scored Green (2). if r.Matrix[engine.BandGreen][engine.BandGreen] != 1 { t.Errorf("matrix[Green][Green] = %d, want 1", r.Matrix[engine.BandGreen][engine.BandGreen]) } } func TestScoreYellowFixtureMatchesEngine(t *testing.T) { fx := fixtureYellowOneMandatorySkillShort() r := eval.Score([]eval.Fixture{fx}) if r.Correct != 1 { t.Fatalf("correct %d, want 1; mismatches: %v", r.Correct, r.Mismatches) } if r.Matrix[engine.BandYellow][engine.BandYellow] != 1 { t.Errorf("matrix[Yellow][Yellow] = %d, want 1", r.Matrix[engine.BandYellow][engine.BandYellow]) } } func TestScoreReportsMismatch(t *testing.T) { // v1.4.0: the expired clearance fixture is labeled Red by the expert. The engine now also // produces Red via the clearance gap cap (expired clearance => clearance 0, required level // TS/SCI >= Confidential, so the cap fires). This is now a CORRECT classification, not a // mismatch. To test mismatch surfacing we use a fixture where the expert says Red but the // engine produces Yellow: a non-mandatory skills gap that lands the total at 85.00 Yellow // while the expert judges it Red because of the customer experience miss. fx := fixtureRedExpertOnlyNoCustomer() r := eval.Score([]eval.Fixture{fx}) if r.Total != 1 { t.Fatalf("total %d, want 1", r.Total) } if r.Correct != 0 { t.Fatalf("expected 0 correct (mismatch), got %d", r.Correct) } if len(r.Mismatches) != 1 { t.Fatalf("expected 1 mismatch, got %d: %v", len(r.Mismatches), r.Mismatches) } mm := r.Mismatches[0] if mm.Expected != engine.BandRed { t.Errorf("mismatch.Expected = %v, want Red", mm.Expected) } if mm.Scored != engine.BandYellow { t.Errorf("mismatch.Scored = %v, want Yellow", mm.Scored) } // Matrix: expected Red (0), scored Yellow (1). if r.Matrix[engine.BandRed][engine.BandYellow] != 1 { t.Errorf("matrix[Red][Yellow] = %d, want 1", r.Matrix[engine.BandRed][engine.BandYellow]) } } func TestScoreMultipleFixturesAccumulatesMatrix(t *testing.T) { // v1.4.0: fixtureRedExpiredClearance is now CORRECT (clearance gap cap fires, Red matches expert). // fixtureRedExpertOnlyNoCustomer is the mismatch (expert Red, engine Yellow 85.00, no cap fires). fixtures := []eval.Fixture{ fixtureGreen(), fixtureYellowOneMandatorySkillShort(), fixtureRedExpiredClearance(), fixtureRedExpertOnlyNoCustomer(), } r := eval.Score(fixtures) if r.Total != 4 { t.Fatalf("total %d, want 4", r.Total) } // Green, Yellow, and Red-expired all agree; only the no-customer fixture mismatches. if r.Correct != 3 { t.Fatalf("correct %d, want 3; mismatches: %v", r.Correct, r.Mismatches) } if len(r.Mismatches) != 1 { t.Fatalf("mismatches %d, want 1", len(r.Mismatches)) } } func TestScoreLaneResultsTracked(t *testing.T) { fx1 := fixtureGreen() fx1.Lane = "lane_a" fx2 := fixtureYellowOneMandatorySkillShort() fx2.Lane = "lane_b" r := eval.Score([]eval.Fixture{fx1, fx2}) la, ok := r.LaneResults["lane_a"] if !ok || la.Total != 1 || la.Correct != 1 { t.Errorf("lane_a: %+v", la) } lb, ok := r.LaneResults["lane_b"] if !ok || lb.Total != 1 || lb.Correct != 1 { t.Errorf("lane_b: %+v", lb) } } func TestConfusionMatrixPrecisionAndRecall(t *testing.T) { var m eval.ConfusionMatrix // Two Green correct, one Red misclassified as Green. m[engine.BandGreen][engine.BandGreen] = 2 m[engine.BandRed][engine.BandGreen] = 1 // Green precision = 2 / (2+1) = 0.666... p := m.BandPrecision(engine.BandGreen) if p < 0.666 || p > 0.667 { t.Errorf("precision %v, want ~0.667", p) } // Green recall = 2 / 2 = 1.0 (all expected Green are scored Green). rec := m.BandRecall(engine.BandGreen) if rec != 1.0 { t.Errorf("recall %v, want 1.0", rec) } // Red recall = 0 / 1 = 0.0 (the one expected Red was scored Green). redRec := m.BandRecall(engine.BandRed) if redRec != 0.0 { t.Errorf("red recall %v, want 0.0", redRec) } }