Janady13/missionmatch / internal/modules/matching/eval/eval.go

1// Package eval loads the labeled GovCon benchmark fixtures and scores them through the frozen 2// engine, producing a band confusion matrix and per lane pass rates. It is the regression and 3// fairness harness for the AEDT engine: a divergence between the domain truth band in a fixture 4// and the engine band is a finding to investigate, not a test failure, because the fixture labels 5// are independent expert judgments. With -strict the CLI exits nonzero when any mismatch is found, 6// enabling a future CI gate once the fixture labels are customer-validated. 7// 8// No demographic/EEO fields appear in any fixture. The engine never receives them. 9package eval 10 11import ( 12 "context" 13 "encoding/json" 14 "fmt" 15 "os" 16 "path/filepath" 17 "strings" 18 19 "github.com/missionworkforce/missionmatch/internal/modules/matching/engine" 20) 21 22// ---------- JSON fixture types ---------- 23 24// CandidateJSON mirrors the candidate block in fixtures.json. 25type CandidateJSON struct { 26 ClearanceLevel string `json:"clearance_level"` 27 ClearancePoly string `json:"clearance_poly"` 28 ClearanceStatus string `json:"clearance_status"` 29 Skills []string `json:"skills"` 30 Certs []string `json:"certs"` 31 CustomerAgencies []string `json:"customer_agencies"` 32 YearsExperience int `json:"years_experience"` 33 Location string `json:"location"` 34 Remote bool `json:"remote"` 35 Relocatable bool `json:"relocatable"` 36 Available bool `json:"available"` 37} 38 39// RequiredSkillJSON mirrors one element of the required_skills array. 40type RequiredSkillJSON struct { 41 Name string `json:"name"` 42 Mandatory bool `json:"mandatory"` 43} 44 45// RequirementJSON mirrors the requirement block in fixtures.json. 46type RequirementJSON struct { 47 RequiredLevel string `json:"required_level"` 48 RequiredPoly string `json:"required_poly"` 49 RequiredSkills []RequiredSkillJSON `json:"required_skills"` 50 RequiredCerts []string `json:"required_certs"` 51 CustomerAgency string `json:"customer_agency"` 52 RequiredYears int `json:"required_years"` 53 WorkLocation string `json:"work_location"` 54 RemoteAllowed bool `json:"remote_allowed"` 55} 56 57// ExpectedJSON mirrors the expected block in fixtures.json. 58type ExpectedJSON struct { 59 Band string `json:"band"` 60 Rationale string `json:"rationale"` 61 MandatoryGaps []string `json:"mandatory_gaps,omitempty"` 62} 63 64// Fixture is one benchmark case: a candidate, a requirement, and the domain truth expected band. 65type Fixture struct { 66 ID string `json:"id"` 67 Candidate CandidateJSON `json:"candidate"` 68 Requirement RequirementJSON `json:"requirement"` 69 Expected ExpectedJSON `json:"expected"` 70 Lane string `json:"-"` // injected from the directory name after loading 71} 72 73// ---------- parsers (string codes to engine enum values) ---------- 74 75func parseClearanceLevel(s string) engine.ClearanceLevel { 76 switch strings.ToLower(strings.TrimSpace(s)) { 77 case "confidential": 78 return engine.ClearanceConfidential 79 case "secret": 80 return engine.ClearanceSecret 81 case "top_secret": 82 return engine.ClearanceTopSecret 83 case "ts_sci": 84 return engine.ClearanceTSSCI 85 case "none": 86 return engine.ClearanceNone 87 default: 88 return engine.ClearanceUnknown 89 } 90} 91 92func parsePoly(s string) engine.Polygraph { 93 switch strings.ToLower(strings.TrimSpace(s)) { 94 case "ci": 95 return engine.PolyCI 96 case "full_scope": 97 return engine.PolyFullScope 98 default: 99 return engine.PolyNone 100 } 101} 102 103func parseClearanceStatus(s string) engine.ClearanceStatus { 104 switch strings.ToLower(strings.TrimSpace(s)) { 105 case "active": 106 return engine.StatusActive 107 case "expired": 108 return engine.StatusExpired 109 case "inactive": 110 return engine.StatusInactive 111 default: 112 return engine.StatusUnknown 113 } 114} 115 116func parseBand(s string) engine.Band { 117 switch strings.ToLower(strings.TrimSpace(s)) { 118 case "green": 119 return engine.BandGreen 120 case "yellow": 121 return engine.BandYellow 122 default: 123 return engine.BandRed 124 } 125} 126 127// ---------- conversion: JSON fixture -> engine types ---------- 128 129// ToEngineCandidate converts a CandidateJSON to an engine.Candidate. The ID is the fixture ID. 130func ToEngineCandidate(id string, cj CandidateJSON) engine.Candidate { 131 skills := make([]engine.CandidateSkill, len(cj.Skills)) 132 for i, s := range cj.Skills { 133 skills[i] = engine.CandidateSkill{Name: s} 134 } 135 return engine.Candidate{ 136 ID: id, 137 Clearance: engine.Clearance{ 138 Level: parseClearanceLevel(cj.ClearanceLevel), 139 Poly: parsePoly(cj.ClearancePoly), 140 Status: parseClearanceStatus(cj.ClearanceStatus), 141 }, 142 Skills: skills, 143 Certifications: cj.Certs, 144 CustomerExperience: cj.CustomerAgencies, 145 YearsExperience: cj.YearsExperience, 146 Location: cj.Location, 147 Remote: cj.Remote, 148 Relocatable: cj.Relocatable, 149 Available: cj.Available, 150 } 151} 152 153// ToEngineRequirement converts a RequirementJSON to an engine.Requirement. 154func ToEngineRequirement(id string, rj RequirementJSON) engine.Requirement { 155 required := make([]engine.RequiredSkill, len(rj.RequiredSkills)) 156 for i, rs := range rj.RequiredSkills { 157 required[i] = engine.RequiredSkill{Name: rs.Name, Mandatory: rs.Mandatory} 158 } 159 return engine.Requirement{ 160 ID: id, 161 RequiredLevel: parseClearanceLevel(rj.RequiredLevel), 162 RequiredPoly: parsePoly(rj.RequiredPoly), 163 RequiredSkills: required, 164 RequiredCerts: rj.RequiredCerts, 165 CustomerAgency: rj.CustomerAgency, 166 RequiredYears: rj.RequiredYears, 167 WorkLocation: rj.WorkLocation, 168 RemoteAllowed: rj.RemoteAllowed, 169 } 170} 171 172// ---------- result types ---------- 173 174// Mismatch records one case where the engine band differed from the domain truth band. 175type Mismatch struct { 176 ID string 177 Lane string 178 Expected engine.Band 179 Scored engine.Band 180 Total string 181} 182 183// ConfusionMatrix is a 3x3 matrix indexed by [expected][scored], both using Band iota order 184// (Red=0, Yellow=1, Green=2). 185type ConfusionMatrix [3][3]int 186 187// Report is the output of Score. 188type Report struct { 189 Matrix ConfusionMatrix 190 Mismatches []Mismatch 191 LaneResults map[string]LaneResult 192 Total int 193 Correct int 194} 195 196// LaneResult summarises one lane. 197type LaneResult struct { 198 Total int 199 Correct int 200} 201 202// BandPrecision returns precision for the given band: TP / (TP + FP). 203// FP = other rows scored into this band. 204func (m ConfusionMatrix) BandPrecision(b engine.Band) float64 { 205 tp := m[b][b] 206 total := 0 207 for exp := 0; exp < 3; exp++ { 208 total += m[exp][b] 209 } 210 if total == 0 { 211 return 1.0 212 } 213 return float64(tp) / float64(total) 214} 215 216// BandRecall returns recall for the given band: TP / (TP + FN). 217// FN = this band's cases that were scored into other bands. 218func (m ConfusionMatrix) BandRecall(b engine.Band) float64 { 219 tp := m[b][b] 220 total := 0 221 for scored := 0; scored < 3; scored++ { 222 total += m[b][scored] 223 } 224 if total == 0 { 225 return 1.0 226 } 227 return float64(tp) / float64(total) 228} 229 230// ---------- loader ---------- 231 232// LoadLane reads all fixtures from <testdataDir>/<lane>/fixtures.json and sets the Lane field. 233func LoadLane(testdataDir, lane string) ([]Fixture, error) { 234 path := filepath.Join(testdataDir, lane, "fixtures.json") 235 data, err := os.ReadFile(path) 236 if err != nil { 237 return nil, fmt.Errorf("read %s: %w", path, err) 238 } 239 var fixtures []Fixture 240 if err := json.Unmarshal(data, &fixtures); err != nil { 241 return nil, fmt.Errorf("parse %s: %w", path, err) 242 } 243 for i := range fixtures { 244 fixtures[i].Lane = lane 245 } 246 return fixtures, nil 247} 248 249// ---------- scoring ---------- 250 251// Score runs every fixture through the frozen engine (nil skill matcher, exact only), compares 252// each result to the expected domain truth band, and returns a Report. 253func Score(fixtures []Fixture) Report { 254 r := Report{ 255 LaneResults: make(map[string]LaneResult), 256 } 257 258 ctx := context.Background() 259 for _, fx := range fixtures { 260 c := ToEngineCandidate(fx.ID, fx.Candidate) 261 req := ToEngineRequirement(fx.ID+"-req", fx.Requirement) 262 263 result := engine.ScoreCandidate(ctx, c, req, nil) 264 265 expected := parseBand(fx.Expected.Band) 266 r.Matrix[expected][result.Band]++ 267 r.Total++ 268 269 lane := r.LaneResults[fx.Lane] 270 lane.Total++ 271 272 if result.Band == expected { 273 r.Correct++ 274 lane.Correct++ 275 } else { 276 r.Mismatches = append(r.Mismatches, Mismatch{ 277 ID: fx.ID, 278 Lane: fx.Lane, 279 Expected: expected, 280 Scored: result.Band, 281 Total: result.Total.StringFixed(2), 282 }) 283 } 284 r.LaneResults[fx.Lane] = lane 285 } 286 287 return r 288} 289