Janady13/missionmatch / internal/modules/matching/eval/eval.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/matching/eval/eval.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	1e9249840a4302ec82e123b9c9cb6f75badea3e4
Branch	master
Path or archive member	internal/modules/matching/eval/eval.go
Original SHA-256	ecd0c1c9658f5303bd8c0cfa6b415d6b00b2f70b60b0902e3c484a1bf8d881c7
Git blob SHA	a7c88c06e7f4049e5597ae6575ed8b1c4e73e9ac
Reading-copy SHA-256	ecd0c1c9658f5303bd8c0cfa6b415d6b00b2f70b60b0902e3c484a1bf8d881c7

Original source ↗

Every recorded occurrence

These 29 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/matching/eval/eval.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/modules/matching/eval/eval.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓289 lines · 8.4 KB

// Package eval loads the labeled GovCon benchmark fixtures and scores them through the frozen
// engine, producing a band confusion matrix and per lane pass rates. It is the regression and
// fairness harness for the AEDT engine: a divergence between the domain truth band in a fixture
// and the engine band is a finding to investigate, not a test failure, because the fixture labels
// are independent expert judgments. With -strict the CLI exits nonzero when any mismatch is found,
// enabling a future CI gate once the fixture labels are customer-validated.
//
// No demographic/EEO fields appear in any fixture. The engine never receives them.
package eval

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/missionworkforce/missionmatch/internal/modules/matching/engine"
)

// ---------- JSON fixture types ----------

// CandidateJSON mirrors the candidate block in fixtures.json.
type CandidateJSON struct {
	ClearanceLevel  string   `json:"clearance_level"`
	ClearancePoly   string   `json:"clearance_poly"`
	ClearanceStatus string   `json:"clearance_status"`
	Skills          []string `json:"skills"`
	Certs           []string `json:"certs"`
	CustomerAgencies []string `json:"customer_agencies"`
	YearsExperience int      `json:"years_experience"`
	Location        string   `json:"location"`
	Remote          bool     `json:"remote"`
	Relocatable     bool     `json:"relocatable"`
	Available       bool     `json:"available"`
}

// RequiredSkillJSON mirrors one element of the required_skills array.
type RequiredSkillJSON struct {
	Name      string `json:"name"`
	Mandatory bool   `json:"mandatory"`
}

// RequirementJSON mirrors the requirement block in fixtures.json.
type RequirementJSON struct {
	RequiredLevel   string              `json:"required_level"`
	RequiredPoly    string              `json:"required_poly"`
	RequiredSkills  []RequiredSkillJSON `json:"required_skills"`
	RequiredCerts   []string            `json:"required_certs"`
	CustomerAgency  string              `json:"customer_agency"`
	RequiredYears   int                 `json:"required_years"`
	WorkLocation    string              `json:"work_location"`
	RemoteAllowed   bool                `json:"remote_allowed"`
}

// ExpectedJSON mirrors the expected block in fixtures.json.
type ExpectedJSON struct {
	Band          string   `json:"band"`
	Rationale     string   `json:"rationale"`
	MandatoryGaps []string `json:"mandatory_gaps,omitempty"`
}

// Fixture is one benchmark case: a candidate, a requirement, and the domain truth expected band.
type Fixture struct {
	ID          string          `json:"id"`
	Candidate   CandidateJSON   `json:"candidate"`
	Requirement RequirementJSON `json:"requirement"`
	Expected    ExpectedJSON    `json:"expected"`
	Lane        string          `json:"-"` // injected from the directory name after loading
}

// ---------- parsers (string codes to engine enum values) ----------

func parseClearanceLevel(s string) engine.ClearanceLevel {
	switch strings.ToLower(strings.TrimSpace(s)) {
	case "confidential":
		return engine.ClearanceConfidential
	case "secret":
		return engine.ClearanceSecret
	case "top_secret":
		return engine.ClearanceTopSecret
	case "ts_sci":
		return engine.ClearanceTSSCI
	case "none":
		return engine.ClearanceNone
	default:
		return engine.ClearanceUnknown
	}
}

func parsePoly(s string) engine.Polygraph {
	switch strings.ToLower(strings.TrimSpace(s)) {
	case "ci":
		return engine.PolyCI
	case "full_scope":
		return engine.PolyFullScope
	default:
		return engine.PolyNone
	}
}

func parseClearanceStatus(s string) engine.ClearanceStatus {
	switch strings.ToLower(strings.TrimSpace(s)) {
	case "active":
		return engine.StatusActive
	case "expired":
		return engine.StatusExpired
	case "inactive":
		return engine.StatusInactive
	default:
		return engine.StatusUnknown
	}
}

func parseBand(s string) engine.Band {
	switch strings.ToLower(strings.TrimSpace(s)) {
	case "green":
		return engine.BandGreen
	case "yellow":
		return engine.BandYellow
	default:
		return engine.BandRed
	}
}

// ---------- conversion: JSON fixture -> engine types ----------

// ToEngineCandidate converts a CandidateJSON to an engine.Candidate. The ID is the fixture ID.
func ToEngineCandidate(id string, cj CandidateJSON) engine.Candidate {
	skills := make([]engine.CandidateSkill, len(cj.Skills))
	for i, s := range cj.Skills {
		skills[i] = engine.CandidateSkill{Name: s}
	}
	return engine.Candidate{
		ID: id,
		Clearance: engine.Clearance{
			Level:  parseClearanceLevel(cj.ClearanceLevel),
			Poly:   parsePoly(cj.ClearancePoly),
			Status: parseClearanceStatus(cj.ClearanceStatus),
		},
		Skills:             skills,
		Certifications:     cj.Certs,
		CustomerExperience: cj.CustomerAgencies,
		YearsExperience:    cj.YearsExperience,
		Location:           cj.Location,
		Remote:             cj.Remote,
		Relocatable:        cj.Relocatable,
		Available:          cj.Available,
	}
}

// ToEngineRequirement converts a RequirementJSON to an engine.Requirement.
func ToEngineRequirement(id string, rj RequirementJSON) engine.Requirement {
	required := make([]engine.RequiredSkill, len(rj.RequiredSkills))
	for i, rs := range rj.RequiredSkills {
		required[i] = engine.RequiredSkill{Name: rs.Name, Mandatory: rs.Mandatory}
	}
	return engine.Requirement{
		ID:             id,
		RequiredLevel:  parseClearanceLevel(rj.RequiredLevel),
		RequiredPoly:   parsePoly(rj.RequiredPoly),
		RequiredSkills: required,
		RequiredCerts:  rj.RequiredCerts,
		CustomerAgency: rj.CustomerAgency,
		RequiredYears:  rj.RequiredYears,
		WorkLocation:   rj.WorkLocation,
		RemoteAllowed:  rj.RemoteAllowed,
	}
}

// ---------- result types ----------

// Mismatch records one case where the engine band differed from the domain truth band.
type Mismatch struct {
	ID       string
	Lane     string
	Expected engine.Band
	Scored   engine.Band
	Total    string
}

// ConfusionMatrix is a 3x3 matrix indexed by [expected][scored], both using Band iota order
// (Red=0, Yellow=1, Green=2).
type ConfusionMatrix [3][3]int

// Report is the output of Score.
type Report struct {
	Matrix      ConfusionMatrix
	Mismatches  []Mismatch
	LaneResults map[string]LaneResult
	Total       int
	Correct     int
}

// LaneResult summarises one lane.
type LaneResult struct {
	Total   int
	Correct int
}

// BandPrecision returns precision for the given band: TP / (TP + FP).
// FP = other rows scored into this band.
func (m ConfusionMatrix) BandPrecision(b engine.Band) float64 {
	tp := m[b][b]
	total := 0
	for exp := 0; exp < 3; exp++ {
		total += m[exp][b]
	}
	if total == 0 {
		return 1.0
	}
	return float64(tp) / float64(total)
}

// BandRecall returns recall for the given band: TP / (TP + FN).
// FN = this band's cases that were scored into other bands.
func (m ConfusionMatrix) BandRecall(b engine.Band) float64 {
	tp := m[b][b]
	total := 0
	for scored := 0; scored < 3; scored++ {
		total += m[b][scored]
	}
	if total == 0 {
		return 1.0
	}
	return float64(tp) / float64(total)
}

// ---------- loader ----------

// LoadLane reads all fixtures from <testdataDir>/<lane>/fixtures.json and sets the Lane field.
func LoadLane(testdataDir, lane string) ([]Fixture, error) {
	path := filepath.Join(testdataDir, lane, "fixtures.json")
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", path, err)
	}
	var fixtures []Fixture
	if err := json.Unmarshal(data, &fixtures); err != nil {
		return nil, fmt.Errorf("parse %s: %w", path, err)
	}
	for i := range fixtures {
		fixtures[i].Lane = lane
	}
	return fixtures, nil
}

// ---------- scoring ----------

// Score runs every fixture through the frozen engine (nil skill matcher, exact only), compares
// each result to the expected domain truth band, and returns a Report.
func Score(fixtures []Fixture) Report {
	r := Report{
		LaneResults: make(map[string]LaneResult),
	}

	ctx := context.Background()
	for _, fx := range fixtures {
		c := ToEngineCandidate(fx.ID, fx.Candidate)
		req := ToEngineRequirement(fx.ID+"-req", fx.Requirement)

		result := engine.ScoreCandidate(ctx, c, req, nil)

		expected := parseBand(fx.Expected.Band)
		r.Matrix[expected][result.Band]++
		r.Total++

		lane := r.LaneResults[fx.Lane]
		lane.Total++

		if result.Band == expected {
			r.Correct++
			lane.Correct++
		} else {
			r.Mismatches = append(r.Mismatches, Mismatch{
				ID:       fx.ID,
				Lane:     fx.Lane,
				Expected: expected,
				Scored:   result.Band,
				Total:    result.Total.StringFixed(2),
			})
		}
		r.LaneResults[fx.Lane] = lane
	}

	return r
}