Janady13/missionmatch / cmd/eval/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/eval/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/eval/main.go
Original SHA-256	e33551af74edef8f434750c5d10a361859bbbfbbe72abc7f5b4fee9155df040a
Git blob SHA	13dc5c6c88fc80b1a7b5a8cfe09000c3976d431c
Reading-copy SHA-256	e33551af74edef8f434750c5d10a361859bbbfbbe72abc7f5b4fee9155df040a

Original source ↗

Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/eval/main.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / cmd/eval/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original

Recovered source transcript

Download complete source text ↓269 lines · 7.6 KB

// cmd/eval scores the GovCon benchmark fixtures through the frozen engine and prints a band
// confusion matrix, per band precision and recall, and per lane pass rates. Mismatches between
// the domain truth band and the engine band are findings to investigate, not failures.
//
// Usage:
//
//	eval [-lanes intelligence_analyst,cyber_architect] [-testdata path] [-verbose] [-json] [-strict]
//
// Exit codes: 0 always (reporting mode). With -strict: exit 1 if any band mismatches found.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/missionworkforce/missionmatch/internal/modules/matching/engine"
	"github.com/missionworkforce/missionmatch/internal/modules/matching/eval"
)

func main() {
	lanesFlag := flag.String("lanes", "", "comma separated lane names to evaluate (default: all from manifest)")
	testdataFlag := flag.String("testdata", "", "path to the versioned testdata directory (default: internal/modules/matching/testdata/v1.3.0)")
	verboseFlag := flag.Bool("verbose", false, "print per fixture result lines")
	jsonFlag := flag.Bool("json", false, "emit JSON output instead of human readable text")
	strictFlag := flag.Bool("strict", false, "exit 1 if any band mismatches are found (CI gate)")
	flag.Parse()

	testdataDir := *testdataFlag
	if testdataDir == "" {
		// Resolve relative to the repo root via the module path convention: walk up from the binary.
		// In practice callers run this via go run from the repo root.
		testdataDir = filepath.Join("internal", "modules", "matching", "testdata", "v1.3.0")
	}

	// Load manifest.
	manifestPath := filepath.Join(testdataDir, "manifest.json")
	manifestBytes, err := os.ReadFile(manifestPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "eval: read manifest: %v\n", err)
		os.Exit(2)
	}
	var manifest struct {
		Version string   `json:"version"`
		Lanes   []string `json:"lanes"`
	}
	if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
		fmt.Fprintf(os.Stderr, "eval: parse manifest: %v\n", err)
		os.Exit(2)
	}

	// Determine which lanes to evaluate.
	lanes := manifest.Lanes
	if *lanesFlag != "" {
		lanes = strings.Split(*lanesFlag, ",")
		for i, l := range lanes {
			lanes[i] = strings.TrimSpace(l)
		}
	}

	// Load all fixtures.
	var all []eval.Fixture
	for _, lane := range lanes {
		fxs, err := eval.LoadLane(testdataDir, lane)
		if err != nil {
			fmt.Fprintf(os.Stderr, "eval: load lane %s: %v\n", lane, err)
			os.Exit(2)
		}
		all = append(all, fxs...)
	}

	// Score through the frozen engine.
	report := eval.Score(all)

	if *jsonFlag {
		emitJSON(report, lanes)
	} else {
		emitText(report, lanes, *verboseFlag, all)
	}

	if *strictFlag && len(report.Mismatches) > 0 {
		os.Exit(1)
	}
}

// ---------- text output ----------

func emitText(r eval.Report, lanes []string, verbose bool, fixtures []eval.Fixture) {
	fmt.Println()
	fmt.Printf("missionMATCH band confusion matrix  (engine %s)\n", engine.ModelVersion)
	fmt.Println(strings.Repeat("=", 60))
	fmt.Printf("Lanes evaluated: %s\n", strings.Join(lanes, ", "))
	fmt.Printf("Total fixtures:  %d\n", r.Total)
	fmt.Printf("Agreement:       %d / %d  (%.1f%%)\n", r.Correct, r.Total, pct(r.Correct, r.Total))
	fmt.Println()

	// Confusion matrix.
	fmt.Println("Confusion matrix (rows=expected, cols=scored)")
	fmt.Printf("%-10s  %-8s  %-8s  %-8s\n", "Expected", "Red", "Yellow", "Green")
	fmt.Println(strings.Repeat("-", 42))
	bandNames := []string{"Red", "Yellow", "Green"}
	bands := []engine.Band{engine.BandRed, engine.BandYellow, engine.BandGreen}
	for ei, exp := range bands {
		fmt.Printf("%-10s  %-8d  %-8d  %-8d\n",
			bandNames[ei],
			r.Matrix[exp][engine.BandRed],
			r.Matrix[exp][engine.BandYellow],
			r.Matrix[exp][engine.BandGreen],
		)
	}
	fmt.Println()

	// Precision and recall.
	fmt.Printf("%-10s  %-12s  %-12s\n", "Band", "Precision", "Recall")
	fmt.Println(strings.Repeat("-", 38))
	for i, b := range bands {
		fmt.Printf("%-10s  %-12s  %-12s\n",
			bandNames[i],
			fmtPct(r.Matrix.BandPrecision(b)),
			fmtPct(r.Matrix.BandRecall(b)),
		)
	}
	fmt.Println()

	// Per lane pass rates.
	fmt.Printf("%-28s  %-8s  %-10s\n", "Lane", "Pass", "Rate")
	fmt.Println(strings.Repeat("-", 50))
	for _, lane := range lanes {
		lr := r.LaneResults[lane]
		fmt.Printf("%-28s  %d / %-5d  %.1f%%\n", lane, lr.Correct, lr.Total, pct(lr.Correct, lr.Total))
	}

	// Mismatches.
	if len(r.Mismatches) == 0 {
		fmt.Println()
		fmt.Println("No band mismatches. All domain truth labels agree with the engine.")
	} else {
		fmt.Println()
		fmt.Printf("Band mismatches (%d):\n", len(r.Mismatches))
		fmt.Println(strings.Repeat("-", 60))
		fmt.Printf("%-20s  %-14s  %-10s  %-10s  %-8s\n", "ID", "Lane", "Expected", "Scored", "Total")
		for _, mm := range r.Mismatches {
			fmt.Printf("%-20s  %-14s  %-10s  %-10s  %-8s\n",
				mm.ID, mm.Lane, mm.Expected.String(), mm.Scored.String(), mm.Total)
		}
	}

	if verbose && len(fixtures) > 0 {
		fmt.Println()
		fmt.Println("Per fixture results:")
		fmt.Println(strings.Repeat("-", 70))
		fmt.Printf("%-20s  %-14s  %-10s  %-8s\n", "ID", "Lane", "Expected", "Agree")
		for _, fx := range fixtures {
			agreed := "yes"
			for _, mm := range r.Mismatches {
				if mm.ID == fx.ID {
					agreed = "MISMATCH"
					break
				}
			}
			fmt.Printf("%-20s  %-14s  %-10s  %s\n", fx.ID, fx.Lane, fx.Expected.Band, agreed)
		}
	}

	fmt.Println()
}

// ---------- JSON output ----------

type jsonOutput struct {
	Version    string              `json:"engine_version"`
	Lanes      []string            `json:"lanes"`
	Total      int                 `json:"total"`
	Correct    int                 `json:"correct"`
	Matrix     [3][3]int           `json:"confusion_matrix"`
	BandStats  []jsonBandStat      `json:"band_stats"`
	Lanes2     map[string]laneJSON `json:"lane_results"`
	Mismatches []jsonMismatch      `json:"mismatches"`
}

type jsonBandStat struct {
	Band      string  `json:"band"`
	Precision float64 `json:"precision"`
	Recall    float64 `json:"recall"`
}

type laneJSON struct {
	Total   int     `json:"total"`
	Correct int     `json:"correct"`
	PassPct float64 `json:"pass_pct"`
}

type jsonMismatch struct {
	ID       string `json:"id"`
	Lane     string `json:"lane"`
	Expected string `json:"expected"`
	Scored   string `json:"scored"`
	Total    string `json:"total"`
}

func emitJSON(r eval.Report, lanes []string) {
	bands := []engine.Band{engine.BandRed, engine.BandYellow, engine.BandGreen}
	bandNames := []string{"Red", "Yellow", "Green"}

	stats := make([]jsonBandStat, 3)
	for i, b := range bands {
		stats[i] = jsonBandStat{
			Band:      bandNames[i],
			Precision: r.Matrix.BandPrecision(b),
			Recall:    r.Matrix.BandRecall(b),
		}
	}

	laneMap := make(map[string]laneJSON, len(lanes))
	for _, lane := range lanes {
		lr := r.LaneResults[lane]
		laneMap[lane] = laneJSON{
			Total:   lr.Total,
			Correct: lr.Correct,
			PassPct: pct(lr.Correct, lr.Total),
		}
	}

	mis := make([]jsonMismatch, len(r.Mismatches))
	for i, mm := range r.Mismatches {
		mis[i] = jsonMismatch{
			ID:       mm.ID,
			Lane:     mm.Lane,
			Expected: mm.Expected.String(),
			Scored:   mm.Scored.String(),
			Total:    mm.Total,
		}
	}

	out := jsonOutput{
		Version:    engine.ModelVersion,
		Lanes:      lanes,
		Total:      r.Total,
		Correct:    r.Correct,
		Matrix:     r.Matrix,
		BandStats:  stats,
		Lanes2:     laneMap,
		Mismatches: mis,
	}

	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", "  ")
	if err := enc.Encode(out); err != nil {
		fmt.Fprintf(os.Stderr, "eval: json encode: %v\n", err)
		os.Exit(2)
	}
}

// ---------- helpers ----------

func pct(n, d int) float64 {
	if d == 0 {
		return 100.0
	}
	return float64(n) / float64(d) * 100.0
}

func fmtPct(f float64) string {
	return fmt.Sprintf("%.1f%%", f*100.0)
}