Janady13/missionmatch / internal/integration/calibration_test.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/integration/calibration_test.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/integration/calibration_test.go
Original SHA-256	3e9801ccabbb0b102e115ceb845a375e5c9e5808aa64be8085ce5e97b125f1d8
Git blob SHA	b617a886181f4f87a3de9c23e03c42ef4c8e97f0
Reading-copy SHA-256	3e9801ccabbb0b102e115ceb845a375e5c9e5808aa64be8085ce5e97b125f1d8

Original source ↗

Every recorded occurrence

These 23 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/integration/calibration_test.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/integration/calibration_test.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓160 lines · 7.1 KB

//go:build integration

package integration

import (
	"fmt"
	"sort"
	"testing"

	"github.com/missionworkforce/missionmatch/internal/modules/matching/engine"
	"github.com/missionworkforce/missionmatch/internal/platform/inference"
)

// skillPair is one labeled comparison: two skill names that either SHOULD match semantically
// (synonyms or adjacent skills, the engine must count them as a met skill) or SHOULD NOT
// (unrelated, the engine must leave the skill unmet). It is the calibration unit.
type skillPair struct {
	a, b string
}

// synonymPairs are the should match group: real cleared workforce skills written under two
// different but synonymous or directly adjacent names. On a correctly calibrated threshold every
// one of these resolves as a semantic match, so a candidate who lists "AWS Cloud Platform" still
// satisfies a requirement that asks for "Amazon Web Services".
var synonymPairs = []skillPair{
	{"Amazon Web Services", "AWS Cloud Platform"},
	{"Kubernetes", "Container Orchestration"},
	{"Penetration Testing", "Ethical Hacking"},
	{"Incident Response", "Security Incident Handling"},
	{"Python Programming", "Python Software Development"},
	{"Threat Hunting", "Threat Detection"},
	{"Linux System Administration", "Unix System Administration"},
	{"Data Analysis", "Data Analytics"},
	{"Signals Intelligence", "SIGINT"},
	{"Vulnerability Assessment", "Vulnerability Scanning"},
	{"Network Engineering", "Network Infrastructure Engineering"},
	{"Cloud Security", "Cloud Security Engineering"},
}

// unrelatedPairs are the should not match group: a real cleared workforce skill paired with an
// unrelated skill from another domain entirely. On a correctly calibrated threshold none of these
// resolves as a match, so the engine never invents a skill a candidate does not hold.
var unrelatedPairs = []skillPair{
	{"Amazon Web Services", "Underwater Basket Weaving"},
	{"Kubernetes", "Pastry Baking"},
	{"Penetration Testing", "Floral Arrangement"},
	{"Incident Response", "Landscape Gardening"},
	{"Python Programming", "Ballroom Dancing"},
	{"Threat Hunting", "Oil Painting"},
	{"Linux System Administration", "Marine Biology"},
	{"Data Analysis", "Automotive Repair"},
	{"Signals Intelligence", "Creative Writing"},
	{"Vulnerability Assessment", "Wedding Photography"},
	{"Network Engineering", "Pottery"},
	{"Cloud Security", "Beekeeping"},
}

// cosine embeds both names through the live bge plane, normalizes exactly as the backfill does (so
// the result equals the stored vectors' cosine), and returns the dot product, which for L2
// normalized vectors is the cosine similarity.
func cosine(t *testing.T, client *inference.Client, p skillPair) float64 {
	t.Helper()
	va, vb := embedNorm(t, client, p.a), embedNorm(t, client, p.b)
	var s float64
	for i := range va {
		s += float64(va[i]) * float64(vb[i])
	}
	return s
}

// TestSemanticThresholdCalibration is the calibration harness. It scores every known should match
// (synonym) and should not (unrelated) skill pair on the live bge-small vectors, reports the full
// cosine distribution of each group, and asserts the engine's DefaultSemanticThreshold separates
// the two groups with margin: it sits strictly below the lowest true synonym AND strictly above
// the highest unrelated pair, with a guard band on each side. This is the evidence behind the
// chosen threshold; if a future model or skill set narrows the gap, this test fails loudly rather
// than letting a miscalibrated threshold silently discard real matches or invent false ones.
func TestSemanticThresholdCalibration(t *testing.T) {
	client := testPlane(t)

	synSims := make([]float64, 0, len(synonymPairs))
	for _, p := range synonymPairs {
		s := cosine(t, client, p)
		synSims = append(synSims, s)
		t.Logf("SYNONYM   %-32s ~ %-34s cosine=%.4f", p.a, p.b, s)
	}
	unrSims := make([]float64, 0, len(unrelatedPairs))
	for _, p := range unrelatedPairs {
		s := cosine(t, client, p)
		unrSims = append(unrSims, s)
		t.Logf("UNRELATED %-32s ~ %-34s cosine=%.4f", p.a, p.b, s)
	}

	synMin, synMax, synMean := stats(synSims)
	unrMin, unrMax, unrMean := stats(unrSims)
	t.Logf("SYNONYM group:   min=%.4f mean=%.4f max=%.4f (n=%d)", synMin, synMean, synMax, len(synSims))
	t.Logf("UNRELATED group: min=%.4f mean=%.4f max=%.4f (n=%d)", unrMin, unrMean, unrMax, len(unrSims))

	thr, _ := engine.DefaultSemanticThreshold.Float64()
	t.Logf("engine DefaultSemanticThreshold = %.4f (ModelVersion %s)", thr, engine.ModelVersion)

	// The decision the calibration must protect is the confusion at the live threshold, not a clean
	// full separation: on a 384 dim model, an acronym (SIGINT) sits below its expansion AND below the
	// unrelated ceiling, so NO single cosine cutoff can capture it; that case belongs to the ontology
	// alias layer. The threshold's job is therefore to admit every GENUINE synonym while admitting NO
	// unrelated pair.
	var falsePos, trueNeg int
	for i, s := range unrSims {
		if s >= thr {
			falsePos++
			t.Logf("FALSE POSITIVE at threshold %.4f: %q ~ %q cosine=%.4f", thr, unrelatedPairs[i].a, unrelatedPairs[i].b, s)
		} else {
			trueNeg++
		}
	}
	var truePos int
	var synMisses []string
	for i, s := range synSims {
		if s >= thr {
			truePos++
		} else {
			synMisses = append(synMisses, fmt.Sprintf("%q~%q=%.4f", synonymPairs[i].a, synonymPairs[i].b, s))
		}
	}
	t.Logf("CONFUSION at threshold %.4f: synonyms matched %d/%d, unrelated rejected %d/%d", thr, truePos, len(synSims), trueNeg, len(unrSims))
	if len(synMisses) > 0 {
		t.Logf("synonym misses (acronym or initialism class, served by the ontology alias layer, not the cosine cutoff): %v", synMisses)
	}

	// HARD: no unrelated pair may leak through. A false positive means the engine would invent a skill
	// the candidate does not hold, the unacceptable failure for an AEDT tool.
	if falsePos != 0 {
		t.Fatalf("CALIBRATION: threshold %.4f admits %d unrelated pair(s) as matches; raise DefaultSemanticThreshold above the unrelated ceiling %.4f", thr, falsePos, unrMax)
	}
	// HARD: the threshold must clear the unrelated ceiling with margin, so embedding noise cannot push
	// an unrelated pair over the line.
	const minMargin = 0.03
	if thr < unrMax+minMargin {
		t.Fatalf("CALIBRATION: threshold %.4f has too little margin over the unrelated ceiling %.4f (need >= %.4f); raise DefaultSemanticThreshold", thr, unrMax, unrMax+minMargin)
	}
	// HARD: at most the acronym class may fall below the threshold; every other synonym must match, or
	// the cutoff is too high and is discarding real matches (the exact defect this recalibration fixed).
	if len(synMisses) > 1 {
		t.Fatalf("CALIBRATION: threshold %.4f discards %d synonym pairs (only the acronym class is allowed to miss): %v; lower DefaultSemanticThreshold", thr, len(synMisses), synMisses)
	}
	t.Logf("OK: threshold %.4f admits every genuine synonym and no unrelated pair (margin %.4f over the unrelated ceiling)", thr, thr-unrMax)
}

// stats returns min, max, mean of a non empty slice.
func stats(xs []float64) (mn, mx, mean float64) {
	sorted := append([]float64(nil), xs...)
	sort.Float64s(sorted)
	mn, mx = sorted[0], sorted[len(sorted)-1]
	var sum float64
	for _, x := range xs {
		sum += x
	}
	return mn, mx, sum / float64(len(xs))
}