Janady13/missionmatch / internal/integration/retrieval_recall_integration_test.go

1//go:build integration 2 3// Retrieval recall regression gate. This is the FIRST evaluation of whether the hybrid search RECALLS the 4// right people, not just whether the engine bands score. It loads a versioned gold set (testdata 5// retrieval_qrels.json), seeds a fresh per run org with those candidates embedded through the REAL inference 6// plane (so the vector leg is meaningful, a deterministic hash embedding would make recall arbitrary), then 7// for each query embeds the query through the SAME plane, runs the production ai.HybridSearch under RLS, maps 8// the returned CandidateIDs back to the seed keys, and computes recall at k, MRR, and nDCG against the qrels 9// with the pure internal/platform/eval metrics. It logs every metric per query and the mean as a readable 10// regression baseline, and asserts conservative floors a correct hybrid search clears comfortably while a real 11// regression (in embeddings, HNSW tuning, the mission gate, or the RRF k constant) would breach. 12// 13// This is MEASUREMENT ONLY. It does not touch retrieval.go, the RRF math, or the frozen matching engine. It is 14// gated on MM_MEGAMIND_URL and skips cleanly when it is unset, exactly like the sibling brain tests, so the 15// orchestrator must set the gateway URL to run it on staging. 16package integration 17 18import ( 19 "context" 20 "encoding/json" 21 "math" 22 "os" 23 "path/filepath" 24 "testing" 25 "time" 26 27 "github.com/pgvector/pgvector-go" 28 29 "github.com/missionworkforce/missionmatch/internal/modules/ai" 30 "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" 31 "github.com/missionworkforce/missionmatch/internal/platform/eval" 32 "github.com/missionworkforce/missionmatch/internal/platform/inference" 33) 34 35// qrelsCandidate is one gold set candidate with the stable seed key the harness maps a returned id back to. 36type qrelsCandidate struct { 37 Key string `json:"key"` 38 FullName string `json:"full_name"` 39 ClearanceLevel string `json:"clearance_level"` 40 ClearancePoly string `json:"clearance_poly"` 41 ClearanceStatus string `json:"clearance_status"` 42 Location string `json:"location"` 43 Headline string `json:"headline"` 44} 45 46// qrelsQuery is one query with the seed keys judged relevant to it. 47type qrelsQuery struct { 48 ID string `json:"id"` 49 Text string `json:"text"` 50 Relevant []string `json:"relevant"` 51} 52 53type qrelsFixture struct { 54 Version string `json:"version"` 55 Candidates []qrelsCandidate `json:"candidates"` 56 Queries []qrelsQuery `json:"queries"` 57} 58 59func loadQrels(t *testing.T) qrelsFixture { 60 t.Helper() 61 raw, err := os.ReadFile(filepath.Join("testdata", "retrieval_qrels.json")) 62 if err != nil { 63 t.Fatalf("read qrels fixture: %v", err) 64 } 65 var fx qrelsFixture 66 if err := json.Unmarshal(raw, &fx); err != nil { 67 t.Fatalf("parse qrels fixture: %v", err) 68 } 69 if fx.Version == "" || len(fx.Candidates) == 0 || len(fx.Queries) == 0 { 70 t.Fatalf("qrels fixture is incomplete: version=%q candidates=%d queries=%d", fx.Version, len(fx.Candidates), len(fx.Queries)) 71 } 72 return fx 73} 74 75// retrievalPlane builds an inference client to the live MEGAMIND plane, or skips when MM_MEGAMIND_URL is unset. 76// The model and dimension match the production embedder so stored and query vectors share one space. 77func retrievalPlane(t *testing.T) *inference.Client { 78 t.Helper() 79 url := os.Getenv("MM_MEGAMIND_URL") 80 if url == "" { 81 t.Skip("MM_MEGAMIND_URL not set; skipping the retrieval recall harness that needs the real embedding plane") 82 } 83 c, err := inference.New(inference.Config{ 84 BaseURL: url, EmbedModel: "qllama/bge-small-en-v1.5", GenerateModel: "llama3.1:8b", 85 Dimension: 384, Timeout: 60 * time.Second, MaxRetries: 1, 86 // The gateway requires a bearer on every route but the health check. Omitting it worked only while 87 // MM_MEGAMIND_URL pointed at the token injecting proxy, so the suite proved the PROXY authenticates. 88 Token: os.Getenv("MM_MEGAMIND_TOKEN"), 89 }) 90 if err != nil { 91 t.Fatalf("inference client: %v", err) 92 } 93 return c 94} 95 96// normalizeVec L2 normalizes a vector, exactly as embed backfill does before storing, so a seeded vector 97// matches the backfill convention and cosine is a clean dot product. 98func normalizeVec(v []float32) []float32 { 99 var s float64 100 for _, x := range v { 101 s += float64(x) * float64(x) 102 } 103 if s == 0 { 104 return v 105 } 106 inv := float32(1.0 / math.Sqrt(s)) 107 out := make([]float32, len(v)) 108 for i, x := range v { 109 out[i] = x * inv 110 } 111 return out 112} 113 114func embedOne(t *testing.T, client *inference.Client, text string) []float32 { 115 t.Helper() 116 resp, err := client.Embed(context.Background(), []string{text}) 117 if err != nil { 118 t.Fatalf("embed %q: %v", text, err) 119 } 120 if len(resp.Embeddings) != 1 || len(resp.Embeddings[0]) != 384 { 121 t.Fatalf("embed %q returned wrong shape", text) 122 } 123 return normalizeVec(resp.Embeddings[0]) 124} 125 126// TestRetrievalRecallGate is the regression gate. It seeds the gold set candidates into a fresh org with real 127// plane embeddings, runs HybridSearch per query, and asserts the mean ranking metrics clear conservative 128// floors. The gold set is constructed so each query's relevant candidates are both lexically present (the full 129// text leg finds them) and semantically near the query (the vector leg finds them), so a correct hybrid search 130// clears the floors comfortably and this is a gate, not a flaky test. 131func TestRetrievalRecallGate(t *testing.T) { 132 client := retrievalPlane(t) 133 fx := loadQrels(t) 134 org := seedOrg(t) 135 ctx := ctxOrg(org) 136 137 // Seed every gold set candidate with its real plane embedding, mapping each seeded id to its stable key so 138 // a returned CandidateID can be resolved back to the key the qrels speak in. Seeding is RLS scoped to this 139 // fresh org (the candidates rows carry organization_id = org; the search runs in the same org context). 140 idToKey := make(map[string]string, len(fx.Candidates)) 141 for _, c := range fx.Candidates { 142 // Embed the SAME text the candidate is findable by (name plus headline), the field the search_tsv and 143 // the resume vector both draw from, so the vector leg recalls the candidate the qrels expects. 144 vec := embedOne(t, client, c.FullName+" "+c.Headline) 145 pv := pgvector.NewVector(vec) 146 hn, loc := c.Headline, c.Location 147 created, err := dbgen.New(migrator).CreateCandidate(context.Background(), dbgen.CreateCandidateParams{ 148 OrganizationID: org, 149 FullName: c.FullName, 150 Headline: &hn, 151 ClearanceLevel: c.ClearanceLevel, 152 ClearancePoly: c.ClearancePoly, 153 ClearanceStatus: c.ClearanceStatus, 154 Location: &loc, 155 Available: true, 156 ResumeEmbedding: &pv, 157 }) 158 if err != nil { 159 t.Fatalf("seed gold candidate %q: %v", c.Key, err) 160 } 161 idToKey[created.ID.String()] = c.Key 162 } 163 164 const ( 165 k5 = 5 166 k10 = 10 167 // limit pulls a generous depth so recall at 10 is measured over the real ranking, not truncated by a 168 // tight limit. The gold set has twelve candidates, so twenty surfaces them all. 169 searchLimit = 20 170 ) 171 172 var sumRecall5, sumRecall10, sumMRR, sumNDCG10 float64 173 nQueries := 0 174 175 for _, q := range fx.Queries { 176 relevant := make(map[string]bool, len(q.Relevant)) 177 for _, key := range q.Relevant { 178 relevant[key] = true 179 } 180 181 // Run the PRODUCTION hybrid search with an empty mission (the default unscoped path, byte for byte the 182 // shipped behavior), RLS scoped to this org. The query embeds through the SAME plane as the seed 183 // vectors, so the vector leg compares like with like. 184 hits, err := ai.HybridSearch(ctx, appDB, client, q.Text, "", searchLimit) 185 if err != nil { 186 t.Fatalf("query %q hybrid search: %v", q.ID, err) 187 } 188 189 // Map the returned CandidateIDs back to seed keys, preserving rank order. A returned id that is not a 190 // gold candidate (there is none here, the org holds only the gold set) maps to an empty key, which is 191 // never relevant, so it neither helps nor hurts the metrics. 192 ranked := make([]string, 0, len(hits)) 193 for _, h := range hits { 194 ranked = append(ranked, idToKey[h.CandidateID]) 195 } 196 197 r5 := eval.RecallAtK(ranked, relevant, k5) 198 r10 := eval.RecallAtK(ranked, relevant, k10) 199 mrr := eval.MRR(ranked, relevant) 200 ndcg10 := eval.NDCGAtK(ranked, relevant, k10) 201 202 t.Logf("query %-26s recall@5=%.3f recall@10=%.3f mrr=%.3f ndcg@10=%.3f (relevant=%d, hits=%d)", 203 q.ID, r5, r10, mrr, ndcg10, len(q.Relevant), len(hits)) 204 205 sumRecall5 += r5 206 sumRecall10 += r10 207 sumMRR += mrr 208 sumNDCG10 += ndcg10 209 nQueries++ 210 } 211 212 meanRecall5 := sumRecall5 / float64(nQueries) 213 meanRecall10 := sumRecall10 / float64(nQueries) 214 meanMRR := sumMRR / float64(nQueries) 215 meanNDCG10 := sumNDCG10 / float64(nQueries) 216 217 t.Logf("MEAN over %d queries (gold set v%s): recall@5=%.3f recall@10=%.3f mrr=%.3f ndcg@10=%.3f", 218 nQueries, fx.Version, meanRecall5, meanRecall10, meanMRR, meanNDCG10) 219 220 // Floors. Each gold query's relevant candidates are lexically present AND semantically near the query, and 221 // the org holds only twelve candidates, so a correct hybrid search ranks the relevant ones at or near the 222 // top and recall at ten is essentially complete. The floors sit well below the expected healthy values so 223 // a passing run is unambiguous, while a real regression (embeddings drift, HNSW ef_search cut, a broken 224 // mission gate, or an RRF k constant change that scrambles the fusion) drops a metric below its floor. 225 // 226 // meanRecall@10 >= 0.8: a healthy search recalls nearly every relevant id within the top ten of a twelve 227 // candidate pool, so 0.8 is a conservative floor that a correct search clears comfortably and a recall 228 // regression breaks. 229 if meanRecall10 < 0.8 { 230 t.Fatalf("REGRESSION: mean recall@10 = %.3f is below the 0.8 floor; the hybrid search is no longer recalling the relevant candidates", meanRecall10) 231 } 232 // meanMRR >= 0.5: a healthy search puts a relevant candidate in the top two on average (rank one is 1.0, 233 // rank two is 0.5), so 0.5 is a conservative floor on first relevant rank that a ranking regression breaks. 234 if meanMRR < 0.5 { 235 t.Fatalf("REGRESSION: mean MRR = %.3f is below the 0.5 floor; the first relevant candidate is ranking too deep", meanMRR) 236 } 237 // meanNDCG@10 >= 0.6: nDCG rewards ranking the relevant ids high, not merely present. 0.6 is a conservative 238 // floor that a correct ordering clears while a fusion or ranking regression that buries relevant ids breaks. 239 if meanNDCG10 < 0.6 { 240 t.Fatalf("REGRESSION: mean nDCG@10 = %.3f is below the 0.6 floor; relevant candidates are ranking too low in the fused order", meanNDCG10) 241 } 242 // recall@5 is logged as a baseline and sanity checked but not floored as tightly, since a five deep cutoff 243 // over a twelve candidate pool is the harder bar; it must still recall a clear majority on average. 244 if meanRecall5 < 0.6 { 245 t.Fatalf("REGRESSION: mean recall@5 = %.3f is below the 0.6 floor; the top five no longer surfaces the relevant candidates", meanRecall5) 246 } 247 248 // Guard the fixture itself against silent identity collisions: every seeded id resolved to a distinct key. 249 if len(idToKey) != len(fx.Candidates) { 250 t.Fatalf("seed key mapping is inconsistent: %d ids for %d candidates", len(idToKey), len(fx.Candidates)) 251 } 252} 253