Janady13/missionmatch / internal/integration/retrieval_recall_integration_test.go

1//go:build integration 2 3// Retrieval recall regression gate. This is the FIRST evaluation of whether the hybrid search RECALLS the 4// right people, not just whether the engine bands score. It loads a versioned gold set (testdata 5// retrieval_qrels.json), seeds a fresh per run org with those candidates embedded through the REAL inference 6// plane (so the vector leg is meaningful, a deterministic hash embedding would make recall arbitrary), then 7// for each query embeds the query through the SAME plane, runs the production ai.HybridSearch under RLS, maps 8// the returned CandidateIDs back to the seed keys, and computes recall at k, MRR, and nDCG against the qrels 9// with the pure internal/platform/eval metrics. It logs every metric per query and the mean as a readable 10// regression baseline, and asserts conservative floors a correct hybrid search clears comfortably while a real 11// regression (in embeddings, HNSW tuning, the mission gate, or the RRF k constant) would breach. 12// 13// This is MEASUREMENT ONLY. It does not touch retrieval.go, the RRF math, or the frozen matching engine. It is 14// gated on MM_MEGAMIND_URL and skips cleanly when it is unset, exactly like the sibling brain tests, so the 15// orchestrator must set the gateway URL to run it on staging. 16package integration 17 18import ( 19 "context" 20 "encoding/json" 21 "math" 22 "os" 23 "path/filepath" 24 "testing" 25 "time" 26 27 "github.com/pgvector/pgvector-go" 28 29 "github.com/missionworkforce/missionmatch/internal/modules/ai" 30 "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" 31 "github.com/missionworkforce/missionmatch/internal/platform/eval" 32 "github.com/missionworkforce/missionmatch/internal/platform/inference" 33) 34 35// qrelsCandidate is one gold set candidate with the stable seed key the harness maps a returned id back to. 36type qrelsCandidate struct { 37 Key string `json:"key"` 38 FullName string `json:"full_name"` 39 ClearanceLevel string `json:"clearance_level"` 40 ClearancePoly string `json:"clearance_poly"` 41 ClearanceStatus string `json:"clearance_status"` 42 Location string `json:"location"` 43 Headline string `json:"headline"` 44} 45 46// qrelsQuery is one query with the seed keys judged relevant to it. 47type qrelsQuery struct { 48 ID string `json:"id"` 49 Text string `json:"text"` 50 Relevant []string `json:"relevant"` 51} 52 53type qrelsFixture struct { 54 Version string `json:"version"` 55 Candidates []qrelsCandidate `json:"candidates"` 56 Queries []qrelsQuery `json:"queries"` 57} 58 59func loadQrels(t *testing.T) qrelsFixture { 60 t.Helper() 61 raw, err := os.ReadFile(filepath.Join("testdata", "retrieval_qrels.json")) 62 if err != nil { 63 t.Fatalf("read qrels fixture: %v", err) 64 } 65 var fx qrelsFixture 66 if err := json.Unmarshal(raw, &fx); err != nil { 67 t.Fatalf("parse qrels fixture: %v", err) 68 } 69 if fx.Version == "" || len(fx.Candidates) == 0 || len(fx.Queries) == 0 { 70 t.Fatalf("qrels fixture is incomplete: version=%q candidates=%d queries=%d", fx.Version, len(fx.Candidates), len(fx.Queries)) 71 } 72 return fx 73} 74 75// retrievalPlane builds an inference client to the live MEGAMIND plane, or skips when MM_MEGAMIND_URL is unset. 76// The model and dimension match the production embedder so stored and query vectors share one space. 77func retrievalPlane(t *testing.T) *inference.Client { 78 t.Helper() 79 url := os.Getenv("MM_MEGAMIND_URL") 80 if url == "" { 81 t.Skip("MM_MEGAMIND_URL not set; skipping the retrieval recall harness that needs the real embedding plane") 82 } 83 c, err := inference.New(inference.Config{ 84 BaseURL: url, EmbedModel: "qllama/bge-small-en-v1.5", GenerateModel: "llama3.1:8b", 85 Dimension: 384, Timeout: 60 * time.Second, MaxRetries: 1, 86 }) 87 if err != nil { 88 t.Fatalf("inference client: %v", err) 89 } 90 return c 91} 92 93// normalizeVec L2 normalizes a vector, exactly as embed backfill does before storing, so a seeded vector 94// matches the backfill convention and cosine is a clean dot product. 95func normalizeVec(v []float32) []float32 { 96 var s float64 97 for _, x := range v { 98 s += float64(x) * float64(x) 99 } 100 if s == 0 { 101 return v 102 } 103 inv := float32(1.0 / math.Sqrt(s)) 104 out := make([]float32, len(v)) 105 for i, x := range v { 106 out[i] = x * inv 107 } 108 return out 109} 110 111func embedOne(t *testing.T, client *inference.Client, text string) []float32 { 112 t.Helper() 113 resp, err := client.Embed(context.Background(), []string{text}) 114 if err != nil { 115 t.Fatalf("embed %q: %v", text, err) 116 } 117 if len(resp.Embeddings) != 1 || len(resp.Embeddings[0]) != 384 { 118 t.Fatalf("embed %q returned wrong shape", text) 119 } 120 return normalizeVec(resp.Embeddings[0]) 121} 122 123// TestRetrievalRecallGate is the regression gate. It seeds the gold set candidates into a fresh org with real 124// plane embeddings, runs HybridSearch per query, and asserts the mean ranking metrics clear conservative 125// floors. The gold set is constructed so each query's relevant candidates are both lexically present (the full 126// text leg finds them) and semantically near the query (the vector leg finds them), so a correct hybrid search 127// clears the floors comfortably and this is a gate, not a flaky test. 128func TestRetrievalRecallGate(t *testing.T) { 129 client := retrievalPlane(t) 130 fx := loadQrels(t) 131 org := seedOrg(t) 132 ctx := ctxOrg(org) 133 134 // Seed every gold set candidate with its real plane embedding, mapping each seeded id to its stable key so 135 // a returned CandidateID can be resolved back to the key the qrels speak in. Seeding is RLS scoped to this 136 // fresh org (the candidates rows carry organization_id = org; the search runs in the same org context). 137 idToKey := make(map[string]string, len(fx.Candidates)) 138 for _, c := range fx.Candidates { 139 // Embed the SAME text the candidate is findable by (name plus headline), the field the search_tsv and 140 // the resume vector both draw from, so the vector leg recalls the candidate the qrels expects. 141 vec := embedOne(t, client, c.FullName+" "+c.Headline) 142 pv := pgvector.NewVector(vec) 143 hn, loc := c.Headline, c.Location 144 created, err := dbgen.New(migrator).CreateCandidate(context.Background(), dbgen.CreateCandidateParams{ 145 OrganizationID: org, 146 FullName: c.FullName, 147 Headline: &hn, 148 ClearanceLevel: c.ClearanceLevel, 149 ClearancePoly: c.ClearancePoly, 150 ClearanceStatus: c.ClearanceStatus, 151 Location: &loc, 152 Available: true, 153 ResumeEmbedding: &pv, 154 }) 155 if err != nil { 156 t.Fatalf("seed gold candidate %q: %v", c.Key, err) 157 } 158 idToKey[created.ID.String()] = c.Key 159 } 160 161 const ( 162 k5 = 5 163 k10 = 10 164 // limit pulls a generous depth so recall at 10 is measured over the real ranking, not truncated by a 165 // tight limit. The gold set has twelve candidates, so twenty surfaces them all. 166 searchLimit = 20 167 ) 168 169 var sumRecall5, sumRecall10, sumMRR, sumNDCG10 float64 170 nQueries := 0 171 172 for _, q := range fx.Queries { 173 relevant := make(map[string]bool, len(q.Relevant)) 174 for _, key := range q.Relevant { 175 relevant[key] = true 176 } 177 178 // Run the PRODUCTION hybrid search with an empty mission (the default unscoped path, byte for byte the 179 // shipped behavior), RLS scoped to this org. The query embeds through the SAME plane as the seed 180 // vectors, so the vector leg compares like with like. 181 hits, err := ai.HybridSearch(ctx, appDB, client, q.Text, "", searchLimit) 182 if err != nil { 183 t.Fatalf("query %q hybrid search: %v", q.ID, err) 184 } 185 186 // Map the returned CandidateIDs back to seed keys, preserving rank order. A returned id that is not a 187 // gold candidate (there is none here, the org holds only the gold set) maps to an empty key, which is 188 // never relevant, so it neither helps nor hurts the metrics. 189 ranked := make([]string, 0, len(hits)) 190 for _, h := range hits { 191 ranked = append(ranked, idToKey[h.CandidateID]) 192 } 193 194 r5 := eval.RecallAtK(ranked, relevant, k5) 195 r10 := eval.RecallAtK(ranked, relevant, k10) 196 mrr := eval.MRR(ranked, relevant) 197 ndcg10 := eval.NDCGAtK(ranked, relevant, k10) 198 199 t.Logf("query %-26s recall@5=%.3f recall@10=%.3f mrr=%.3f ndcg@10=%.3f (relevant=%d, hits=%d)", 200 q.ID, r5, r10, mrr, ndcg10, len(q.Relevant), len(hits)) 201 202 sumRecall5 += r5 203 sumRecall10 += r10 204 sumMRR += mrr 205 sumNDCG10 += ndcg10 206 nQueries++ 207 } 208 209 meanRecall5 := sumRecall5 / float64(nQueries) 210 meanRecall10 := sumRecall10 / float64(nQueries) 211 meanMRR := sumMRR / float64(nQueries) 212 meanNDCG10 := sumNDCG10 / float64(nQueries) 213 214 t.Logf("MEAN over %d queries (gold set v%s): recall@5=%.3f recall@10=%.3f mrr=%.3f ndcg@10=%.3f", 215 nQueries, fx.Version, meanRecall5, meanRecall10, meanMRR, meanNDCG10) 216 217 // Floors. Each gold query's relevant candidates are lexically present AND semantically near the query, and 218 // the org holds only twelve candidates, so a correct hybrid search ranks the relevant ones at or near the 219 // top and recall at ten is essentially complete. The floors sit well below the expected healthy values so 220 // a passing run is unambiguous, while a real regression (embeddings drift, HNSW ef_search cut, a broken 221 // mission gate, or an RRF k constant change that scrambles the fusion) drops a metric below its floor. 222 // 223 // meanRecall@10 >= 0.8: a healthy search recalls nearly every relevant id within the top ten of a twelve 224 // candidate pool, so 0.8 is a conservative floor that a correct search clears comfortably and a recall 225 // regression breaks. 226 if meanRecall10 < 0.8 { 227 t.Fatalf("REGRESSION: mean recall@10 = %.3f is below the 0.8 floor; the hybrid search is no longer recalling the relevant candidates", meanRecall10) 228 } 229 // meanMRR >= 0.5: a healthy search puts a relevant candidate in the top two on average (rank one is 1.0, 230 // rank two is 0.5), so 0.5 is a conservative floor on first relevant rank that a ranking regression breaks. 231 if meanMRR < 0.5 { 232 t.Fatalf("REGRESSION: mean MRR = %.3f is below the 0.5 floor; the first relevant candidate is ranking too deep", meanMRR) 233 } 234 // meanNDCG@10 >= 0.6: nDCG rewards ranking the relevant ids high, not merely present. 0.6 is a conservative 235 // floor that a correct ordering clears while a fusion or ranking regression that buries relevant ids breaks. 236 if meanNDCG10 < 0.6 { 237 t.Fatalf("REGRESSION: mean nDCG@10 = %.3f is below the 0.6 floor; relevant candidates are ranking too low in the fused order", meanNDCG10) 238 } 239 // recall@5 is logged as a baseline and sanity checked but not floored as tightly, since a five deep cutoff 240 // over a twelve candidate pool is the harder bar; it must still recall a clear majority on average. 241 if meanRecall5 < 0.6 { 242 t.Fatalf("REGRESSION: mean recall@5 = %.3f is below the 0.6 floor; the top five no longer surfaces the relevant candidates", meanRecall5) 243 } 244 245 // Guard the fixture itself against silent identity collisions: every seeded id resolved to a distinct key. 246 if len(idToKey) != len(fx.Candidates) { 247 t.Fatalf("seed key mapping is inconsistent: %d ids for %d candidates", len(idToKey), len(fx.Candidates)) 248 } 249} 250