Janady13/missionmatch / internal/modules/ai/retrieval.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/ai/retrieval.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2
Branch	ws7-federal-signal
Path or archive member	internal/modules/ai/retrieval.go
Original SHA-256	736307e62ef486a6503e4e03e17dc49dbedcaf410c265fb8f6e0cc51bc597a8b
Git blob SHA	9a22e9c92e0e33f5b805bde6e771dac6c587d9eb
Reading-copy SHA-256	736307e62ef486a6503e4e03e17dc49dbedcaf410c265fb8f6e0cc51bc597a8b

Original source ↗

Every recorded occurrence

These 5 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/ai/retrieval.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/ai/retrieval.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/ai/retrieval.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/ai/retrieval.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/ai/retrieval.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original

Recovered source transcript

Download complete source text ↓128 lines · 5.5 KB

package ai

import (
	"context"
	"errors"

	"github.com/jackc/pgx/v5"
	pgvector "github.com/pgvector/pgvector-go"

	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/inference"
	"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)

// Hit is one retrieved candidate with its fused relevance score.
type Hit struct {
	CandidateID string  `json:"candidate_id"`
	FullName    string  `json:"full_name"`
	Score       float64 `json:"rrf_score"`
}

// HybridSearch is the RAG retrieval leg: it embeds the query through the MEGAMIND plane and runs the
// existing hybrid query that fuses pgvector cosine kNN with full text rank via Reciprocal Rank Fusion
// (docs/AI_RESEARCH.md, pgvector + RRF). It runs RLS scoped as the tenant, so it returns only the
// caller's candidates, and it tunes the HNSW scan per the pgvector 0.8 guidance before the search.
//
// When mission is a non empty mission keyword (from MissionFromQuestion, drawn only from the fixed
// mission_spine vocabulary), retrieval is additionally constrained to that mission. An empty mission runs the
// unscoped query byte for byte as before, so the default path is unchanged.
//
// WS6 to WS7 upgrade (the mission gate is now structured, additively, with the WS6 text filter kept as a
// fallback): when the tenant has been backfilled by the mission tagging pass (candidate_mission_tags has at
// least one row, probed RLS scoped in the same transaction) AND the keyword maps to a known mission code, the
// scoped query is SearchCandidatesHybridMissionTagged, which gates both arms on a STRUCTURED EXISTS predicate
// over candidate_mission_tags rather than the WS6 search_tsv text match. That is the candidate's real
// classified mission, so it reaches a candidate whose mission lives in the resume body, not only one whose
// full_name, headline, or location text happens to contain the mission keyword (the WS6 limit, lifted once
// tags exist). When the tenant is NOT yet tagged, or the keyword has no code, retrieval falls back to the WS6
// SearchCandidatesHybridMission text query, so an un backfilled tenant keeps exact WS6 behavior. Either scoped
// path keeps the RLS scoping, the ForbidEEO guard, and the HNSW SET LOCAL tuning of the unscoped path.
//
// It never touches the eeo schema (the query reads only candidates and their own mission tags, never the eeo
// schema, and mission tags are never a scoring input), and the query string is guarded so a demographic
// reference cannot enter the retrieval path.
func HybridSearch(ctx context.Context, db *tenant.DB, client *inference.Client, query, mission string, limit int) ([]Hit, error) {
	if db == nil || client == nil {
		return nil, errors.New("ai: retrieval needs a database and an inference client")
	}
	if err := ForbidEEO(query); err != nil {
		return nil, err
	}
	if limit <= 0 || limit > 100 {
		limit = 20
	}
	emb, err := client.Embed(ctx, []string{query})
	if err != nil {
		return nil, err
	}
	if len(emb.Embeddings) != 1 {
		return nil, errors.New("ai: query embedding failed")
	}
	qv := pgvector.NewVector(emb.Embeddings[0])

	var hits []Hit
	err = db.RunInTenantTx(ctx, func(ctx context.Context, tx pgx.Tx) error {
		// pgvector 0.8 tuning: a wider ef_search lifts recall on the filtered (RLS) scan, and relaxed
		// iterative scan avoids overfiltering under the tenant predicate (docs/AI_RESEARCH.md, pgvector).
		if _, err := tx.Exec(ctx, "SET LOCAL hnsw.ef_search = 100"); err != nil {
			return err
		}
		if _, err := tx.Exec(ctx, "SET LOCAL hnsw.iterative_scan = 'relaxed_order'"); err != nil {
			return err
		}
		q := dbgen.New(tx)
		if mission != "" {
			// WS7 structured mission gate, preferred when the tenant has been tagged and the keyword maps to a
			// known mission code. The probe is RLS scoped, so it counts only this tenant's tags. When either
			// condition is unmet the WS6 text query runs instead, so an un backfilled tenant is byte identical
			// to WS6.
			code := MissionCodeForTerm(mission)
			if code != "" {
				tagCount, err := q.CountCandidateMissionTags(ctx)
				if err != nil {
					return err
				}
				if tagCount > 0 {
					rows, err := q.SearchCandidatesHybridMissionTagged(ctx, dbgen.SearchCandidatesHybridMissionTaggedParams{
						QueryVector: qv, RowLimit: int32(limit), TextQuery: query, MissionCode: code,
					})
					if err != nil {
						return err
					}
					hits = make([]Hit, len(rows))
					for i, r := range rows {
						hits[i] = Hit{CandidateID: r.ID.String(), FullName: r.FullName, Score: r.RrfScore}
					}
					return nil
				}
			}
			// WS6 fallback: the tenant has no mission tags yet, or the keyword has no structured code, so the
			// documented WS6 search_tsv text filter scopes the mission (byte for byte the WS6 behavior).
			rows, err := q.SearchCandidatesHybridMission(ctx, dbgen.SearchCandidatesHybridMissionParams{
				QueryVector: qv, RowLimit: int32(limit), TextQuery: query, MissionQuery: mission,
			})
			if err != nil {
				return err
			}
			hits = make([]Hit, len(rows))
			for i, r := range rows {
				hits[i] = Hit{CandidateID: r.ID.String(), FullName: r.FullName, Score: r.RrfScore}
			}
			return nil
		}
		rows, err := q.SearchCandidatesHybrid(ctx, dbgen.SearchCandidatesHybridParams{
			Column1: qv, Limit: int32(limit), WebsearchToTsquery: query,
		})
		if err != nil {
			return err
		}
		hits = make([]Hit, len(rows))
		for i, r := range rows {
			hits[i] = Hit{CandidateID: r.ID.String(), FullName: r.FullName, Score: r.RrfScore}
		}
		return nil
	})
	return hits, err
}