Janady13/missionmatch / cmd/verify-vector-model/main.go

1// Command verify-vector-model runs the brain's boot vector model gate against a target database, out of 2// band, so an operator can confirm the deployed binary's gate will PASS before the cutover swaps the 3// binary. It calls the exact same ai.VerifyVectorModel the API runs at boot, with the exact same 4// inference client config (base URL, embed model, dimension), so its verdict is the binary's verdict. 5// 6// It is read only: it samples stored skill vectors and re embeds their names through the plane, then 7// compares. It writes nothing and touches no PII. On a PASS the brain will serve; on a mismatch it prints 8// the offending skill and cosine, exactly what the 503 vector_model_mismatch would report. 9// 10// Usage: 11// 12// verify-vector-model -dsn <app or migrator DSN> -megamind http://host:8090 -model qllama/bge-small-en-v1.5 -dim 384 13// 14// The gate is role independent (the skills table is global and the dimension checks read the catalog), 15// so either the app or the migrator DSN yields the identical verdict. 16package main 17 18import ( 19 "context" 20 "errors" 21 "flag" 22 "fmt" 23 "log" 24 "math" 25 "os" 26 "time" 27 28 "github.com/jackc/pgx/v5/pgxpool" 29 "github.com/pgvector/pgvector-go" 30 31 "github.com/missionworkforce/missionmatch/internal/modules/ai" 32 "github.com/missionworkforce/missionmatch/internal/platform/database" 33 "github.com/missionworkforce/missionmatch/internal/platform/embeddings" 34 "github.com/missionworkforce/missionmatch/internal/platform/inference" 35 "github.com/missionworkforce/missionmatch/internal/platform/tenant" 36) 37 38func main() { 39 dsn := flag.String("dsn", "", "target database DSN, defaults to MM_MIGRATOR_DATABASE_URL") 40 // NOT an env derived default, because flag usage prints defaults and a gateway URL can carry credentials. 41 mega := flag.String("megamind", "", "MEGAMIND gateway base URL, defaults to MM_MEGAMIND_URL") 42 model := flag.String("model", envDefault("MM_MEGAMIND_EMBED_MODEL", "qllama/bge-small-en-v1.5"), "configured embed model") 43 dim := flag.Int("dim", 384, "expected embedding width") 44 flag.Parse() 45 if *dsn == "" { 46 *dsn = os.Getenv("MM_MIGRATOR_DATABASE_URL") 47 } 48 if *mega == "" { 49 *mega = envDefault("MM_MEGAMIND_URL", "http://100.94.8.94:8090") 50 } 51 52 if *dsn == "" || *mega == "" { 53 log.Fatal("verify-vector-model: -dsn and -megamind are required") 54 } 55 ctx := context.Background() 56 57 pool, err := database.NewPool(ctx, *dsn) 58 if err != nil { 59 log.Fatalf("db: %v", err) 60 } 61 defer pool.Close() 62 db := tenant.New(pool) 63 64 client, err := inference.New(inference.Config{ 65 BaseURL: *mega, EmbedModel: *model, Dimension: *dim, Timeout: 60 * time.Second, MaxRetries: 2, 66 }) 67 if err != nil { 68 log.Fatalf("inference: %v", err) 69 } 70 if err := client.Health(ctx); err != nil { 71 log.Fatalf("plane not reachable at %s: %v", *mega, err) 72 } 73 74 // Transparency loop: print the cosine for the same sample the gate uses, so a PASS shows margin and a 75 // FAIL shows exactly how far each sampled skill sits from its fresh embedding. This mirrors the gate's 76 // SELECT and its SkillEmbedText composition; the authoritative verdict still comes from the gate call. 77 if err := printSampleCosines(ctx, pool, client); err != nil { 78 log.Printf("transparency sample failed (continuing to the authoritative gate): %v", err) 79 } 80 81 // The authoritative verdict: the exact function the API binary runs at boot. 82 switch err := ai.VerifyVectorModel(ctx, db, client, *dim); { 83 case err == nil: 84 fmt.Printf("\nVECTOR MODEL GATE: PASS (stored vectors match %s at dim %d). The brain will serve.\n", *model, *dim) 85 case errors.Is(err, ai.ErrVectorModelMismatch): 86 fmt.Printf("\nVECTOR MODEL GATE: FAIL (mismatch). The brain would return 503 vector_model_mismatch.\n %v\n", err) 87 os.Exit(1) 88 default: 89 fmt.Printf("\nVECTOR MODEL GATE: INCONCLUSIVE (plane or config error, the gate would retry on first query).\n %v\n", err) 90 os.Exit(2) 91 } 92} 93 94// printSampleCosines re embeds the same 5 sample skills the gate samples and prints each cosine, so the 95// operator sees the numbers behind the verdict. 96func printSampleCosines(ctx context.Context, pool *pgxpool.Pool, client *inference.Client) error { 97 rows, err := pool.Query(ctx, "SELECT name, COALESCE(category,''), embedding FROM skills WHERE embedding IS NOT NULL ORDER BY id LIMIT 5") 98 if err != nil { 99 return err 100 } 101 defer rows.Close() 102 var names, texts []string 103 var stored [][]float32 104 for rows.Next() { 105 var name, category string 106 var v pgvector.Vector 107 if err := rows.Scan(&name, &category, &v); err != nil { 108 return err 109 } 110 names = append(names, name) 111 texts = append(texts, embeddings.SkillEmbedText(name, category)) 112 stored = append(stored, v.Slice()) 113 } 114 if err := rows.Err(); err != nil { 115 return err 116 } 117 if len(texts) == 0 { 118 fmt.Println("no backfilled skill embeddings to sample") 119 return nil 120 } 121 res, err := client.Embed(ctx, texts) 122 if err != nil { 123 return err 124 } 125 fmt.Println("sample cosines (stored vs fresh embedding of the same text):") 126 for i := range stored { 127 if i >= len(res.Embeddings) { 128 break 129 } 130 fmt.Printf(" %-40q cosine=%.6f\n", texts[i]+" ["+names[i]+"]", cosineSim(stored[i], res.Embeddings[i])) 131 } 132 return nil 133} 134 135func envDefault(k, def string) string { 136 if v := os.Getenv(k); v != "" { 137 return v 138 } 139 return def 140} 141 142// cosineSim is the same true cosine the gate uses (divides by both norms), for the transparency print. 143func cosineSim(a, b []float32) float64 { 144 var dot, na, nb float64 145 for i := range a { 146 dot += float64(a[i]) * float64(b[i]) 147 na += float64(a[i]) * float64(a[i]) 148 nb += float64(b[i]) * float64(b[i]) 149 } 150 if na == 0 || nb == 0 { 151 return 0 152 } 153 return dot / (math.Sqrt(na) * math.Sqrt(nb)) 154} 155