// Command missionclass-sync runs the WS7 federal signal classification pass. It reads the global // opportunity_records, usaspending_awards, and sam_opportunities catalogs, runs the pure deterministic // missionclass classifier over the record text plus, for SAM notices, the text extracted from their // attachments, and upserts mission tags (federal_mission_tags) and incumbent labor roles // (award_incumbent_roles) idempotently. It writes ONE append only audit event per run with counts only, // never any PII or record text. It is a one shot job and a cron candidate like mm-opp-sync, safe to // rerun: every upsert is idempotent. // // Mission tags are reference data, NEVER a scoring input, and the pass never reaches the eeo schema. // // Env (server side only): MM_MIGRATOR_DATABASE_URL (the BYPASSRLS migrator DSN, the global catalogs are // app read only; falls back to MM_DATABASE_URL), SAM_API_KEY (required for SAM attachment mining), // SAM_API_BASE (optional), MM_OPP_RAW_DIR (object storage root for retained attachments, default // ./data/opp-raw). // // Flags: -max-attachments bounds how many SAM notices the run fetches attachments for, so one run stays // within the SAM API daily quota. // // Build and run FIPS validated: CGO_ENABLED=0 GOFIPS140=v1.0.0 go build, run with GODEBUG=fips140=on. package main import ( "context" "crypto/fips140" "crypto/pbkdf2" "crypto/sha256" "flag" "log" "os" "strings" "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/missionworkforce/missionmatch/internal/modules/federalsignal" "github.com/missionworkforce/missionmatch/internal/platform/audit" "github.com/missionworkforce/missionmatch/internal/platform/database" "github.com/missionworkforce/missionmatch/internal/platform/security" "github.com/missionworkforce/missionmatch/internal/platform/storage" "github.com/missionworkforce/missionmatch/internal/platform/tenant" ) func main() { maxAttachments := flag.Int("max-attachments", 50, "max SAM notices to fetch attachments for in one run, bounds SAM API quota use") flag.Parse() // FIPS self test: derive one PBKDF2 key at the production cost under the active crypto module and fail // closed if it errors, so a build that dropped the validated module is caught at boot, not mid run. // This is the same in build FIPS gate the api binary uses, the only FIPS path the repo ships. if err := fipsSelfTest(); err != nil { log.Fatalf("missionclass-sync: FIPS self test (PBKDF2 HMAC SHA 256) failed: %v", err) } log.Printf("missionclass-sync: fips_enabled=%v module=%s", fips140.Enabled(), fips140.Version()) dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL")) if dsn == "" { log.Fatal("missionclass-sync: set MM_MIGRATOR_DATABASE_URL (the migrator DSN)") } samKey := strings.TrimSpace(os.Getenv("SAM_API_KEY")) if samKey == "" { // SAM attachment mining is the only path that needs the key. Without it, mission tags and award // incumbent roles are still written; only the SAM notice incumbent roles are skipped. Make the // degraded mode explicit rather than failing the whole run. log.Print("missionclass-sync: SAM_API_KEY not set, SAM attachment mining disabled for this run") } rawDir := firstNonEmpty(os.Getenv("MM_OPP_RAW_DIR"), "./data/opp-raw") ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() pool, err := pgxpool.New(ctx, dsn) if err != nil { log.Fatalf("missionclass-sync: db connect: %v", err) } defer pool.Close() if err := pool.Ping(ctx); err != nil { log.Fatalf("missionclass-sync: db ping: %v", err) } // federal_mission_tags and award_incumbent_roles are global catalogs the application role may only // read, so this writer needs the BYPASSRLS migrator DSN; assert it now for a legible failure instead // of a mid run permission denied. if err := database.AssertMigratorRole(ctx, pool); err != nil { log.Fatalf("missionclass-sync: %v", err) } store, err := storage.NewLocal(rawDir) if err != nil { log.Fatalf("missionclass-sync: storage: %v", err) } // The SAM fetch client holds the api_key server side only. New registers the production SAM hosts // (api.sam.gov and sam.gov) by default, so the key is appended only to a SAM URL and never to a // foreign resource link. Retained raw attachments land under the federalsignal prefix for retention. fetcher := federalsignal.New(samKey, federalsignal.WithStore(store, "federalsignal")) // When the key is absent, disable SAM attachment fetching by bounding at zero so the pass still tags // mission areas and mines award incumbent roles, and reports the SAM notices as skipped. bound := *maxAttachments if samKey == "" { bound = 0 } repo := federalsignal.NewRepo(pool) auditor := audit.New(tenant.New(pool)) svc := federalsignal.NewService(repo, fetcher, auditor, bound) res, err := svc.ClassifyAll(ctx) if err != nil { log.Fatalf("missionclass-sync: classify: %v", err) } log.Printf("missionclass-sync: DONE, scanned %d records, wrote %d mission tags and %d incumbent roles, fetched %d SAM notices, skipped %d over the attachment cap", res.RecordsScanned, res.MissionTagsWritten, res.IncumbentRoles, res.SamFetched, res.SamSkippedCap) } // fipsSelfTest derives one PBKDF2 key at the production iteration count under the active crypto module. // It is the in build FIPS gate, identical to the api binary self test, so the sync binary is held to the // same validated crypto posture. func fipsSelfTest() error { _, err := pbkdf2.Key(sha256.New, "missionmatch fips self test", []byte("self test salt"), security.PBKDF2Iterations, 32) return err } func firstNonEmpty(vals ...string) string { for _, v := range vals { if s := strings.TrimSpace(v); s != "" { return s } } return "" }