// Command awards-sync pulls federal awards from USASpending (no key) into the cached // usaspending_awards catalog: a market intelligence pass (Galapagos NAICS, DoD, place of performance // HI and NM, last 3 years, largest first) plus Galapagos own past performance by UEI. One shot job: // run with `make sync-awards` or daily via the systemd timer. It logs per pass counts and a grand // total, and stops gracefully on a 429. package main import ( "context" "errors" "log" "os" "strings" "time" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "github.com/missionworkforce/missionmatch/internal/platform/database" "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" "github.com/missionworkforce/missionmatch/internal/platform/marketintel" "github.com/missionworkforce/missionmatch/internal/platform/samgov" ) // galapagosUEI is Galapagos Federal Systems LLC, used to pull their own past performance. const galapagosUEI = "N3GDEB78RNF8" const pageCap = 5 // up to 500 awards per pass; keeps a run well within API limits func main() { ctx := context.Background() dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL")) if dsn == "" { log.Fatal("awards-sync: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)") } pool, err := pgxpool.New(ctx, dsn) if err != nil { log.Fatalf("awards-sync: db connect: %v", err) } defer pool.Close() if err := pool.Ping(ctx); err != nil { log.Fatalf("awards-sync: db ping: %v", err) } // usaspending_awards is a global catalog the application role may only read, so this writer needs // the BYPASSRLS migrator DSN; assert it now for a legible failure instead of a mid run permission denied. if err := database.AssertMigratorRole(ctx, pool); err != nil { log.Fatalf("awards-sync: %v", err) } q := dbgen.New(pool) prov := marketintel.NewUSASpending(os.Getenv("USASPENDING_BASE")) now := time.Now() win := []marketintel.TimePeriod{{ Start: now.AddDate(-3, 0, 0).Format("2006-01-02"), End: now.Format("2006-01-02"), }} naics := samgov.Galapagos.NAICS states := samgov.Galapagos.States log.Printf("awards-sync: provider=%s naics=%v states=%v window=%s..%s", prov.Name(), naics, states, win[0].Start, win[0].End) market := syncPass(ctx, prov, q, marketintel.AwardQuery{ NAICS: naics, States: states, AwardTypeCodes: []string{"A", "B", "C", "D"}, AwardingToptier: "Department of Defense", TimePeriods: win, Limit: 100, }, "market (DoD, HI and NM)", false) // USASpending treats recipient_search_text as a fuzzy substring match, so a UEI query also returns // look alike recipients. The own pass exists to catalog Galapagos OWN past performance, so keep only // awards whose recipient UEI is an exact match; ownOnly drops the rest before they reach the store. own := syncPass(ctx, prov, q, marketintel.AwardQuery{ RecipientSearch: []string{galapagosUEI}, AwardTypeCodes: []string{"A", "B", "C", "D"}, TimePeriods: win, Limit: 100, }, "galapagos past performance", true) var total, gal int64 = -1, -1 if meta, err := q.AwardsSyncMeta(ctx); err == nil { total, gal = meta.Total, meta.Galapagos } log.Printf("awards-sync: DONE, upserted market=%d own=%d grand=%d; store holds %d awards (galapagos %d)", market, own, market+own, total, gal) } func syncPass(ctx context.Context, prov marketintel.Provider, q *dbgen.Queries, base marketintel.AwardQuery, label string, ownOnly bool) int { upserted := 0 skipped := 0 for page := 1; page <= pageCap; page++ { aq := base aq.Page = page awards, hasNext, err := prov.FetchAwards(ctx, aq) if err != nil { if errors.Is(err, marketintel.ErrThrottled) { log.Printf("awards-sync: %s page %d: 429 throttled, stopping run gracefully", label, page) break } log.Printf("awards-sync: %s page %d: %v", label, page, err) break } for _, a := range awards { a.IsGalapagos = a.IsGalapagos || strings.EqualFold(a.RecipientUEI, galapagosUEI) // On the own pass, the fuzzy recipient text search can return look alike recipients; only the // exact UEI is Galapagos own past performance, so drop everything else before it is stored. if ownOnly && !a.IsGalapagos { skipped++ continue } if err := q.UpsertAward(ctx, toParams(a)); err != nil { log.Printf("awards-sync: upsert %s: %v", a.GeneratedInternalID, err) continue } upserted++ } log.Printf("awards-sync: %s page %d: %d awards", label, page, len(awards)) if !hasNext || len(awards) == 0 { break } select { case <-ctx.Done(): return upserted case <-time.After(300 * time.Millisecond): } } if ownOnly && skipped > 0 { log.Printf("awards-sync: %s: dropped %d non Galapagos look alike recipient(s)", label, skipped) } log.Printf("awards-sync: %s: upserted %d", label, upserted) return upserted } func toParams(a marketintel.Award) dbgen.UpsertAwardParams { return dbgen.UpsertAwardParams{ GeneratedInternalID: a.GeneratedInternalID, AwardID: a.AwardID, RecipientName: a.RecipientName, RecipientUei: a.RecipientUEI, RecipientID: a.RecipientID, AwardAmount: a.AwardAmount, AwardingAgency: a.AwardingAgency, AwardingSubAgency: a.AwardingSubAgency, PopStartDate: dateOf(a.POPStart), PopEndDate: dateOf(a.POPEnd), NaicsCode: a.NAICSCode, NaicsDescription: a.NAICSDescription, Description: a.Description, IsGalapagos: a.IsGalapagos, Raw: []byte(a.Raw), Source: "USASpending", } } func dateOf(t *time.Time) pgtype.Date { if t == nil { return pgtype.Date{} } return pgtype.Date{Time: *t, Valid: true} } func firstNonEmpty(vals ...string) string { for _, v := range vals { if s := strings.TrimSpace(v); s != "" { return s } } return "" }