Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / cmd/sam-sync/main.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 39df94f92b2a33f852481fe1954dabed841524bb |
| Branch | w0-aedt-compliance |
| Path or archive member | cmd/sam-sync/main.go |
| Original SHA-256 | 5e38e4b08544c5801c88f0b3e6b944be5de3182b2617b5fb39c308d39ee09c72 |
| Git blob SHA | 23555ed15b88766f0c70d374d0c29c088f7456db |
| Reading-copy SHA-256 | 5e38e4b08544c5801c88f0b3e6b944be5de3182b2617b5fb39c308d39ee09c72 |
Recovered source transcript
Download complete source text ↓174 lines · 5.2 KB
// Command sam-sync pulls federal Contract Opportunities from SAM.gov and upserts them into the cached
// sam_opportunities catalog. It is a one-shot job: run it for a demo (`make sync-sam`) or daily via
// the systemd timer in infra/systemd. It pulls by NAICS only (the Galapagos profile), normalizes, and
// upserts on noticeId. On a 429 it logs and stops gracefully — it never hammers the API.
//
// Env (server-side only): SAM_API_KEY (required), SAM_API_BASE (optional, defaults prod),
// MM_MIGRATOR_DATABASE_URL (write DSN; falls back to MM_DATABASE_URL).
package main
import (
"context"
"errors"
"log"
"os"
"strings"
"time"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
"github.com/missionworkforce/missionmatch/internal/platform/samgov"
)
// pageCap bounds a single NAICS pull (limit 1000 * 5 = up to 5000 notices/NAICS) so one run can never
// exhaust the ~1000 req/day quota. A cap hit is logged (never silent).
const pageCap = 5
func main() {
ctx := context.Background()
key := strings.TrimSpace(os.Getenv("SAM_API_KEY"))
base := strings.TrimSpace(os.Getenv("SAM_API_BASE"))
dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
if key == "" {
log.Fatal("sam-sync: SAM_API_KEY not set (server-side env only — never commit it)")
}
if dsn == "" {
log.Fatal("sam-sync: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
}
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
log.Fatalf("sam-sync: db connect: %v", err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
log.Fatalf("sam-sync: db ping: %v", err)
}
q := dbgen.New(pool)
client := samgov.New(base, key)
profile := samgov.Galapagos
now := time.Now()
from := now.AddDate(0, 0, -profile.LookbackDays)
log.Printf("sam-sync: profile=%q naics=%v window=%s..%s base=%s",
profile.Name, profile.NAICS, from.Format("2006-01-02"), now.Format("2006-01-02"), baseLabel(base))
grand := 0
throttled := false
for _, naics := range profile.NAICS {
recs, total, capped, serr := client.SearchAll(ctx, samgov.SearchParams{
PostedFrom: from, PostedTo: now, NAICS: naics, Limit: 1000,
}, pageCap)
// Persist whatever we gathered, even on a partial/throttled result.
upserted := 0
for _, rec := range recs {
n := samgov.Normalize(rec)
if n.NoticeID == "" {
continue
}
if err := q.UpsertSamOpportunity(ctx, toParams(n)); err != nil {
log.Printf("sam-sync: upsert %s: %v", n.NoticeID, err)
continue
}
upserted++
}
grand += upserted
if serr != nil {
if errors.Is(serr, samgov.ErrThrottled) {
var te *samgov.ThrottleError
errors.As(serr, &te)
log.Printf("sam-sync: NAICS %s: 429 throttled after %d upserts (retry after %s) — stopping run gracefully",
naics, upserted, retryAfter(te))
throttled = true
break
}
log.Printf("sam-sync: NAICS %s: error after %d upserts: %v", naics, upserted, serr)
continue
}
if capped {
log.Printf("sam-sync: NAICS %s: upserted %d (page cap hit — %d available, not all retrieved)", naics, upserted, total)
} else {
log.Printf("sam-sync: NAICS %s: upserted %d of %d available", naics, upserted, total)
}
}
total := int64(-1)
if meta, err := q.SamSyncMeta(ctx); err == nil {
total = meta.Total
}
tail := ""
if throttled {
tail = " (run stopped early on 429)"
}
log.Printf("sam-sync: DONE — upserted %d this run; store now holds %d opportunities%s", grand, total, tail)
}
// toParams bridges a pure samgov.Normalized into the dbgen upsert params (Go-native -> pg types).
func toParams(n samgov.Normalized) dbgen.UpsertSamOpportunityParams {
var posted pgtype.Date
if n.PostedDate != nil {
posted = pgtype.Date{Time: *n.PostedDate, Valid: true}
}
return dbgen.UpsertSamOpportunityParams{
NoticeID: n.NoticeID,
Title: n.Title,
SolicitationNumber: n.SolicitationNumber,
Department: n.Department,
FullParentPath: n.FullParentPath,
SubAgency: n.SubAgency,
NaicsCode: n.NaicsCode,
PscCode: n.PSCCode,
SetAsideCode: n.SetAsideCode,
SetAside: n.SetAside,
NoticeType: n.NoticeType,
BaseType: n.BaseType,
PostedDate: posted,
ResponseDeadline: n.ResponseDeadline,
Active: n.Active,
PopCity: n.POPCity,
PopState: n.POPState,
PopZip: n.POPZip,
UiLink: n.UILink,
DescriptionUrl: n.DescriptionURL,
ResourceLinks: []byte(n.ResourceLinks),
Poc: []byte(n.POC),
Raw: []byte(n.Raw),
Source: n.Source,
}
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if s := strings.TrimSpace(v); s != "" {
return s
}
}
return ""
}
// baseLabel returns a safe label for logs (the base URL never carries the api_key).
func baseLabel(base string) string {
switch {
case base == "" || base == samgov.BaseProd:
return "prod"
case base == samgov.BaseAlpha:
return "alpha"
default:
return base
}
}
func retryAfter(te *samgov.ThrottleError) string {
if te == nil || te.RetryAfter == "" {
return "unknown"
}
return te.RetryAfter
}