Janady13/missionmatch / cmd/sam-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/sam-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	09015496d565a172bcaa15cfe9a8e4b072b13df0
Branch	workstream-b-screenmatch
Path or archive member	cmd/sam-sync/main.go
Original SHA-256	60b61af4abd5066d153bb263d14273c25ccd0b7de0129a83e021819fa3ff9501
Git blob SHA	2ca4da20b8d040e2b0ffc75895b0ef068e77c481
Reading-copy SHA-256	60b61af4abd5066d153bb263d14273c25ccd0b7de0129a83e021819fa3ff9501

Original source ↗

Every recorded occurrence

These 67 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / cmd/sam-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓180 lines · 5.5 KB

// Command sam-sync pulls federal Contract Opportunities from SAM.gov and upserts them into the cached
// sam_opportunities catalog. It is a one-shot job: run it for a demo (`make sync-sam`) or daily via
// the systemd timer in infra/systemd. It pulls by NAICS only (the Galapagos profile), normalizes, and
// upserts on noticeId. On a 429 it logs and stops gracefully, it never hammers the API.
//
// Env (server-side only): SAM_API_KEY (required), SAM_API_BASE (optional, defaults prod),
// MM_MIGRATOR_DATABASE_URL (write DSN; falls back to MM_DATABASE_URL).
package main

import (
	"context"
	"errors"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/database"
	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/samgov"
)

// pageCap bounds a single NAICS pull (limit 1000 * 5 = up to 5000 notices/NAICS) so one run can never
// exhaust the ~1000 req/day quota. A cap hit is logged (never silent).
const pageCap = 5

func main() {
	ctx := context.Background()

	key := strings.TrimSpace(os.Getenv("SAM_API_KEY"))
	base := strings.TrimSpace(os.Getenv("SAM_API_BASE"))
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))

	if key == "" {
		log.Fatal("sam-sync: SAM_API_KEY not set (server-side env only, never commit it)")
	}
	if dsn == "" {
		log.Fatal("sam-sync: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
	}

	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("sam-sync: db connect: %v", err)
	}
	defer pool.Close()
	if err := pool.Ping(ctx); err != nil {
		log.Fatalf("sam-sync: db ping: %v", err)
	}
	// sam_opportunities is a global catalog the application role may only read, so this writer needs the
	// BYPASSRLS migrator DSN; assert it now for a legible failure instead of a mid run permission denied.
	if err := database.AssertMigratorRole(ctx, pool); err != nil {
		log.Fatalf("sam-sync: %v", err)
	}
	q := dbgen.New(pool)

	client := samgov.New(base, key)
	profile := samgov.Galapagos
	now := time.Now()
	from := now.AddDate(0, 0, -profile.LookbackDays)

	log.Printf("sam-sync: profile=%q naics=%v window=%s..%s base=%s",
		profile.Name, profile.NAICS, from.Format("2006-01-02"), now.Format("2006-01-02"), baseLabel(base))

	grand := 0
	throttled := false
	for _, naics := range profile.NAICS {
		recs, total, capped, serr := client.SearchAll(ctx, samgov.SearchParams{
			PostedFrom: from, PostedTo: now, NAICS: naics, Limit: 1000,
		}, pageCap)

		// Persist whatever we gathered, even on a partial/throttled result.
		upserted := 0
		for _, rec := range recs {
			n := samgov.Normalize(rec)
			if n.NoticeID == "" {
				continue
			}
			if err := q.UpsertSamOpportunity(ctx, toParams(n)); err != nil {
				log.Printf("sam-sync: upsert %s: %v", n.NoticeID, err)
				continue
			}
			upserted++
		}
		grand += upserted

		if serr != nil {
			if errors.Is(serr, samgov.ErrThrottled) {
				var te *samgov.ThrottleError
				errors.As(serr, &te)
				log.Printf("sam-sync: NAICS %s: 429 throttled after %d upserts (retry after %s), stopping run gracefully",
					naics, upserted, retryAfter(te))
				throttled = true
				break
			}
			log.Printf("sam-sync: NAICS %s: error after %d upserts: %v", naics, upserted, serr)
			continue
		}
		if capped {
			log.Printf("sam-sync: NAICS %s: upserted %d (page cap hit, %d available, not all retrieved)", naics, upserted, total)
		} else {
			log.Printf("sam-sync: NAICS %s: upserted %d of %d available", naics, upserted, total)
		}
	}

	total := int64(-1)
	if meta, err := q.SamSyncMeta(ctx); err == nil {
		total = meta.Total
	}
	tail := ""
	if throttled {
		tail = " (run stopped early on 429)"
	}
	log.Printf("sam-sync: DONE, upserted %d this run; store now holds %d opportunities%s", grand, total, tail)
}

// toParams bridges a pure samgov.Normalized into the dbgen upsert params (Go-native -> pg types).
func toParams(n samgov.Normalized) dbgen.UpsertSamOpportunityParams {
	var posted pgtype.Date
	if n.PostedDate != nil {
		posted = pgtype.Date{Time: *n.PostedDate, Valid: true}
	}
	return dbgen.UpsertSamOpportunityParams{
		NoticeID:           n.NoticeID,
		Title:              n.Title,
		SolicitationNumber: n.SolicitationNumber,
		Department:         n.Department,
		FullParentPath:     n.FullParentPath,
		SubAgency:          n.SubAgency,
		NaicsCode:          n.NaicsCode,
		PscCode:            n.PSCCode,
		SetAsideCode:       n.SetAsideCode,
		SetAside:           n.SetAside,
		NoticeType:         n.NoticeType,
		BaseType:           n.BaseType,
		PostedDate:         posted,
		ResponseDeadline:   n.ResponseDeadline,
		Active:             n.Active,
		PopCity:            n.POPCity,
		PopState:           n.POPState,
		PopZip:             n.POPZip,
		UiLink:             n.UILink,
		DescriptionUrl:     n.DescriptionURL,
		ResourceLinks:      []byte(n.ResourceLinks),
		Poc:                []byte(n.POC),
		Raw:                []byte(n.Raw),
		Source:             n.Source,
	}
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}

// baseLabel returns a safe label for logs (the base URL never carries the api_key).
func baseLabel(base string) string {
	switch {
	case base == "" || base == samgov.BaseProd:
		return "prod"
	case base == samgov.BaseAlpha:
		return "alpha"
	default:
		return base
	}
}

func retryAfter(te *samgov.ThrottleError) string {
	if te == nil || te.RetryAfter == "" {
		return "unknown"
	}
	return te.RetryAfter
}