Janady13/missionmatch / cmd/awards-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/awards-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/awards-sync/main.go
Original SHA-256	5ca98029b11e4ed64bc5f2c75b067264f521d7307f78bc9592c13b7d233c79e4
Git blob SHA	ed7f4e92cd35032a0ef9b8647470dba203c6ef44
Reading-copy SHA-256	5ca98029b11e4ed64bc5f2c75b067264f521d7307f78bc9592c13b7d233c79e4

Original source ↗

Every recorded occurrence

These 35 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/awards-sync/main.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / cmd/awards-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓166 lines · 5.7 KB

// Command awards-sync pulls federal awards from USASpending (no key) into the cached
// usaspending_awards catalog: a market intelligence pass (Galapagos NAICS, DoD, place of performance
// HI and NM, last 3 years, largest first) plus Galapagos own past performance by UEI. One shot job:
// run with `make sync-awards` or daily via the systemd timer. It logs per pass counts and a grand
// total, and stops gracefully on a 429.
package main

import (
	"context"
	"errors"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/database"
	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/marketintel"
	"github.com/missionworkforce/missionmatch/internal/platform/samgov"
)

// galapagosUEI is Galapagos Federal Systems LLC, used to pull their own past performance.
const galapagosUEI = "N3GDEB78RNF8"

const pageCap = 5 // up to 500 awards per pass; keeps a run well within API limits

func main() {
	ctx := context.Background()
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	if dsn == "" {
		log.Fatal("awards-sync: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
	}
	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("awards-sync: db connect: %v", err)
	}
	defer pool.Close()
	if err := pool.Ping(ctx); err != nil {
		log.Fatalf("awards-sync: db ping: %v", err)
	}
	// usaspending_awards is a global catalog the application role may only read, so this writer needs
	// the BYPASSRLS migrator DSN; assert it now for a legible failure instead of a mid run permission denied.
	if err := database.AssertMigratorRole(ctx, pool); err != nil {
		log.Fatalf("awards-sync: %v", err)
	}
	q := dbgen.New(pool)

	prov := marketintel.NewUSASpending(os.Getenv("USASPENDING_BASE"))
	now := time.Now()
	win := []marketintel.TimePeriod{{
		Start: now.AddDate(-3, 0, 0).Format("2006-01-02"),
		End:   now.Format("2006-01-02"),
	}}
	naics := samgov.Galapagos.NAICS
	states := samgov.Galapagos.States

	log.Printf("awards-sync: provider=%s naics=%v states=%v window=%s..%s",
		prov.Name(), naics, states, win[0].Start, win[0].End)

	market := syncPass(ctx, prov, q, marketintel.AwardQuery{
		NAICS: naics, States: states, AwardTypeCodes: []string{"A", "B", "C", "D"},
		AwardingToptier: "Department of Defense", TimePeriods: win, Limit: 100,
	}, "market (DoD, HI and NM)", false)

	// USASpending treats recipient_search_text as a fuzzy substring match, so a UEI query also returns
	// look alike recipients. The own pass exists to catalog Galapagos OWN past performance, so keep only
	// awards whose recipient UEI is an exact match; ownOnly drops the rest before they reach the store.
	own := syncPass(ctx, prov, q, marketintel.AwardQuery{
		RecipientSearch: []string{galapagosUEI}, AwardTypeCodes: []string{"A", "B", "C", "D"},
		TimePeriods: win, Limit: 100,
	}, "galapagos past performance", true)

	var total, gal int64 = -1, -1
	if meta, err := q.AwardsSyncMeta(ctx); err == nil {
		total, gal = meta.Total, meta.Galapagos
	}
	log.Printf("awards-sync: DONE, upserted market=%d own=%d grand=%d; store holds %d awards (galapagos %d)",
		market, own, market+own, total, gal)
}

func syncPass(ctx context.Context, prov marketintel.Provider, q *dbgen.Queries, base marketintel.AwardQuery, label string, ownOnly bool) int {
	upserted := 0
	skipped := 0
	for page := 1; page <= pageCap; page++ {
		aq := base
		aq.Page = page
		awards, hasNext, err := prov.FetchAwards(ctx, aq)
		if err != nil {
			if errors.Is(err, marketintel.ErrThrottled) {
				log.Printf("awards-sync: %s page %d: 429 throttled, stopping run gracefully", label, page)
				break
			}
			log.Printf("awards-sync: %s page %d: %v", label, page, err)
			break
		}
		for _, a := range awards {
			a.IsGalapagos = a.IsGalapagos || strings.EqualFold(a.RecipientUEI, galapagosUEI)
			// On the own pass, the fuzzy recipient text search can return look alike recipients; only the
			// exact UEI is Galapagos own past performance, so drop everything else before it is stored.
			if ownOnly && !a.IsGalapagos {
				skipped++
				continue
			}
			if err := q.UpsertAward(ctx, toParams(a)); err != nil {
				log.Printf("awards-sync: upsert %s: %v", a.GeneratedInternalID, err)
				continue
			}
			upserted++
		}
		log.Printf("awards-sync: %s page %d: %d awards", label, page, len(awards))
		if !hasNext || len(awards) == 0 {
			break
		}
		select {
		case <-ctx.Done():
			return upserted
		case <-time.After(300 * time.Millisecond):
		}
	}
	if ownOnly && skipped > 0 {
		log.Printf("awards-sync: %s: dropped %d non Galapagos look alike recipient(s)", label, skipped)
	}
	log.Printf("awards-sync: %s: upserted %d", label, upserted)
	return upserted
}

func toParams(a marketintel.Award) dbgen.UpsertAwardParams {
	return dbgen.UpsertAwardParams{
		GeneratedInternalID: a.GeneratedInternalID,
		AwardID:             a.AwardID,
		RecipientName:       a.RecipientName,
		RecipientUei:        a.RecipientUEI,
		RecipientID:         a.RecipientID,
		AwardAmount:         a.AwardAmount,
		AwardingAgency:      a.AwardingAgency,
		AwardingSubAgency:   a.AwardingSubAgency,
		PopStartDate:        dateOf(a.POPStart),
		PopEndDate:          dateOf(a.POPEnd),
		NaicsCode:           a.NAICSCode,
		NaicsDescription:    a.NAICSDescription,
		Description:         a.Description,
		IsGalapagos:         a.IsGalapagos,
		Raw:                 []byte(a.Raw),
		Source:              "USASpending",
	}
}

func dateOf(t *time.Time) pgtype.Date {
	if t == nil {
		return pgtype.Date{}
	}
	return pgtype.Date{Time: *t, Valid: true}
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}