Janady13/missionmatch / cmd/awards-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/awards-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	39df94f92b2a33f852481fe1954dabed841524bb
Branch	w0-aedt-compliance
Path or archive member	cmd/awards-sync/main.go
Original SHA-256	f5479f662036a01d0d08d106b6cbcd324b82d8dc18e59bbc679c8bfd5c5c34e9
Git blob SHA	5a5979da1a79f764d0acdac2b864ed6cc2d5081d
Reading-copy SHA-256	f5479f662036a01d0d08d106b6cbcd324b82d8dc18e59bbc679c8bfd5c5c34e9

Original source ↗

Recovered source transcript

Download complete source text ↓147 lines · 4.6 KB

// Command awards-sync pulls federal awards from USASpending (no key) into the cached
// usaspending_awards catalog: a market intelligence pass (Galapagos NAICS, DoD, place of performance
// HI and NM, last 3 years, largest first) plus Galapagos own past performance by UEI. One shot job:
// run with `make sync-awards` or daily via the systemd timer. It logs per pass counts and a grand
// total, and stops gracefully on a 429.
package main

import (
	"context"
	"errors"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/marketintel"
	"github.com/missionworkforce/missionmatch/internal/platform/samgov"
)

// galapagosUEI is Galapagos Federal Systems LLC, used to pull their own past performance.
const galapagosUEI = "N3GDEB78RNF8"

const pageCap = 5 // up to 500 awards per pass; keeps a run well within API limits

func main() {
	ctx := context.Background()
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	if dsn == "" {
		log.Fatal("awards-sync: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
	}
	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("awards-sync: db connect: %v", err)
	}
	defer pool.Close()
	if err := pool.Ping(ctx); err != nil {
		log.Fatalf("awards-sync: db ping: %v", err)
	}
	q := dbgen.New(pool)

	prov := marketintel.NewUSASpending(os.Getenv("USASPENDING_BASE"))
	now := time.Now()
	win := []marketintel.TimePeriod{{
		Start: now.AddDate(-3, 0, 0).Format("2006-01-02"),
		End:   now.Format("2006-01-02"),
	}}
	naics := samgov.Galapagos.NAICS
	states := samgov.Galapagos.States

	log.Printf("awards-sync: provider=%s naics=%v states=%v window=%s..%s",
		prov.Name(), naics, states, win[0].Start, win[0].End)

	market := syncPass(ctx, prov, q, marketintel.AwardQuery{
		NAICS: naics, States: states, AwardTypeCodes: []string{"A", "B", "C", "D"},
		AwardingToptier: "Department of Defense", TimePeriods: win, Limit: 100,
	}, "market (DoD, HI and NM)")

	own := syncPass(ctx, prov, q, marketintel.AwardQuery{
		RecipientSearch: []string{galapagosUEI}, AwardTypeCodes: []string{"A", "B", "C", "D"},
		TimePeriods: win, Limit: 100,
	}, "galapagos past performance")

	var total, gal int64 = -1, -1
	if meta, err := q.AwardsSyncMeta(ctx); err == nil {
		total, gal = meta.Total, meta.Galapagos
	}
	log.Printf("awards-sync: DONE, upserted market=%d own=%d grand=%d; store holds %d awards (galapagos %d)",
		market, own, market+own, total, gal)
}

func syncPass(ctx context.Context, prov marketintel.Provider, q *dbgen.Queries, base marketintel.AwardQuery, label string) int {
	upserted := 0
	for page := 1; page <= pageCap; page++ {
		aq := base
		aq.Page = page
		awards, hasNext, err := prov.FetchAwards(ctx, aq)
		if err != nil {
			if errors.Is(err, marketintel.ErrThrottled) {
				log.Printf("awards-sync: %s page %d: 429 throttled, stopping run gracefully", label, page)
				break
			}
			log.Printf("awards-sync: %s page %d: %v", label, page, err)
			break
		}
		for _, a := range awards {
			a.IsGalapagos = a.IsGalapagos || strings.EqualFold(a.RecipientUEI, galapagosUEI)
			if err := q.UpsertAward(ctx, toParams(a)); err != nil {
				log.Printf("awards-sync: upsert %s: %v", a.GeneratedInternalID, err)
				continue
			}
			upserted++
		}
		log.Printf("awards-sync: %s page %d: %d awards", label, page, len(awards))
		if !hasNext || len(awards) == 0 {
			break
		}
		select {
		case <-ctx.Done():
			return upserted
		case <-time.After(300 * time.Millisecond):
		}
	}
	log.Printf("awards-sync: %s: upserted %d", label, upserted)
	return upserted
}

func toParams(a marketintel.Award) dbgen.UpsertAwardParams {
	return dbgen.UpsertAwardParams{
		GeneratedInternalID: a.GeneratedInternalID,
		AwardID:             a.AwardID,
		RecipientName:       a.RecipientName,
		RecipientUei:        a.RecipientUEI,
		RecipientID:         a.RecipientID,
		AwardAmount:         a.AwardAmount,
		AwardingAgency:      a.AwardingAgency,
		AwardingSubAgency:   a.AwardingSubAgency,
		PopStartDate:        dateOf(a.POPStart),
		PopEndDate:          dateOf(a.POPEnd),
		NaicsCode:           a.NAICSCode,
		NaicsDescription:    a.NAICSDescription,
		Description:         a.Description,
		IsGalapagos:         a.IsGalapagos,
		Raw:                 []byte(a.Raw),
		Source:              "USASpending",
	}
}

func dateOf(t *time.Time) pgtype.Date {
	if t == nil {
		return pgtype.Date{}
	}
	return pgtype.Date{Time: *t, Valid: true}
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}