Janady13/missionmatch / cmd/opp-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/opp-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/opp-sync/main.go
Original SHA-256	93de0762a0689398ccd56f4d1637e6493520f6948400e0e77044ce0d3d333e3a
Git blob SHA	aacde10e2ef6ce21e3645448a985fa1b196c58cc
Reading-copy SHA-256	d081432f0048df3ce4e1ffc187fe094af67b7d60724a36ecab5605a9f4e7d1ad

Original source ↗

Every recorded occurrence

These 33 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/opp-sync/main.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / cmd/opp-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓133 lines · 5.5 KB

Credential-bearing and unrelated personal details were removed from this reading copy. The source identity and original checksum remain recorded.

// Command opp-sync runs the Opportunity Intelligence ingestion. It pulls the configured federal feeds
// (SAM active solicitations, and federal contract awards from the SAM Contract Awards API with a
// USASpending fallback) through the OpportunitySource adapters and upserts them into the unified
// opportunity_records catalog, retaining each raw payload to object storage and writing one audit event
// per source. It runs as the migrator role and is safe to rerun: every upsert is idempotent.
//
// Env: MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL), SAM_API_KEY, SAM_API_BASE (optional),
// SAM_AWARDS_BASE (optional), USASPENDING_BASE (optional), MM_FORECAST_CSV (optional, a URL or a file
// path to a GSA or agency forecast export), MM_OPP_RAW_DIR (object storage root, default ./data/opp-raw).
package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"

	oppintel "github.com/missionworkforce/missionmatch/internal/modules/opportunity_intelligence"
	"github.com/missionworkforce/missionmatch/internal/modules/opportunity_intelligence/adapters"
	"github.com/missionworkforce/missionmatch/internal/platform/audit"
	"github.com/missionworkforce/missionmatch/internal/platform/forecast"
	"github.com/missionworkforce/missionmatch/internal/platform/marketintel"
	"github.com/missionworkforce/missionmatch/internal/platform/samawards"
	"github.com/missionworkforce/missionmatch/internal/platform/samgov"
	"github.com/missionworkforce/missionmatch/internal/platform/storage"
	"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)

func main() {
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	if dsn == "" {
		log.Fatal("opp-sync: set MM_MIGRATOR_DATABASE_URL (the migrator DSN)")
	}
	samKey := strings.TrimSpace(os.Getenv("SAM_API_KEY"))
	if samKey == "" {
		log.Fatal("opp-sync: SAM_API_KEY is [credential removed]")
	}
	rawDir := firstNonEmpty(os.Getenv("MM_OPP_RAW_DIR"), "./data/opp-raw")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("opp-sync: connect: %v", err)
	}
	defer pool.Close()

	store, err := storage.NewLocal(rawDir)
	if err != nil {
		log.Fatalf("opp-sync: storage: %v", err)
	}

	ing := oppintel.NewIngester(store, oppintel.NewDBWriter(pool), audit.New(tenant.New(pool)))

	// The federal award feed is the SAM Contract Awards API (the FPDS replacement) as the primary, with
	// the free USASpending pull as the resilient fallback if SAM is throttled or unavailable.
	samAwards := adapters.NewSAMContractAwardsSource(samawards.New(os.Getenv("SAM_AWARDS_BASE"), samKey), galapagosAwardsParams(), 5)
	usaspending := adapters.NewUSASpendingSource(marketintel.NewUSASpending(os.Getenv("USASPENDING_BASE")), galapagosAwardQuery(), 5)

	sources := []oppintel.OpportunitySource{
		adapters.NewSAMSource(samgov.New(os.Getenv("SAM_API_BASE"), samKey), samgov.Galapagos, 5),
		adapters.NewFallbackAwardsSource(samAwards, usaspending),
	}

	// The forecast feed (the GSA Forecast of Contracting Opportunities, or any agency forecast CSV) is
	// enabled only when an export is configured, because the GSA export is a manual download from a gated
	// dashboard rather than a clean API. MM_FORECAST_CSV is a URL or a local file path.
	if fc := strings.TrimSpace(os.Getenv("MM_FORECAST_CSV")); fc != "" {
		var fetch forecast.Fetcher
		if strings.HasPrefix(fc, "http://") || strings.HasPrefix(fc, "https://") {
			fetch = forecast.HTTPFetcher(&http.Client{Timeout: 60 * time.Second}, fc)
		} else {
			fetch = forecast.FileFetcher(fc)
		}
		sources = append(sources, adapters.NewForecastSource(forecast.NewSource("GSA Forecast", fetch, forecast.GSAFCOMapping())))
	}

	var grand int
	for _, src := range sources {
		res, ierr := ing.Ingest(ctx, src)
		if ierr != nil {
			log.Fatalf("opp-sync: ingest %s: %v", src.Name(), ierr)
		}
		log.Printf("opp-sync: %s fetched %d upserted %d by_kind %v", res.Source, res.Fetched, res.Upserted, res.ByKind)
		grand += res.Upserted
	}
	log.Printf("opp-sync: done, %d records upserted into opportunity_records", grand)
}

// galapagosAwardsParams is the SAM Contract Awards pull (the Galapagos NAICS family under the Department
// of Defense, the last three years of signed actions). The Contract Awards API takes all NAICS in one
// query, and place of performance is filtered in the read layer, the same way the opportunities feed
// filters state against the cache.
func galapagosAwardsParams() samawards.SearchParams {
	now := time.Now().UTC()
	return samawards.SearchParams{
		NAICS:      samgov.Galapagos.NAICS,
		Department: "DEFENSE",
		SignedFrom: now.AddDate(-3, 0, 0),
		SignedTo:   now,
		Limit:      100,
	}
}

// galapagosAwardQuery is the USASpending fallback pull (DoD, the Galapagos NAICS family, HI and NM, the
// last three years of contract actions). It mirrors the standalone awards-sync market pass.
func galapagosAwardQuery() marketintel.AwardQuery {
	now := time.Now().UTC()
	start := now.AddDate(-3, 0, 0)
	return marketintel.AwardQuery{
		NAICS:           samgov.Galapagos.NAICS,
		States:          samgov.Galapagos.States,
		AwardTypeCodes:  []string{"A", "B", "C", "D"},
		AwardingToptier: "Department of Defense",
		TimePeriods:     []marketintel.TimePeriod{{Start: start.Format("2006-01-02"), End: now.Format("2006-01-02")}},
		Limit:           100,
	}
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}