Janady13/missionmatch / cmd/ontology-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/ontology-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/ontology-sync/main.go
Original SHA-256	8ef3faf6db042adffaffb005d3085566a81e3253b0fcac66378ced29773fe0e9
Git blob SHA	90eff0d82ab3de715e55a2e2ba7d0cb2915157f4
Reading-copy SHA-256	8ef3faf6db042adffaffb005d3085566a81e3253b0fcac66378ced29773fe0e9

Original source ↗

Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/ontology-sync/main.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original

Recovered source transcript

Download complete source text ↓114 lines · 4.7 KB

// Command ontology-sync ingests the GovCon vocabulary from free federal sources into the global
// ontology_terms table, mirroring cmd/opp-sync. It runs as the migrator role and is safe to rerun: every
// upsert is idempotent. PSC, NICE Framework, corpus, and GSA CALC adapters are wired. The O*NET adapter
// stays dormant until ONET_API_KEY is set.
//
// Env: MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL), MM_ONTOLOGY_RAW_DIR (object storage root, default
// ./data/ontology-raw), MM_PSC_XLSX_URL (the PSC manual XLSX URL),
// MM_NICE_JSON_URL (the NICE components JSON URL, defaults to the NIST CPRT v2-2-0 published file),
// MM_CALC_BASE_URL (CALC ceiling rates endpoint, defaults to the public GSA API),
// DATA_GOV_API_KEY (optional, lifts the CALC gateway rate limit),
// ONET_API_KEY (optional, dormant adapter).
package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/modules/ontology"
	"github.com/missionworkforce/missionmatch/internal/modules/ontology/adapters"
	"github.com/missionworkforce/missionmatch/internal/platform/audit"
	"github.com/missionworkforce/missionmatch/internal/platform/storage"
	"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)

func main() {
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	if dsn == "" {
		log.Fatal("ontology-sync: set MM_MIGRATOR_DATABASE_URL (the migrator DSN)")
	}
	rawDir := firstNonEmpty(os.Getenv("MM_ONTOLOGY_RAW_DIR"), "./data/ontology-raw")
	pscURL := strings.TrimSpace(os.Getenv("MM_PSC_XLSX_URL"))
	niceURL := firstNonEmpty(os.Getenv("MM_NICE_JSON_URL"), niceDefaultURL)
	calcBase := firstNonEmpty(os.Getenv("MM_CALC_BASE_URL"), calcDefaultURL)

	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer cancel()

	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("ontology-sync: connect: %v", err)
	}
	defer pool.Close()

	store, err := storage.NewLocal(rawDir)
	if err != nil {
		log.Fatalf("ontology-sync: storage: %v", err)
	}

	ing := ontology.NewIngester(store, ontology.NewDBWriter(pool), audit.New(tenant.New(pool)))

	var sources []ontology.OntologySource
	if pscURL != "" {
		sources = append(sources, adapters.NewPSCSource(pscURL, &http.Client{Timeout: 60 * time.Second}))
	} else {
		log.Print("ontology-sync: MM_PSC_XLSX_URL not set, skipping the PSC adapter")
	}
	sources = append(sources, adapters.NewCorpusSource(adapters.NewPoolText(pool)))
	if niceURL != "" {
		sources = append(sources, adapters.NewNICESource(niceURL, &http.Client{Timeout: 120 * time.Second}))
	} else {
		log.Print("ontology-sync: MM_NICE_JSON_URL not set, skipping the NICE adapter")
	}
	// GSA CALC ceiling rates adapter: enumerates the federal labor category vocabulary by sweeping
	// a seed list of common terms. No auth required; DATA_GOV_API_KEY lifts the gateway rate limit.
	sources = append(sources, adapters.NewCALCSource(calcBase, os.Getenv("DATA_GOV_API_KEY"), nil, &http.Client{Timeout: 60 * time.Second}))

	// The O*NET adapter is active only when ONET_API_KEY is set. Without a key the adapter is
	// dormant and the skip is logged so operators know it is intentional, not a bug.
	if onetKey := strings.TrimSpace(os.Getenv("ONET_API_KEY")); onetKey != "" {
		sources = append(sources, adapters.NewONETSource(onetKey, &http.Client{Timeout: 30 * time.Second}))
	} else {
		log.Print("ontology-sync: ONET_API_KEY not set, O*NET adapter is dormant (skipped)")
	}

	if len(sources) == 0 {
		log.Fatal("ontology-sync: no sources configured")
	}

	var grand int
	for _, src := range sources {
		res, ierr := ing.Ingest(ctx, src)
		if ierr != nil {
			log.Fatalf("ontology-sync: ingest %s: %v", src.Name(), ierr)
		}
		log.Printf("ontology-sync: %s fetched %d upserted %d by_kind %v", res.Source, res.Fetched, res.Upserted, res.ByKind)
		grand += res.Upserted
	}
	log.Printf("ontology-sync: done, %d terms upserted into ontology_terms", grand)
}

// niceDefaultURL is the canonical NIST CPRT v2.2.0 NICE components JSON location. It is public
// domain and requires no authentication. Set MM_NICE_JSON_URL to override (for example to pin a
// mirror or a later version), or set it to empty to disable the adapter.
const niceDefaultURL = "https://csrc.nist.gov/csrc/media/Projects/cprt/documents/nice/v2-2-0_nf_components.json"

// calcDefaultURL is the GSA CALC v3 ceiling rates API base URL. It requires no authentication
// though DATA_GOV_API_KEY lifts the rate limit. Set MM_CALC_BASE_URL to override.
const calcDefaultURL = "https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/"

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}