Janady13/missionmatch / cmd/ontology-sync/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/ontology-sync/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	1e9249840a4302ec82e123b9c9cb6f75badea3e4
Branch	master
Path or archive member	cmd/ontology-sync/main.go
Original SHA-256	e4b5fc7e3b852722a364b9fb8cea2c7afaf3dfe38480f7ecdc8cdf9069f1a45f
Git blob SHA	455df1cc1e9f4788e15cbf431516ab9a28d7f330
Reading-copy SHA-256	e4b5fc7e3b852722a364b9fb8cea2c7afaf3dfe38480f7ecdc8cdf9069f1a45f

Original source ↗

Every recorded occurrence

These 35 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/ontology-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/ontology-sync/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓114 lines · 4.7 KB

// Command ontology-sync ingests the GovCon vocabulary from free federal sources into the global
// ontology_terms table, mirroring cmd/opp-sync. It runs as the migrator role and is safe to rerun: every
// upsert is idempotent. PSC, NICE Framework, corpus, and GSA CALC adapters are wired. The O*NET adapter
// stays dormant until ONET_API_KEY is set.
//
// Env: MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL), MM_ONTOLOGY_RAW_DIR (object storage root, default
// ./data/ontology-raw), MM_PSC_XLSX_URL (the PSC manual XLSX URL),
// MM_NICE_JSON_URL (the NICE components JSON URL, defaults to the NIST CPRT v2-2-0 published file),
// MM_CALC_BASE_URL (CALC ceiling rates endpoint, defaults to the public GSA API),
// DATA_GOV_API_KEY (optional, lifts the CALC gateway rate limit),
// ONET_API_KEY (optional, dormant adapter).
package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/modules/ontology"
	"github.com/missionworkforce/missionmatch/internal/modules/ontology/adapters"
	"github.com/missionworkforce/missionmatch/internal/platform/audit"
	"github.com/missionworkforce/missionmatch/internal/platform/storage"
	"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)

func main() {
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	if dsn == "" {
		log.Fatal("ontology-sync: set MM_MIGRATOR_DATABASE_URL (the migrator DSN)")
	}
	rawDir   := firstNonEmpty(os.Getenv("MM_ONTOLOGY_RAW_DIR"), "./data/ontology-raw")
	pscURL   := strings.TrimSpace(os.Getenv("MM_PSC_XLSX_URL"))
	niceURL  := firstNonEmpty(os.Getenv("MM_NICE_JSON_URL"), niceDefaultURL)
	calcBase := firstNonEmpty(os.Getenv("MM_CALC_BASE_URL"), calcDefaultURL)

	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer cancel()

	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("ontology-sync: connect: %v", err)
	}
	defer pool.Close()

	store, err := storage.NewLocal(rawDir)
	if err != nil {
		log.Fatalf("ontology-sync: storage: %v", err)
	}

	ing := ontology.NewIngester(store, ontology.NewDBWriter(pool), audit.New(tenant.New(pool)))

	var sources []ontology.OntologySource
	if pscURL != "" {
		sources = append(sources, adapters.NewPSCSource(pscURL, &http.Client{Timeout: 60 * time.Second}))
	} else {
		log.Print("ontology-sync: MM_PSC_XLSX_URL not set, skipping the PSC adapter")
	}
	sources = append(sources, adapters.NewCorpusSource(adapters.NewPoolText(pool)))
	if niceURL != "" {
		sources = append(sources, adapters.NewNICESource(niceURL, &http.Client{Timeout: 120 * time.Second}))
	} else {
		log.Print("ontology-sync: MM_NICE_JSON_URL not set, skipping the NICE adapter")
	}
	// GSA CALC ceiling rates adapter: enumerates the federal labor category vocabulary by sweeping
	// a seed list of common terms. No auth required; DATA_GOV_API_KEY lifts the gateway rate limit.
	sources = append(sources, adapters.NewCALCSource(calcBase, os.Getenv("DATA_GOV_API_KEY"), nil, &http.Client{Timeout: 60 * time.Second}))

	// The O*NET adapter is active only when ONET_API_KEY is set. Without a key the adapter is
	// dormant and the skip is logged so operators know it is intentional, not a bug.
	if onetKey := strings.TrimSpace(os.Getenv("ONET_API_KEY")); onetKey != "" {
		sources = append(sources, adapters.NewONETSource(onetKey, &http.Client{Timeout: 30 * time.Second}))
	} else {
		log.Print("ontology-sync: ONET_API_KEY not set, O*NET adapter is dormant (skipped)")
	}

	if len(sources) == 0 {
		log.Fatal("ontology-sync: no sources configured")
	}

	var grand int
	for _, src := range sources {
		res, ierr := ing.Ingest(ctx, src)
		if ierr != nil {
			log.Fatalf("ontology-sync: ingest %s: %v", src.Name(), ierr)
		}
		log.Printf("ontology-sync: %s fetched %d upserted %d by_kind %v", res.Source, res.Fetched, res.Upserted, res.ByKind)
		grand += res.Upserted
	}
	log.Printf("ontology-sync: done, %d terms upserted into ontology_terms", grand)
}

// niceDefaultURL is the canonical NIST CPRT v2.2.0 NICE components JSON location. It is public
// domain and requires no authentication. Set MM_NICE_JSON_URL to override (for example to pin a
// mirror or a later version), or set it to empty to disable the adapter.
const niceDefaultURL = "https://csrc.nist.gov/csrc/media/Projects/cprt/documents/nice/v2-2-0_nf_components.json"

// calcDefaultURL is the GSA CALC v3 ceiling rates API base URL. It requires no authentication
// though DATA_GOV_API_KEY lifts the rate limit. Set MM_CALC_BASE_URL to override.
const calcDefaultURL = "https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/"

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}