package adapters import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/missionworkforce/missionmatch/internal/modules/ontology" ) // defaultCALCSeeds is the vocabulary sweep list used when the caller does not supply seeds. // Each term is sent as the q parameter to the CALC ceiling rates API to enumerate the distinct // labor category vocabulary. The terms are broad enough to cover the common GovCon labor // taxonomy without requiring a dedicated catalog endpoint (CALC has none). var defaultCALCSeeds = []string{ "engineer", "analyst", "manager", "architect", "specialist", "developer", "scientist", "administrator", "consultant", "technician", } // CALCSource ingests the GSA CALC (Contract-Awarded Labor Category) ceiling rates vocabulary by // sweeping a seed list of common terms against the CALC v3 API, deduplicating the returned labor // category strings, and yielding one KindLaborCategory VocabRecord per distinct category. // // The CALC API is Elasticsearch backed and has no dedicated catalog endpoint, so a seed sweep is // the correct enumeration strategy. The real response shape (confirmed from // https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/?q=engineer) is: // // hits.hits[]._source.labor_category type CALCSource struct { baseURL string apiKey string seeds []string client *http.Client } // NewCALCSource builds the adapter. baseURL is the CALC ceiling rates endpoint (injected so tests // can point at an httptest.Server). apiKey is the optional DATA_GOV_API_KEY that lifts the gateway // rate limit; pass an empty string when no key is available. seeds overrides the default seed list; // pass nil to use the built-in defaults. client is the HTTP client; pass nil for a 60s default. func NewCALCSource(baseURL, apiKey string, seeds []string, client *http.Client) *CALCSource { if client == nil { client = &http.Client{Timeout: 60 * time.Second} } if len(seeds) == 0 { seeds = defaultCALCSeeds } return &CALCSource{ baseURL: strings.TrimRight(baseURL, "/") + "/", apiKey: [credential removed], seeds: seeds, client: client, } } // Name is the provenance label stamped on every record this source yields. func (s *CALCSource) Name() string { return "GSA CALC" } // Fetch sweeps the seed list against the CALC API, deduplicates labor categories across all // seed responses, and yields one VocabRecord per distinct labor category. A 429 from the // provider returns ontology.ErrThrottled so the ingester can stop gracefully. func (s *CALCSource) Fetch(ctx context.Context, yield func(ontology.VocabRecord) error) error { seen := make(map[string]bool) for _, seed := range s.seeds { raw, status, err := s.get(ctx, seed) if err != nil { return fmt.Errorf("calc: fetch seed %q: %w", seed, err) } if status == http.StatusTooManyRequests { return ontology.ErrThrottled } if status != http.StatusOK { return fmt.Errorf("calc: seed %q: provider returned status %d", seed, status) } categories, err := parseCALC(raw) if err != nil { return fmt.Errorf("calc: parse seed %q: %w", seed, err) } for _, lc := range categories { key := strings.ToLower(lc) if seen[key] { continue } seen[key] = true rawJSON, _ := json.Marshal(map[string]string{ "labor_category": lc, "source": "GSA CALC ceiling rates API", }) if err := yield(ontology.VocabRecord{ SourceName: "GSA CALC", SourceRecordID: key, Kind: ontology.KindLaborCategory, Term: lc, Category: "labor_category", Raw: rawJSON, }); err != nil { return err } } } return nil } // get performs one GET request against the CALC ceiling rates endpoint for the given search term // and returns the raw body bytes plus the HTTP status code. func (s *CALCSource) get(ctx context.Context, q string) ([]byte, int, error) { u, err := url.Parse(s.baseURL) if err != nil { return nil, 0, fmt.Errorf("calc: parse baseURL: %w", err) } params := url.Values{"q": {q}} if s.apiKey != "" { params.Set("api_key", s.apiKey) } u.RawQuery = params.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, 0, fmt.Errorf("calc: new request: %w", err) } resp, err := s.client.Do(req) if err != nil { return nil, 0, fmt.Errorf("calc: do request: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, resp.StatusCode, fmt.Errorf("calc: read body: %w", err) } return body, resp.StatusCode, nil } // calcResponse mirrors the Elasticsearch backed shape of the GSA CALC v3 ceiling rates API. // Confirmed from https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/?q=engineer: // labor category lives at hits.hits[]._source.labor_category. type calcResponse struct { Hits struct { Hits []struct { Source struct { LaborCategory string `json:"labor_category"` } `json:"_source"` } `json:"hits"` } `json:"hits"` } // parseCALC extracts the distinct labor category strings from one CALC ceiling rates response // page. It is kept pure (no I/O) so it is directly unit testable. Deduplication within a single // page is applied here; cross-seed deduplication is the caller's responsibility. func parseCALC(data []byte) ([]string, error) { var doc calcResponse if err := json.Unmarshal(data, &doc); err != nil { return nil, fmt.Errorf("calc: decode JSON: %w", err) } seen := make(map[string]bool) var out []string for _, hit := range doc.Hits.Hits { lc := strings.TrimSpace(hit.Source.LaborCategory) if lc == "" { continue } key := strings.ToLower(lc) if !seen[key] { seen[key] = true out = append(out, lc) } } return out, nil }