Janady13/missionmatch / internal/modules/ontology/adapters/calc.go

1package adapters 2 3import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io" 8 "net/http" 9 "net/url" 10 "strings" 11 "time" 12 13 "github.com/missionworkforce/missionmatch/internal/modules/ontology" 14) 15 16// defaultCALCSeeds is the vocabulary sweep list used when the caller does not supply seeds. 17// Each term is sent as the q parameter to the CALC ceiling rates API to enumerate the distinct 18// labor category vocabulary. The terms are broad enough to cover the common GovCon labor 19// taxonomy without requiring a dedicated catalog endpoint (CALC has none). 20var defaultCALCSeeds = []string{ 21 "engineer", 22 "analyst", 23 "manager", 24 "architect", 25 "specialist", 26 "developer", 27 "scientist", 28 "administrator", 29 "consultant", 30 "technician", 31} 32 33// CALCSource ingests the GSA CALC (Contract-Awarded Labor Category) ceiling rates vocabulary by 34// sweeping a seed list of common terms against the CALC v3 API, deduplicating the returned labor 35// category strings, and yielding one KindLaborCategory VocabRecord per distinct category. 36// 37// The CALC API is Elasticsearch backed and has no dedicated catalog endpoint, so a seed sweep is 38// the correct enumeration strategy. The real response shape (confirmed from 39// https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/?q=engineer) is: 40// 41// hits.hits[]._source.labor_category 42type CALCSource struct { 43 baseURL string 44 apiKey string 45 seeds []string 46 client *http.Client 47} 48 49// NewCALCSource builds the adapter. baseURL is the CALC ceiling rates endpoint (injected so tests 50// can point at an httptest.Server). apiKey is the optional DATA_GOV_API_KEY that lifts the gateway 51// rate limit; pass an empty string when no key is available. seeds overrides the default seed list; 52// pass nil to use the built-in defaults. client is the HTTP client; pass nil for a 60s default. 53func NewCALCSource(baseURL, apiKey string, seeds []string, client *http.Client) *CALCSource { 54 if client == nil { 55 client = &http.Client{Timeout: 60 * time.Second} 56 } 57 if len(seeds) == 0 { 58 seeds = defaultCALCSeeds 59 } 60 return &CALCSource{ 61 baseURL: strings.TrimRight(baseURL, "/") + "/", 62 apiKey: [credential removed], 63 seeds: seeds, 64 client: client, 65 } 66} 67 68// Name is the provenance label stamped on every record this source yields. 69func (s *CALCSource) Name() string { return "GSA CALC" } 70 71// Fetch sweeps the seed list against the CALC API, deduplicates labor categories across all 72// seed responses, and yields one VocabRecord per distinct labor category. A 429 from the 73// provider returns ontology.ErrThrottled so the ingester can stop gracefully. 74func (s *CALCSource) Fetch(ctx context.Context, yield func(ontology.VocabRecord) error) error { 75 seen := make(map[string]bool) 76 77 for _, seed := range s.seeds { 78 raw, status, err := s.get(ctx, seed) 79 if err != nil { 80 return fmt.Errorf("calc: fetch seed %q: %w", seed, err) 81 } 82 if status == http.StatusTooManyRequests { 83 return ontology.ErrThrottled 84 } 85 if status != http.StatusOK { 86 return fmt.Errorf("calc: seed %q: provider returned status %d", seed, status) 87 } 88 89 categories, err := parseCALC(raw) 90 if err != nil { 91 return fmt.Errorf("calc: parse seed %q: %w", seed, err) 92 } 93 94 for _, lc := range categories { 95 key := strings.ToLower(lc) 96 if seen[key] { 97 continue 98 } 99 seen[key] = true 100 101 rawJSON, _ := json.Marshal(map[string]string{ 102 "labor_category": lc, 103 "source": "GSA CALC ceiling rates API", 104 }) 105 if err := yield(ontology.VocabRecord{ 106 SourceName: "GSA CALC", 107 SourceRecordID: key, 108 Kind: ontology.KindLaborCategory, 109 Term: lc, 110 Category: "labor_category", 111 Raw: rawJSON, 112 }); err != nil { 113 return err 114 } 115 } 116 } 117 return nil 118} 119 120// get performs one GET request against the CALC ceiling rates endpoint for the given search term 121// and returns the raw body bytes plus the HTTP status code. 122func (s *CALCSource) get(ctx context.Context, q string) ([]byte, int, error) { 123 u, err := url.Parse(s.baseURL) 124 if err != nil { 125 return nil, 0, fmt.Errorf("calc: parse baseURL: %w", err) 126 } 127 params := url.Values{"q": {q}} 128 if s.apiKey != "" { 129 params.Set("api_key", s.apiKey) 130 } 131 u.RawQuery = params.Encode() 132 133 req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) 134 if err != nil { 135 return nil, 0, fmt.Errorf("calc: new request: %w", err) 136 } 137 resp, err := s.client.Do(req) 138 if err != nil { 139 return nil, 0, fmt.Errorf("calc: do request: %w", err) 140 } 141 defer resp.Body.Close() 142 143 body, err := io.ReadAll(resp.Body) 144 if err != nil { 145 return nil, resp.StatusCode, fmt.Errorf("calc: read body: %w", err) 146 } 147 return body, resp.StatusCode, nil 148} 149 150// calcResponse mirrors the Elasticsearch backed shape of the GSA CALC v3 ceiling rates API. 151// Confirmed from https://api.gsa.gov/acquisition/calc/v3/api/ceilingrates/?q=engineer: 152// labor category lives at hits.hits[]._source.labor_category. 153type calcResponse struct { 154 Hits struct { 155 Hits []struct { 156 Source struct { 157 LaborCategory string `json:"labor_category"` 158 } `json:"_source"` 159 } `json:"hits"` 160 } `json:"hits"` 161} 162 163// parseCALC extracts the distinct labor category strings from one CALC ceiling rates response 164// page. It is kept pure (no I/O) so it is directly unit testable. Deduplication within a single 165// page is applied here; cross-seed deduplication is the caller's responsibility. 166func parseCALC(data []byte) ([]string, error) { 167 var doc calcResponse 168 if err := json.Unmarshal(data, &doc); err != nil { 169 return nil, fmt.Errorf("calc: decode JSON: %w", err) 170 } 171 seen := make(map[string]bool) 172 var out []string 173 for _, hit := range doc.Hits.Hits { 174 lc := strings.TrimSpace(hit.Source.LaborCategory) 175 if lc == "" { 176 continue 177 } 178 key := strings.ToLower(lc) 179 if !seen[key] { 180 seen[key] = true 181 out = append(out, lc) 182 } 183 } 184 return out, nil 185} 186