Janady13/missionmatch / internal/platform/samawards/client.go

1// Package samawards is a typed client for the SAM.gov Contract Awards API 2// (open.gsa.gov/api/contract-awards), the RESTful replacement for the retiring FPDS Atom feed. It 3// fetches federal contract award and incumbent data using the same SAM api_key the opportunities feed 4// uses, and normalizes it to a flat app shape. It has no database dependency: it fetches and 5// normalizes, the ingest writes the cache. The api_key is held here and only ever added as a query 6// parameter, never logged or placed in an error string. 7package samawards 8 9import ( 10 "context" 11 "encoding/json" 12 "errors" 13 "fmt" 14 "io" 15 "net/http" 16 "net/url" 17 "strconv" 18 "strings" 19 "time" 20) 21 22// Base endpoints. Prod is the default; the alpha (prodlike) is selectable via the env for testing. 23const ( 24 BaseProd = "https://api.sam.gov/contract-awards/v1/search" 25 BaseAlpha = "https://api-alpha.sam.gov/contract-awards/v1/search" 26) 27 28const ( 29 maxLimit = 100 // the API caps a page at 100 30 offsetWindow = 400000 // offset times limit may not exceed this 31 throttleCap = 1024 // max body bytes captured from a 429 32 pageGap = 250 * time.Millisecond 33) 34 35// ErrThrottled is the sentinel for a 429 (daily quota). Detect with errors.Is(err, ErrThrottled). 36var ErrThrottled = errors.New("samawards: rate limited (429)") 37 38// ThrottleError carries the Retry-After hint. It satisfies errors.Is(err, ErrThrottled). 39type ThrottleError struct { 40 RetryAfter string 41 Body string 42} 43 44func (e *ThrottleError) Error() string { 45 if e.RetryAfter != "" { 46 return fmt.Sprintf("samawards: rate limited (429), retry after %s", e.RetryAfter) 47 } 48 return "samawards: rate limited (429)" 49} 50 51func (e *ThrottleError) Is(target error) bool { return target == ErrThrottled } 52 53// Client is a typed Contract Awards client over the built in net/http. 54type Client struct { 55 base string 56 key string 57 http *http.Client 58} 59 60// New builds a client. An empty base defaults to prod. 61func New(base, key string) *Client { 62 if strings.TrimSpace(base) == "" { 63 base = BaseProd 64 } 65 return &Client{base: base, key: key, http: &http.Client{Timeout: 60 * time.Second}} 66} 67 68// SearchParams are the supported filters. NAICS is up to 100 codes (the API joins them with a tilde). 69// Department is a partial contracting department name (for example DEFENSE). AwardeeUEIs scopes to 70// specific recipients for own past performance. The Signed window filters by dateSigned. 71type SearchParams struct { 72 NAICS []string 73 Department string 74 AwardeeUEIs []string 75 SignedFrom time.Time 76 SignedTo time.Time 77 Limit int 78 Offset int 79} 80 81// SearchResult is one page: the raw award records and the total match count the API reports. 82type SearchResult struct { 83 Records []json.RawMessage 84 Total int 85} 86 87// Search fetches one page. It maps a 429 to *ThrottleError and any other non 200 to an error that never 88// includes the api_key. 89func (c *Client) Search(ctx context.Context, p SearchParams) (SearchResult, error) { 90 limit := p.Limit 91 if limit < 1 { 92 limit = 10 93 } 94 if limit > maxLimit { 95 limit = maxLimit 96 } 97 q := url.Values{} 98 q.Set("api_key", c.key) 99 q.Set("limit", strconv.Itoa(limit)) 100 q.Set("offset", strconv.Itoa(p.Offset)) 101 if len(p.NAICS) > 0 { 102 q.Set("naicsCode", strings.Join(cap100(p.NAICS), "~")) 103 } 104 if d := strings.TrimSpace(p.Department); d != "" { 105 q.Set("contractingDepartmentName", d) 106 } 107 if len(p.AwardeeUEIs) > 0 { 108 q.Set("awardeeUniqueEntityId", strings.Join(cap100(p.AwardeeUEIs), "~")) 109 } 110 if !p.SignedFrom.IsZero() && !p.SignedTo.IsZero() { 111 q.Set("dateSigned", fmt.Sprintf("[%s,%s]", p.SignedFrom.Format("01/02/2006"), p.SignedTo.Format("01/02/2006"))) 112 } 113 114 req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"?"+q.Encode(), nil) 115 if err != nil { 116 return SearchResult{}, err 117 } 118 req.Header.Set("Accept", "application/json") 119 120 resp, err := c.http.Do(req) 121 if err != nil { 122 return SearchResult{}, fmt.Errorf("samawards: request: %w", err) 123 } 124 defer resp.Body.Close() 125 126 if resp.StatusCode == http.StatusTooManyRequests { 127 body, _ := io.ReadAll(io.LimitReader(resp.Body, throttleCap)) 128 return SearchResult{}, &ThrottleError{RetryAfter: resp.Header.Get("Retry-After"), Body: string(body)} 129 } 130 if resp.StatusCode != http.StatusOK { 131 body, _ := io.ReadAll(io.LimitReader(resp.Body, throttleCap)) 132 return SearchResult{}, fmt.Errorf("samawards: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) 133 } 134 135 var env struct { 136 AwardSummary []json.RawMessage `json:"awardSummary"` 137 TotalRecords NumericText `json:"totalRecords"` 138 } 139 if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { 140 return SearchResult{}, fmt.Errorf("samawards: decode: %w", err) 141 } 142 return SearchResult{Records: env.AwardSummary, Total: env.TotalRecords.Int()}, nil 143} 144 145// NumericText decodes a field the Contract Awards API returns inconsistently as either a JSON number or 146// a quoted string (totalRecords and the dollar amounts both arrive as strings) into the literal text, so 147// both forms parse the same way downstream. This was a real wire mismatch: the count and the dollars are 148// strings on the wire, which a plain int or json.Number field cannot decode. 149type NumericText string 150 151// UnmarshalJSON accepts a JSON number, a quoted string, or null. 152func (t *NumericText) UnmarshalJSON(b []byte) error { 153 s := strings.TrimSpace(string(b)) 154 if s == "null" { 155 *t = "" 156 return nil 157 } 158 *t = NumericText(strings.TrimSpace(strings.Trim(s, `"`))) 159 return nil 160} 161 162// Int parses the text as an integer, tolerating a decimal point. It returns 0 when the text is empty or 163// unparseable, which for a record count safely stops pagination rather than looping. 164func (t NumericText) Int() int { 165 s := string(t) 166 if n, err := strconv.Atoi(s); err == nil { 167 return n 168 } 169 if f, err := strconv.ParseFloat(s, 64); err == nil { 170 return int(f) 171 } 172 return 0 173} 174 175// SearchAll paginates by offset up to pageCap pages, stopping at the total or the API offset ceiling. On 176// a 429 it returns the records collected so far together with ErrThrottled so the caller keeps the 177// partial pull and stops gracefully. 178func (c *Client) SearchAll(ctx context.Context, p SearchParams, pageCap int) (recs []json.RawMessage, total int, capped bool, err error) { 179 limit := p.Limit 180 if limit < 1 { 181 limit = maxLimit 182 } 183 if limit > maxLimit { 184 limit = maxLimit 185 } 186 for page := 0; page < pageCap; page++ { 187 pp := p 188 pp.Limit = limit 189 pp.Offset = page * limit 190 if pp.Offset+limit > offsetWindow { 191 capped = true 192 break 193 } 194 res, e := c.Search(ctx, pp) 195 if e != nil { 196 if errors.Is(e, ErrThrottled) { 197 return recs, total, true, ErrThrottled 198 } 199 return recs, total, capped, e 200 } 201 total = res.Total 202 recs = append(recs, res.Records...) 203 if len(res.Records) < limit || len(recs) >= total { 204 return recs, total, capped, nil 205 } 206 time.Sleep(pageGap) 207 } 208 if len(recs) < total { 209 capped = true 210 } 211 return recs, total, capped, nil 212} 213 214// cap100 caps a slice to the API limit of 100 values per multi value parameter. 215func cap100(s []string) []string { 216 if len(s) > 100 { 217 return s[:100] 218 } 219 return s 220} 221