Janady13/missionmatch / internal/platform/samgov/client.go

1package samgov 2 3import ( 4 "context" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "io" 9 "log" 10 "net/http" 11 "net/url" 12 "strconv" 13 "strings" 14 "time" 15) 16 17// Base endpoints. Prod is the default; the dev alpha (prodlike) is selectable via SAM_API_BASE. 18const ( 19 BaseProd = "https://api.sam.gov/prod/opportunities/v2/search" 20 BaseAlpha = "https://api-alpha.sam.gov/prodlike/opportunities/v2/search" 21) 22 23const ( 24 maxLimit = 1000 // SAM hard cap 25 samDateLayout = "01/02/2006" 26 throttleCap = 1024 // max body bytes captured from a 429 27 pageGap = 250 * time.Millisecond 28 maxWindow = 364 * 24 * time.Hour // SAM rejects 365+ civil days 29) 30 31// civilDate truncates t to midnight UTC of its calendar date (SAM compares date parts only). 32func civilDate(t time.Time) time.Time { 33 return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) 34} 35 36// ErrThrottled is the sentinel for a SAM 429 (daily quota). Detect with errors.Is(err, ErrThrottled). 37var ErrThrottled = errors.New("samgov: rate limited (429)") 38 39// ThrottleError carries the Retry-After hint (SAM points it at midnight UTC). It satisfies 40// errors.Is(err, ErrThrottled). 41type ThrottleError struct { 42 RetryAfter string 43 Body string 44} 45 46func (e *ThrottleError) Error() string { 47 if e.RetryAfter != "" { 48 return fmt.Sprintf("samgov: rate limited (429), retry after %s", e.RetryAfter) 49 } 50 return "samgov: rate limited (429)" 51} 52 53func (e *ThrottleError) Is(target error) bool { return target == ErrThrottled } 54 55// Client is a typed SAM search client over the built-in net/http. The api_key is held here and only 56// ever added as a query parameter, it is never placed in logs or error strings. 57type Client struct { 58 base string 59 key string 60 http *http.Client 61} 62 63// New builds a client. An empty base defaults to prod. 64func New(base, key string) *Client { 65 if strings.TrimSpace(base) == "" { 66 base = BaseProd 67 } 68 return &Client{base: base, key: key, http: &http.Client{Timeout: 45 * time.Second}} 69} 70 71// SearchParams are the supported filters. NAICS is one 6 digit code per call (ncode). The window 72// [PostedFrom, PostedTo] may not exceed one year. 73type SearchParams struct { 74 PostedFrom time.Time 75 PostedTo time.Time 76 NAICS string // ncode 77 State string // place of performance state 78 SetAside string // typeOfSetAside (combining with ncode can return zero, the sync omits it) 79 PType string // notice type: o,k,p,r,a 80 Limit int // 1..1000 81 Offset int 82} 83 84// Search fetches a single page. It enforces the one year window BEFORE any network call and maps a 85// 429 to *ThrottleError. 86func (c *Client) Search(ctx context.Context, p SearchParams) (SearchResult, error) { 87 if p.PostedFrom.IsZero() || p.PostedTo.IsZero() { 88 return SearchResult{}, errors.New("samgov: postedFrom and postedTo are required") 89 } 90 // SAM compares the date parts only (MM/dd/yyyy) and rejects a span of 365 civil days or more with a 91 // 400 ("must be 1 year(s) apart"), so the effective maximum is 364 days. Compare on civil dates so a 92 // wall clock time or a DST boundary cannot tip a valid request over the edge. 93 fromDay := civilDate(p.PostedFrom) 94 toDay := civilDate(p.PostedTo) 95 if toDay.Before(fromDay) { 96 return SearchResult{}, errors.New("samgov: postedTo is before postedFrom") 97 } 98 if toDay.Sub(fromDay) > maxWindow { 99 return SearchResult{}, fmt.Errorf("samgov: window %s..%s exceeds SAM's maximum of 364 days", 100 fromDay.Format("2006-01-02"), toDay.Format("2006-01-02")) 101 } 102 103 limit := p.Limit 104 if limit < 1 { 105 limit = 1 106 } 107 if limit > maxLimit { 108 limit = maxLimit 109 } 110 111 q := url.Values{} 112 q.Set("api_key", c.key) 113 q.Set("postedFrom", p.PostedFrom.Format(samDateLayout)) 114 q.Set("postedTo", p.PostedTo.Format(samDateLayout)) 115 q.Set("limit", strconv.Itoa(limit)) 116 q.Set("offset", strconv.Itoa(p.Offset)) 117 if p.NAICS != "" { 118 q.Set("ncode", p.NAICS) 119 } 120 if p.State != "" { 121 q.Set("state", p.State) 122 } 123 if p.SetAside != "" { 124 q.Set("typeOfSetAside", p.SetAside) 125 } 126 if p.PType != "" { 127 q.Set("ptype", p.PType) 128 } 129 130 req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"?"+q.Encode(), nil) 131 if err != nil { 132 // A parse error would echo the full URL (incl. ?api_key=…), keep it generic [review SEC-1]. 133 return SearchResult{}, errors.New("samgov: failed to build request") 134 } 135 req.Header.Set("Accept", "application/json") 136 137 resp, err := c.http.Do(req) 138 if err != nil { 139 // *url.Error.Error() embeds the full request URL incl. ?api_key=…. NEVER wrap it: ue.Err is the 140 // transport-level cause and carries host:port at most, never the query string [review SEC-1]. 141 var ue *url.Error 142 if errors.As(err, &ue) { 143 return SearchResult{}, fmt.Errorf("samgov: %s request failed: %v", ue.Op, ue.Err) 144 } 145 return SearchResult{}, errors.New("samgov: request failed") 146 } 147 defer func() { _ = resp.Body.Close() }() 148 149 if resp.StatusCode == http.StatusTooManyRequests { 150 snippet, _ := io.ReadAll(io.LimitReader(resp.Body, throttleCap)) 151 return SearchResult{}, &ThrottleError{RetryAfter: resp.Header.Get("Retry-After"), Body: string(snippet)} 152 } 153 if resp.StatusCode != http.StatusOK { 154 snippet, _ := io.ReadAll(io.LimitReader(resp.Body, throttleCap)) 155 return SearchResult{}, fmt.Errorf("samgov: status %d: %s", resp.StatusCode, strings.TrimSpace(string(snippet))) 156 } 157 158 var page rawPage 159 if err := json.NewDecoder(resp.Body).Decode(&page); err != nil { 160 return SearchResult{}, fmt.Errorf("samgov: decode response: %w", err) 161 } 162 163 recs := make([]Record, 0, len(page.OpportunitiesData)) 164 skipped := 0 165 for _, raw := range page.OpportunitiesData { 166 var op Opportunity 167 if err := json.Unmarshal(raw, &op); err != nil { 168 // A type regression on ONE notice must not discard the whole page and halt pagination for 169 // this NAICS. Record.Raw still holds the verbatim JSON downstream, so resilience matches the 170 // documented promise; count the skip and continue. Do not log the notice (it may carry CUI). 171 skipped++ 172 continue 173 } 174 recs = append(recs, Record{Op: op, Raw: raw}) 175 } 176 if skipped > 0 { 177 log.Printf("samgov: skipped %d malformed notice(s) on this page", skipped) 178 } 179 return SearchResult{ 180 TotalRecords: page.TotalRecords, Limit: page.Limit, Offset: page.Offset, 181 Fetched: len(page.OpportunitiesData), Records: recs, 182 }, nil 183} 184 185// SearchAll paginates by offset until every record is retrieved or pageCap pages have been fetched. 186// It returns whatever it has gathered so far alongside any error, so a 429 during pagination yields 187// the partial set plus ErrThrottled, letting the caller persist progress and stop gracefully. capped 188// reports whether the page cap (not exhaustion) ended the run. 189func (c *Client) SearchAll(ctx context.Context, p SearchParams, pageCap int) (recs []Record, total int, capped bool, err error) { 190 if pageCap < 1 { 191 pageCap = 1 192 } 193 if p.Limit < 1 || p.Limit > maxLimit { 194 p.Limit = maxLimit 195 } 196 offset := p.Offset 197 for page := 0; page < pageCap; page++ { 198 pp := p 199 pp.Offset = offset 200 res, e := c.Search(ctx, pp) 201 if e != nil { 202 return recs, total, false, e // includes *ThrottleError; caller keeps recs gathered so far 203 } 204 total = res.TotalRecords 205 recs = append(recs, res.Records...) 206 // Advance by what SAM RETURNED (Fetched), not by what we kept, so a skipped malformed notice does 207 // not re fetch or stall pagination. A page that decodes entirely empty still advances. 208 offset += res.Fetched 209 if res.Fetched == 0 || offset >= total { 210 return recs, total, false, nil // exhausted 211 } 212 select { 213 case <-ctx.Done(): 214 return recs, total, false, ctx.Err() 215 case <-time.After(pageGap): // be gentle, never hammer the API 216 } 217 } 218 // Ran out of pages before exhausting the result set. 219 return recs, total, offset < total, nil 220} 221