package marketintel import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "strings" "time" "github.com/shopspring/decimal" ) // USASpendingBase is the public spending_by_award endpoint (no key required). const USASpendingBase = "https://api.usaspending.gov/api/v2/search/spending_by_award/" const usaThrottleCap = 1024 // ErrThrottled is the sentinel for a 429 from the awards provider. var ErrThrottled = errors.New("marketintel: rate limited (429)") // awardFields are the exact result fields requested (validated live against the API). var awardFields = []string{ "Award ID", "Recipient Name", "Recipient UEI", "Award Amount", "Awarding Agency", "Awarding Sub Agency", "Period of Performance Start Date", "Period of Performance Current End Date", "NAICS", "Description", "recipient_id", "generated_internal_id", } // USASpending is the free Provider implementation over the built in net/http. type USASpending struct { base string http *http.Client } // NewUSASpending builds the provider. An empty base uses the public endpoint. func NewUSASpending(base string) *USASpending { if strings.TrimSpace(base) == "" { base = USASpendingBase } return &USASpending{base: base, http: &http.Client{Timeout: 60 * time.Second}} } func (u *USASpending) Name() string { return "USASpending" } // FetchAwards posts one page of spending_by_award and normalizes the results. func (u *USASpending) FetchAwards(ctx context.Context, q AwardQuery) ([]Award, bool, error) { limit := q.Limit if limit < 1 { limit = 100 } if limit > 100 { limit = 100 // USASpending caps page size at 100 } page := q.Page if page < 1 { page = 1 } filters := map[string]any{} if len(q.AwardTypeCodes) > 0 { filters["award_type_codes"] = q.AwardTypeCodes } if len(q.TimePeriods) > 0 { tp := make([]map[string]string, 0, len(q.TimePeriods)) for _, p := range q.TimePeriods { tp = append(tp, map[string]string{"start_date": p.Start, "end_date": p.End}) } filters["time_period"] = tp } if len(q.NAICS) > 0 { filters["naics_codes"] = q.NAICS } if len(q.States) > 0 { locs := make([]map[string]string, 0, len(q.States)) for _, s := range q.States { locs = append(locs, map[string]string{"country": "USA", "state": s}) } filters["place_of_performance_locations"] = locs } if q.AwardingToptier != "" { filters["agencies"] = []map[string]string{ {"type": "awarding", "tier": "toptier", "name": q.AwardingToptier}, } } if len(q.RecipientSearch) > 0 { filters["recipient_search_text"] = q.RecipientSearch } body := map[string]any{ "filters": filters, "fields": awardFields, "sort": "Award Amount", "order": "desc", "limit": limit, "page": page, } payload, err := json.Marshal(body) if err != nil { return nil, false, fmt.Errorf("marketintel: marshal request: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.base, bytes.NewReader(payload)) if err != nil { return nil, false, errors.New("marketintel: failed to build request") } req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") resp, err := u.http.Do(req) if err != nil { return nil, false, errors.New("marketintel: request failed") } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusTooManyRequests { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, usaThrottleCap)) return nil, false, ErrThrottled } if resp.StatusCode != http.StatusOK { snippet, _ := io.ReadAll(io.LimitReader(resp.Body, usaThrottleCap)) return nil, false, fmt.Errorf("marketintel: status %d: %s", resp.StatusCode, strings.TrimSpace(string(snippet))) } var page1 awardPage if err := json.NewDecoder(resp.Body).Decode(&page1); err != nil { return nil, false, fmt.Errorf("marketintel: decode response: %w", err) } awards := make([]Award, 0, len(page1.Results)) for _, raw := range page1.Results { a, ok := normalizeAward(raw, q.RecipientSearch != nil) if !ok { continue } awards = append(awards, a) } return awards, page1.PageMetadata.HasNext, nil } type awardPage struct { Results []json.RawMessage `json:"results"` PageMetadata struct { HasNext bool `json:"hasNext"` Page int `json:"page"` } `json:"page_metadata"` } type usaAward struct { GeneratedInternalID string `json:"generated_internal_id"` AwardID string `json:"Award ID"` RecipientName string `json:"Recipient Name"` RecipientUEI string `json:"Recipient UEI"` RecipientID string `json:"recipient_id"` AwardAmount json.Number `json:"Award Amount"` AwardingAgency string `json:"Awarding Agency"` AwardingSubAgency string `json:"Awarding Sub Agency"` POPStart *string `json:"Period of Performance Start Date"` POPEnd *string `json:"Period of Performance Current End Date"` NAICS struct { Code string `json:"code"` Description string `json:"description"` } `json:"NAICS"` Description string `json:"Description"` } // normalizeAward maps one raw result. isGalapagos marks own past performance pulled by recipient. func normalizeAward(raw json.RawMessage, isGalapagos bool) (Award, bool) { var u usaAward if err := json.Unmarshal(raw, &u); err != nil { return Award{}, false } id := strings.TrimSpace(u.GeneratedInternalID) if id == "" { return Award{}, false } a := Award{ GeneratedInternalID: id, AwardID: strings.TrimSpace(u.AwardID), RecipientName: strings.TrimSpace(u.RecipientName), RecipientUEI: strings.TrimSpace(u.RecipientUEI), RecipientID: strings.TrimSpace(u.RecipientID), AwardAmount: parseAmount(u.AwardAmount), AwardingAgency: strings.TrimSpace(u.AwardingAgency), AwardingSubAgency: strings.TrimSpace(u.AwardingSubAgency), POPStart: parseAwardDate(u.POPStart), POPEnd: parseAwardDate(u.POPEnd), NAICSCode: strings.TrimSpace(u.NAICS.Code), NAICSDescription: strings.TrimSpace(u.NAICS.Description), Description: strings.TrimSpace(u.Description), IsGalapagos: isGalapagos, Raw: raw, } return a, true } func parseAmount(n json.Number) *decimal.Decimal { s := strings.TrimSpace(n.String()) if s == "" { return nil } d, err := decimal.NewFromString(s) if err != nil { return nil } return &d } func parseAwardDate(s *string) *time.Time { if s == nil || strings.TrimSpace(*s) == "" { return nil } t, err := time.Parse("2006-01-02", strings.TrimSpace(*s)) if err != nil { return nil } return &t }