Janady13/missionmatch / internal/platform/marketintel/usaspending.go

1package marketintel 2 3import ( 4 "bytes" 5 "context" 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "net/http" 11 "strings" 12 "time" 13 14 "github.com/shopspring/decimal" 15) 16 17// USASpendingBase is the public spending_by_award endpoint (no key required). 18const USASpendingBase = "https://api.usaspending.gov/api/v2/search/spending_by_award/" 19 20const usaThrottleCap = 1024 21 22// ErrThrottled is the sentinel for a 429 from the awards provider. 23var ErrThrottled = errors.New("marketintel: rate limited (429)") 24 25// awardFields are the exact result fields requested (validated live against the API). 26var awardFields = []string{ 27 "Award ID", "Recipient Name", "Recipient UEI", "Award Amount", 28 "Awarding Agency", "Awarding Sub Agency", 29 "Period of Performance Start Date", "Period of Performance Current End Date", 30 "NAICS", "Description", "recipient_id", "generated_internal_id", 31} 32 33// USASpending is the free Provider implementation over the built in net/http. 34type USASpending struct { 35 base string 36 http *http.Client 37} 38 39// NewUSASpending builds the provider. An empty base uses the public endpoint. 40func NewUSASpending(base string) *USASpending { 41 if strings.TrimSpace(base) == "" { 42 base = USASpendingBase 43 } 44 return &USASpending{base: base, http: &http.Client{Timeout: 60 * time.Second}} 45} 46 47func (u *USASpending) Name() string { return "USASpending" } 48 49// FetchAwards posts one page of spending_by_award and normalizes the results. 50func (u *USASpending) FetchAwards(ctx context.Context, q AwardQuery) ([]Award, bool, error) { 51 limit := q.Limit 52 if limit < 1 { 53 limit = 100 54 } 55 if limit > 100 { 56 limit = 100 // USASpending caps page size at 100 57 } 58 page := q.Page 59 if page < 1 { 60 page = 1 61 } 62 63 filters := map[string]any{} 64 if len(q.AwardTypeCodes) > 0 { 65 filters["award_type_codes"] = q.AwardTypeCodes 66 } 67 if len(q.TimePeriods) > 0 { 68 tp := make([]map[string]string, 0, len(q.TimePeriods)) 69 for _, p := range q.TimePeriods { 70 tp = append(tp, map[string]string{"start_date": p.Start, "end_date": p.End}) 71 } 72 filters["time_period"] = tp 73 } 74 if len(q.NAICS) > 0 { 75 filters["naics_codes"] = q.NAICS 76 } 77 if len(q.States) > 0 { 78 locs := make([]map[string]string, 0, len(q.States)) 79 for _, s := range q.States { 80 locs = append(locs, map[string]string{"country": "USA", "state": s}) 81 } 82 filters["place_of_performance_locations"] = locs 83 } 84 if q.AwardingToptier != "" { 85 filters["agencies"] = []map[string]string{ 86 {"type": "awarding", "tier": "toptier", "name": q.AwardingToptier}, 87 } 88 } 89 if len(q.RecipientSearch) > 0 { 90 filters["recipient_search_text"] = q.RecipientSearch 91 } 92 93 body := map[string]any{ 94 "filters": filters, 95 "fields": awardFields, 96 "sort": "Award Amount", 97 "order": "desc", 98 "limit": limit, 99 "page": page, 100 } 101 payload, err := json.Marshal(body) 102 if err != nil { 103 return nil, false, fmt.Errorf("marketintel: marshal request: %w", err) 104 } 105 106 req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.base, bytes.NewReader(payload)) 107 if err != nil { 108 return nil, false, errors.New("marketintel: failed to build request") 109 } 110 req.Header.Set("Content-Type", "application/json") 111 req.Header.Set("Accept", "application/json") 112 113 resp, err := u.http.Do(req) 114 if err != nil { 115 return nil, false, errors.New("marketintel: request failed") 116 } 117 defer func() { _ = resp.Body.Close() }() 118 119 if resp.StatusCode == http.StatusTooManyRequests { 120 _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, usaThrottleCap)) 121 return nil, false, ErrThrottled 122 } 123 if resp.StatusCode != http.StatusOK { 124 snippet, _ := io.ReadAll(io.LimitReader(resp.Body, usaThrottleCap)) 125 return nil, false, fmt.Errorf("marketintel: status %d: %s", resp.StatusCode, strings.TrimSpace(string(snippet))) 126 } 127 128 var page1 awardPage 129 if err := json.NewDecoder(resp.Body).Decode(&page1); err != nil { 130 return nil, false, fmt.Errorf("marketintel: decode response: %w", err) 131 } 132 133 awards := make([]Award, 0, len(page1.Results)) 134 for _, raw := range page1.Results { 135 // IsGalapagos must track whether the recipient filter was ACTUALLY applied (line 89 uses len > 0), 136 // not merely whether the slice is non nil; an empty non nil slice is an unfiltered market pull. 137 a, ok := normalizeAward(raw, len(q.RecipientSearch) > 0) 138 if !ok { 139 continue 140 } 141 awards = append(awards, a) 142 } 143 return awards, page1.PageMetadata.HasNext, nil 144} 145 146type awardPage struct { 147 Results []json.RawMessage `json:"results"` 148 PageMetadata struct { 149 HasNext bool `json:"hasNext"` 150 Page int `json:"page"` 151 } `json:"page_metadata"` 152} 153 154type usaAward struct { 155 GeneratedInternalID string `json:"generated_internal_id"` 156 AwardID string `json:"Award ID"` 157 RecipientName string `json:"Recipient Name"` 158 RecipientUEI string `json:"Recipient UEI"` 159 RecipientID string `json:"recipient_id"` 160 AwardAmount json.Number `json:"Award Amount"` 161 AwardingAgency string `json:"Awarding Agency"` 162 AwardingSubAgency string `json:"Awarding Sub Agency"` 163 POPStart *string `json:"Period of Performance Start Date"` 164 POPEnd *string `json:"Period of Performance Current End Date"` 165 NAICS struct { 166 Code string `json:"code"` 167 Description string `json:"description"` 168 } `json:"NAICS"` 169 Description string `json:"Description"` 170} 171 172// normalizeAward maps one raw result. isGalapagos marks own past performance pulled by recipient. 173func normalizeAward(raw json.RawMessage, isGalapagos bool) (Award, bool) { 174 var u usaAward 175 if err := json.Unmarshal(raw, &u); err != nil { 176 return Award{}, false 177 } 178 id := strings.TrimSpace(u.GeneratedInternalID) 179 if id == "" { 180 return Award{}, false 181 } 182 a := Award{ 183 GeneratedInternalID: id, 184 AwardID: strings.TrimSpace(u.AwardID), 185 RecipientName: strings.TrimSpace(u.RecipientName), 186 RecipientUEI: strings.TrimSpace(u.RecipientUEI), 187 RecipientID: strings.TrimSpace(u.RecipientID), 188 AwardAmount: parseAmount(u.AwardAmount), 189 AwardingAgency: strings.TrimSpace(u.AwardingAgency), 190 AwardingSubAgency: strings.TrimSpace(u.AwardingSubAgency), 191 POPStart: parseAwardDate(u.POPStart), 192 POPEnd: parseAwardDate(u.POPEnd), 193 NAICSCode: strings.TrimSpace(u.NAICS.Code), 194 NAICSDescription: strings.TrimSpace(u.NAICS.Description), 195 Description: strings.TrimSpace(u.Description), 196 IsGalapagos: isGalapagos, 197 Raw: raw, 198 } 199 return a, true 200} 201 202func parseAmount(n json.Number) *decimal.Decimal { 203 s := strings.TrimSpace(n.String()) 204 if s == "" { 205 return nil 206 } 207 d, err := decimal.NewFromString(s) 208 if err != nil { 209 return nil 210 } 211 return &d 212} 213 214func parseAwardDate(s *string) *time.Time { 215 if s == nil || strings.TrimSpace(*s) == "" { 216 return nil 217 } 218 t, err := time.Parse("2006-01-02", strings.TrimSpace(*s)) 219 if err != nil { 220 return nil 221 } 222 return &t 223} 224