Janady13/missionmatch / internal/modules/federalsignal/fetch.go

1// Package federalsignal mines public federal solicitation attachments for the deterministic mission and 2// labor classifier. This file is the server side SAM attachment fetch client: it downloads a notice 3// description and its resource links, runs each body through the pure Go docextract extractor, and 4// returns the concatenated plain text. The SAM api_key is held here and only ever appended as a query 5// parameter to a SAM host. It is NEVER logged, NEVER placed in an error string, and NEVER returned in 6// the extracted text. The client is best effort: one bad attachment is skipped, never fatal. 7package federalsignal 8 9import ( 10 "context" 11 "crypto/sha256" 12 "encoding/hex" 13 "errors" 14 "fmt" 15 "io" 16 "log" 17 "net/http" 18 "net/url" 19 "strings" 20 "time" 21 22 "github.com/missionworkforce/missionmatch/internal/platform/docextract" 23 "github.com/missionworkforce/missionmatch/internal/platform/storage" 24) 25 26const ( 27 defaultTimeout = 45 * time.Second 28 defaultMaxRetries = 2 29 maxBodyBytes = 8 << 20 // cap one download so a huge attachment cannot exhaust memory 30 maxTotalBytes = 4 << 20 // cap the concatenated text across all attachments for one record 31 separator = "\n\n----\n\n" 32 backoffStep = 200 * time.Millisecond 33) 34 35// Fetcher downloads SAM solicitation attachments and extracts their text. Build one with New. It is safe 36// for concurrent use, the http.Client and the option fields are read only after construction. 37type Fetcher struct { 38 key string // SAM api_key, server side only, never logged or returned 39 http *http.Client // bounded by Timeout 40 maxRetry int // additional attempts on a transient failure 41 samHosts map[string]struct{} // hosts the key may be appended to, lowercased 42 store storage.Storer // optional, nil skips retention 43 storePref string // key prefix for retained raw bytes 44} 45 46// Option configures a Fetcher. 47type Option func(*Fetcher) 48 49// WithTimeout sets the per download timeout. 50func WithTimeout(d time.Duration) Option { 51 return func(f *Fetcher) { 52 if d > 0 { 53 f.http.Timeout = d 54 } 55 } 56} 57 58// WithMaxRetries sets the number of additional attempts on a transient download failure. 59func WithMaxRetries(n int) Option { 60 return func(f *Fetcher) { 61 if n >= 0 { 62 f.maxRetry = n 63 } 64 } 65} 66 67// WithSAMHosts registers the hosts the api_key may be appended to. The key is sent ONLY to these hosts, 68// so a foreign resource link in a notice can never receive the secret. Hosts are matched case 69// insensitively on the URL host (with any port). Production registers api.sam.gov and sam.gov; the test 70// registers the httptest host. 71func WithSAMHosts(hosts ...string) Option { 72 return func(f *Fetcher) { 73 for _, h := range hosts { 74 h = strings.ToLower(strings.TrimSpace(h)) 75 if h != "" { 76 f.samHosts[h] = struct{}{} 77 } 78 } 79 } 80} 81 82// WithStore wires an optional object store for raw attachment retention under the given key prefix. A 83// nil store (or not calling this option) skips retention entirely. 84func WithStore(s storage.Storer, prefix string) Option { 85 return func(f *Fetcher) { 86 f.store = s 87 f.storePref = strings.TrimSpace(prefix) 88 } 89} 90 91// New builds a Fetcher holding the SAM api_key. Defaults: a 45 second timeout, two retries, the two 92// production SAM hosts, and no retention. Options override these. 93func New(samKey string, opts ...Option) *Fetcher { 94 f := &Fetcher{ 95 key: samKey, 96 http: &http.Client{Timeout: defaultTimeout}, 97 maxRetry: defaultMaxRetries, 98 samHosts: map[string]struct{}{ 99 "api.sam.gov": {}, 100 "sam.gov": {}, 101 }, 102 } 103 for _, o := range opts { 104 o(f) 105 } 106 return f 107} 108 109// FetchSolicitationText downloads the notice description URL and each resource link, extracts the plain 110// text of each, and returns the concatenated text capped at maxTotalBytes. It is best effort: a per 111// attachment failure is counted and logged (never with the URL or the key) and skipped, it never fails 112// the whole record. The api_key is [credential removed] only to a registered SAM host and never appears in the 113// returned string. 114func (f *Fetcher) FetchSolicitationText(ctx context.Context, descURL string, resourceLinks []string) (string, error) { 115 urls := make([]string, 0, 1+len(resourceLinks)) 116 if strings.TrimSpace(descURL) != "" { 117 urls = append(urls, descURL) 118 } 119 for _, l := range resourceLinks { 120 if strings.TrimSpace(l) != "" { 121 urls = append(urls, l) 122 } 123 } 124 if len(urls) == 0 { 125 return "", nil 126 } 127 128 var sb strings.Builder 129 failures := 0 130 for _, raw := range urls { 131 if err := ctx.Err(); err != nil { 132 return sb.String(), err // a cancelled context is the caller's signal, not a per attachment skip 133 } 134 text, err := f.fetchOne(ctx, raw) 135 if err != nil { 136 // Count the failure without echoing the URL (it carries the api_key once appended) or the cause 137 // detail that could include it. Best effort: continue to the next attachment. 138 failures++ 139 continue 140 } 141 if text == "" { 142 continue 143 } 144 if sb.Len() > 0 { 145 sb.WriteString(separator) 146 } 147 remaining := maxTotalBytes - sb.Len() 148 if remaining <= 0 { 149 break 150 } 151 if len(text) > remaining { 152 text = truncateOnRune(text, remaining) 153 } 154 sb.WriteString(text) 155 if sb.Len() >= maxTotalBytes { 156 break 157 } 158 } 159 if failures > 0 { 160 log.Printf("federalsignal: skipped %d unreadable attachment(s) of %d on one record", failures, len(urls)) 161 } 162 return sb.String(), nil 163} 164 165// fetchOne downloads a single URL (appending the api_key only for a SAM host), optionally retains the 166// raw bytes, and returns the extracted text. It is the only place the key touches a URL, and it returns 167// errors that never embed the key bearing URL. 168func (f *Fetcher) fetchOne(ctx context.Context, raw string) (string, error) { 169 parsed, err := url.Parse(raw) 170 if err != nil { 171 return "", errors.New("federalsignal: invalid attachment url") 172 } 173 reqURL := raw 174 if f.isSAMHost(parsed.Host) { 175 // Append the key to the existing query. Build it from the parsed URL so the key is never string 176 // concatenated into a malformed URL, and so it is added once even if the URL already has a query. 177 q := parsed.Query() 178 q.Set("api_key", f.key) 179 parsed.RawQuery = q.Encode() 180 reqURL = parsed.String() 181 } 182 183 body, err := f.download(ctx, reqURL) 184 if err != nil { 185 return "", err // already key safe, see download 186 } 187 188 if f.store != nil { 189 // Retain under a deterministic key derived from the ORIGINAL url (without the key), so retention is 190 // idempotent and the key never lands in the object key. A storage failure is not fatal to mining. 191 if perr := f.store.Put(ctx, rawKey(f.storePref, raw), body); perr != nil { 192 log.Printf("federalsignal: retain attachment failed: %v", perr) 193 } 194 } 195 196 // Name the extractor's dispatch off the URL path so the extension drives the format choice. 197 return docextract.ExtractText(ctx, parsed.Path, body) 198} 199 200// download GETs url with bounded retries on a transient failure and returns the body bytes capped at 201// maxBodyBytes. Every error string is key safe: a *url.Error embeds the full request URL (which carries 202// the api_key), so it is unwrapped to the transport cause which carries host:port at most. 203func (f *Fetcher) download(ctx context.Context, target string) ([]byte, error) { 204 var lastErr error 205 for attempt := 0; attempt <= f.maxRetry; attempt++ { 206 if attempt > 0 { 207 select { 208 case <-ctx.Done(): 209 return nil, ctx.Err() 210 case <-time.After(time.Duration(attempt) * backoffStep): 211 } 212 } 213 body, err := f.attempt(ctx, target) 214 if err == nil { 215 return body, nil 216 } 217 lastErr = err 218 } 219 return nil, lastErr 220} 221 222func (f *Fetcher) attempt(ctx context.Context, target string) ([]byte, error) { 223 req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) 224 if err != nil { 225 // A build error would echo the full URL incl. api_key, keep it generic. 226 return nil, errors.New("federalsignal: failed to build request") 227 } 228 resp, err := f.http.Do(req) 229 if err != nil { 230 // *url.Error.Error() embeds the full request URL incl. ?api_key=... NEVER wrap it whole: ue.Err is 231 // the transport cause and carries host:port at most, never the query string. 232 var ue *url.Error 233 if errors.As(err, &ue) { 234 return nil, fmt.Errorf("federalsignal: %s request failed: %v", ue.Op, ue.Err) 235 } 236 return nil, errors.New("federalsignal: request failed") 237 } 238 defer func() { _ = resp.Body.Close() }() 239 240 if resp.StatusCode != http.StatusOK { 241 // Drain a small bounded snippet so the connection can be reused, but do not include it: a SAM error 242 // body can echo the request URL with the key. 243 _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024)) 244 return nil, fmt.Errorf("federalsignal: status %d", resp.StatusCode) 245 } 246 body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) 247 if err != nil { 248 return nil, errors.New("federalsignal: read body failed") 249 } 250 return body, nil 251} 252 253// isSAMHost reports whether host (which may include a port) is a registered SAM host. 254func (f *Fetcher) isSAMHost(host string) bool { 255 _, ok := f.samHosts[strings.ToLower(host)] 256 return ok 257} 258 259// rawKey is the deterministic object key for a retained attachment: the prefix, then the lowercase hex 260// sha256 of the original URL, with a .bin suffix. The api_key is [credential removed] part of it because the original 261// URL (pre key) is hashed. Two runs of the same URL collide on one object. 262func rawKey(prefix, originalURL string) string { 263 sum := sha256.Sum256([]byte(originalURL)) 264 h := hex.EncodeToString(sum[:]) 265 p := strings.Trim(prefix, "/") 266 if p == "" { 267 p = "federalsignal" 268 } 269 return p + "/attachments/" + h + ".bin" 270} 271 272// truncateOnRune cuts s to at most n bytes on a UTF-8 rune boundary so the cap never splits a rune. 273func truncateOnRune(s string, n int) string { 274 if n >= len(s) { 275 return s 276 } 277 cut := n 278 for cut > 0 && s[cut]&0xC0 == 0x80 { 279 cut-- 280 } 281 return s[:cut] 282} 283