Janady13/missionmatch / internal/platform/forecast/forecast.go

1// Package forecast normalizes agency contract forecasts (planned, not yet posted opportunities) from 2// CSV exports into a flat record. Most agencies, and the governmentwide GSA Forecast of Contracting 3// Opportunities, publish forecasts as a CSV with their own column names and no clean API, so the parser 4// is mapping driven: each source supplies a Mapping from the normalized fields to its candidate CSV 5// header names, and the same parser handles every source without new code. Forecasts are planning data 6// subject to change, never a posted solicitation. Money and dates are parsed only from a real single 7// value, never fabricated from a range or a category. 8package forecast 9 10import ( 11 "encoding/csv" 12 "errors" 13 "fmt" 14 "io" 15 "regexp" 16 "strconv" 17 "strings" 18 "time" 19 20 "github.com/shopspring/decimal" 21) 22 23// Record is one normalized forecast row. Optional money and dates are pointers (nil = absent). Raw keeps 24// the original row keyed by header for provenance. 25type Record struct { 26 SourceID string 27 Title string 28 Agency string 29 Organization string 30 NAICS string 31 PlaceState string 32 SetAside string // acquisition strategy or set aside status 33 ContractType string 34 AwardStatus string 35 EstimatedValue *decimal.Decimal 36 EstimatedAward *time.Time 37 Raw map[string]string 38} 39 40// Field is a normalized forecast field the parser maps a source column onto. 41type Field string 42 43const ( 44 FieldSourceID Field = "source_id" 45 FieldTitle Field = "title" 46 FieldAgency Field = "agency" 47 FieldOrganization Field = "organization" 48 FieldNAICS Field = "naics" 49 FieldPlaceState Field = "place_state" 50 FieldSetAside Field = "set_aside" 51 FieldContractType Field = "contract_type" 52 FieldAwardStatus Field = "award_status" 53 FieldEstimatedValue Field = "estimated_value" 54 FieldEstimatedAward Field = "estimated_award_date" 55) 56 57// Mapping gives, for each field, the candidate CSV header names to look for. Matching is case 58// insensitive and the first candidate that exists wins, so one source definition tolerates header drift 59// between exports. 60type Mapping map[Field][]string 61 62// Parse reads a CSV and maps each row to a Record using the mapping. The first row is the header. A row 63// with no identity (no source id and no title) is skipped, and a malformed row is skipped rather than 64// fatal, so one bad line never loses the whole export. The count of skipped rows is returned for the 65// caller to log. 66func Parse(r io.Reader, m Mapping) (records []Record, skipped int, err error) { 67 cr := csv.NewReader(r) 68 cr.FieldsPerRecord = -1 // tolerate ragged rows from a hand exported CSV 69 cr.TrimLeadingSpace = true 70 header, err := cr.Read() 71 if err != nil { 72 return nil, 0, fmt.Errorf("forecast: read header: %w", err) 73 } 74 col := resolveColumns(header, m) 75 for { 76 row, e := cr.Read() 77 if errors.Is(e, io.EOF) { 78 break 79 } 80 if e != nil { 81 skipped++ 82 continue 83 } 84 rec, ok := mapRow(row, header, col) 85 if !ok { 86 skipped++ 87 continue 88 } 89 records = append(records, rec) 90 } 91 return records, skipped, nil 92} 93 94// resolveColumns maps each field to a column index using the first matching candidate header, or minus 95// one when no candidate header is present in this export. 96func resolveColumns(header []string, m Mapping) map[Field]int { 97 pos := make(map[string]int, len(header)) 98 for i, h := range header { 99 pos[normHeader(h)] = i 100 } 101 out := make(map[Field]int, len(m)) 102 for field, candidates := range m { 103 idx := -1 104 for _, c := range candidates { 105 if i, ok := pos[normHeader(c)]; ok { 106 idx = i 107 break 108 } 109 } 110 out[field] = idx 111 } 112 return out 113} 114 115func mapRow(row, header []string, col map[Field]int) (Record, bool) { 116 get := func(f Field) string { 117 i := col[f] 118 if i < 0 || i >= len(row) { 119 return "" 120 } 121 return strings.TrimSpace(row[i]) 122 } 123 rec := Record{ 124 SourceID: get(FieldSourceID), 125 Title: get(FieldTitle), 126 Agency: get(FieldAgency), 127 Organization: get(FieldOrganization), 128 NAICS: get(FieldNAICS), 129 PlaceState: get(FieldPlaceState), 130 SetAside: get(FieldSetAside), 131 ContractType: get(FieldContractType), 132 AwardStatus: get(FieldAwardStatus), 133 EstimatedValue: parseValue(get(FieldEstimatedValue)), 134 EstimatedAward: parseDate(get(FieldEstimatedAward)), 135 } 136 if rec.SourceID == "" && rec.Title == "" { 137 return Record{}, false 138 } 139 rec.Raw = make(map[string]string, len(header)) 140 for i, h := range header { 141 if i < len(row) { 142 rec.Raw[h] = strings.TrimSpace(row[i]) 143 } 144 } 145 return rec, true 146} 147 148func normHeader(h string) string { 149 return strings.ToLower(strings.TrimSpace(h)) 150} 151 152var ( 153 quarterRe = regexp.MustCompile(`Q([1-4])`) 154 ordinalQRe = regexp.MustCompile(`([1-4])(?:ST|ND|RD|TH)?\s+QUARTER`) 155 yearRe = regexp.MustCompile(`(20\d\d)`) 156 // numberRe is anchored to the whole trimmed field so a value is only read when the entire field is a 157 // single dollar amount. Without the anchors, descriptive text like "Approx. 500K" would still yield 158 // one match and parseValue would read a figure the forecast never stated. 159 numberRe = regexp.MustCompile(`^\$?\s*([0-9][0-9,]*(?:\.[0-9]+)?)\s*(K|M|B|THOUSAND|MILLION|BILLION)?$`) 160) 161 162// parseDate normalizes the estimated award date. It accepts an explicit date in several layouts, a month 163// and year, and a federal fiscal quarter (mapped to the quarter's first month, since a forecast states a 164// quarter not a day). TBD and similar placeholders, and anything unrecognized, return nil rather than a 165// fabricated date. 166func parseDate(s string) *time.Time { 167 s = strings.TrimSpace(s) 168 if s == "" { 169 return nil 170 } 171 up := strings.ToUpper(s) 172 switch up { 173 case "TBD", "N/A", "NA", "UNKNOWN", "PENDING": 174 return nil 175 } 176 layouts := []string{ 177 "2006-01-02", "01/02/2006", "1/2/2006", time.RFC3339, 178 "January 2, 2006", "Jan 2, 2006", "2006-01", "01/2006", "1/2006", 179 "January 2006", "Jan 2006", "Jan-2006", 180 } 181 for _, l := range layouts { 182 if t, err := time.Parse(l, s); err == nil { 183 d := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) 184 return &d 185 } 186 } 187 return parseFiscalQuarter(up) 188} 189 190// parseFiscalQuarter maps a stated federal fiscal quarter to the quarter's first calendar month, the 191// honest sortable approximation of a quarter (the original text is retained in the record's Raw). The 192// federal fiscal year starts in October, so FY Q1 is October of the prior calendar year. 193func parseFiscalQuarter(up string) *time.Time { 194 q := 0 195 if mm := quarterRe.FindStringSubmatch(up); mm != nil { 196 q, _ = strconv.Atoi(mm[1]) 197 } else if mm := ordinalQRe.FindStringSubmatch(up); mm != nil { 198 q, _ = strconv.Atoi(mm[1]) 199 } 200 ym := yearRe.FindStringSubmatch(up) 201 if q == 0 || ym == nil { 202 return nil 203 } 204 fy, _ := strconv.Atoi(ym[1]) 205 var month int 206 var year int 207 switch q { 208 case 1: 209 month, year = 10, fy-1 210 case 2: 211 month, year = 1, fy 212 case 3: 213 month, year = 4, fy 214 case 4: 215 month, year = 7, fy 216 } 217 d := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) 218 return &d 219} 220 221// parseValue parses the estimated value only when the field is a single clean dollar amount (optionally 222// with a K, M, or B suffix). A range, a category, or two numbers yields nil, because representing a range 223// as one point would fabricate a figure the forecast does not state. The raw text stays in the record. 224func parseValue(s string) *decimal.Decimal { 225 up := strings.ToUpper(strings.TrimSpace(s)) 226 if up == "" { 227 return nil 228 } 229 matches := numberRe.FindAllStringSubmatch(up, -1) 230 if len(matches) != 1 { 231 return nil 232 } 233 digits := strings.ReplaceAll(matches[0][1], ",", "") 234 d, err := decimal.NewFromString(digits) 235 if err != nil { 236 return nil 237 } 238 switch matches[0][2] { 239 case "K", "THOUSAND": 240 d = d.Mul(decimal.NewFromInt(1000)) 241 case "M", "MILLION": 242 d = d.Mul(decimal.NewFromInt(1000000)) 243 case "B", "BILLION": 244 d = d.Mul(decimal.NewFromInt(1000000000)) 245 } 246 return &d 247} 248