Janady13/missionmatch / internal/platform/bls/occupations.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/platform/bls/occupations.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/platform/bls/occupations.go
Original SHA-256	13a12cc4e801c5aa77b4a684595f67c58d365ee1b01fcf05e3e07c9a14adf538
Git blob SHA	5aa28a44a94811397d7c9dbf1fc802660805921c
Reading-copy SHA-256	13a12cc4e801c5aa77b4a684595f67c58d365ee1b01fcf05e3e07c9a14adf538

Original source ↗

Every recorded occurrence

These 11 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/platform/bls/occupations.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/platform/bls/occupations.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓109 lines · 4.7 KB

package bls

// The OEWS occupation list drives the national wage expansion (M5 Phase 1). The BLS flat file at
// download.bls.gov carries every OEWS occupation with a display level; display level 3 marks a DETAILED
// SOC occupation (there are roughly 800). We keep only those, capturing the six digit SOC and the OEWS
// occupation title, then build the two national series (annual mean 04, annual median 13) per SOC.
//
// The download requires a descriptive User Agent that names a contact email; BLS returns 403 to a default
// or empty User Agent. The parser here is pure (no network, no database) so it is unit testable: it takes
// the already fetched flat file bytes and returns the detailed occupations.

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
)

// OccupationListURL is the BLS OEWS occupation flat file: tab delimited, one occupation per row.
const OccupationListURL = "https://download.bls.gov/pub/time.series/oe/oe.occupation"

// detailedDisplayLevel is the display level that marks a DETAILED SOC occupation in the OEWS flat file.
// Aggregate roll up levels (0, 1, 2) are NOT detailed occupations and are skipped.
const detailedDisplayLevel = "3"

// DetailedOccupation is one detailed OEWS occupation: the six digit SOC code and the OEWS occupation
// title. It carries no wage; the sync builds the national series from the SOC and pulls the wages.
type DetailedOccupation struct {
	SOC   string
	Title string
}

// ParseDetailedOccupations parses the tab delimited oe.occupation flat file and returns only the DETAILED
// SOC occupations (display_level = 3), capturing the occupation_code (SOC) and occupation_name (title).
// The columns are occupation_code, occupation_name, occupation_description, display_level, selectable,
// sort_sequence. The header row (a row whose first column is the literal occupation_code) is skipped, as
// is any short or blank row. The result is deterministic: the occupations come back in the file's order,
// which the caller may re sort. It is pure: no network and no database.
func ParseDetailedOccupations(r io.Reader) ([]DetailedOccupation, error) {
	var out []DetailedOccupation
	sc := bufio.NewScanner(r)
	// A flat file row can be long (the description column); raise the scanner buffer well past the default.
	sc.Buffer(make([]byte, 0, 1<<20), 1<<20)
	for sc.Scan() {
		line := sc.Text()
		if strings.TrimSpace(line) == "" {
			continue
		}
		cols := strings.Split(line, "\t")
		if len(cols) < 4 {
			continue // a malformed or truncated row cannot carry a display level; skip it
		}
		code := strings.TrimSpace(cols[0])
		title := strings.TrimSpace(cols[1])
		level := strings.TrimSpace(cols[3])
		if code == "" || code == "occupation_code" {
			continue // the header row or a blank code
		}
		if level != detailedDisplayLevel {
			continue // an aggregate roll up level, not a detailed SOC
		}
		out = append(out, DetailedOccupation{SOC: code, Title: title})
	}
	if err := sc.Err(); err != nil {
		return nil, fmt.Errorf("bls: scan occupation list: %w", err)
	}
	return out, nil
}

// FetchDetailedOccupations downloads the oe.occupation flat file with the required descriptive User Agent
// and returns the detailed SOC occupations. userAgent MUST be a descriptive string that names a contact
// email (for example "missionMATCH wage sync (joseph.w.anady@gmail.com)"); BLS returns 403 to a default
// or empty User Agent, so an empty userAgent is rejected before the request is made. A non 200 response
// returns a clear error carrying the status and a short body snippet, never the whole page.
func FetchDetailedOccupations(ctx context.Context, httpClient *http.Client, url, userAgent string) ([]DetailedOccupation, error) {
	if strings.TrimSpace(userAgent) == "" {
		return nil, fmt.Errorf("bls: a descriptive User Agent naming a contact email is required for the BLS download")
	}
	if strings.TrimSpace(url) == "" {
		url = OccupationListURL
	}
	if httpClient == nil {
		httpClient = &http.Client{Timeout: 60 * time.Second}
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, fmt.Errorf("bls: build occupation list request: %w", err)
	}
	req.Header.Set("User-Agent", userAgent)
	resp, err := httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("bls: occupation list request failed: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode != http.StatusOK {
		snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return nil, fmt.Errorf("bls: occupation list status %d: %s", resp.StatusCode, strings.TrimSpace(string(snippet)))
	}
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("bls: read occupation list: %w", err)
	}
	return ParseDetailedOccupations(bytes.NewReader(body))
}