Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / internal/modules/opportunities/extract.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | internal/modules/opportunities/extract.go |
| Original SHA-256 | 73c5d906bc46f74e8e60b02598ac40199bda889277f14b289ad1a63d2f4cafaf |
| Git blob SHA | 960182e2dbc66efa91b4eb966133d6b5fc9005ec |
| Reading-copy SHA-256 | 73c5d906bc46f74e8e60b02598ac40199bda889277f14b289ad1a63d2f4cafaf |
Recovered source transcript
Download complete source text ↓209 lines · 8.6 KB
package opportunities
import (
"crypto/sha256"
"encoding/hex"
"regexp"
"strconv"
"strings"
)
// Deterministic extraction, directive Section 5.1. This runs BEFORE any model call, and that ordering is the
// design, not an optimization: what a solicitation states plainly should be found by parsing it, so the model
// is left to NORMALIZE language rather than to extract facts. Anything found here is reproducible, explainable,
// and free, and it needs no grounding check because it is a literal span of the source.
//
// Everything in this file is pure. No database, no I/O, no model. Same input, same output, forever.
// Chunk is one extracted span with the provenance that makes it citable.
type Chunk struct {
SectionLabel string // for example "C.4", or a heading line
PageNumber int // 1 based when the source paginates, 0 when it does not
Text string // the literal span, never a paraphrase
ContentHash string // sha256 of the normalized text, the citation fingerprint
Method string // how it was found, for example "numbered_clause" or "paragraph"
Confidence float64 // deterministic extraction is certain about WHAT it copied, so this is 1.0
}
// Signals are the structured facts a solicitation usually states outright. Each is either found literally or
// left at its zero value; nothing here is inferred, so a zero means "not stated", never "probably none".
type Signals struct {
ClearanceLevel string
Polygraph string
TravelPercent int
Quantity int
MinimumYears int
Money []string
Dates []string
Certifications []string
LaborCategory string
RemoteAllowed bool
}
var (
// A numbered clause label such as C.4, 1.2.3, or Section 3.
clauseRe = regexp.MustCompile(`(?m)^\s*((?:[A-Z]\.\d+(?:\.\d+)*)|(?:\d+\.\d+(?:\.\d+)*)|(?:SECTION\s+[A-Z0-9.]+))[\s.:)-]+`)
headingRe = regexp.MustCompile(`(?m)^\s*([A-Z][A-Z \t,'&/]{6,80})\s*$`)
moneyRe = regexp.MustCompile(`\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?|\$\s?\d+(?:\.\d+)?\s?(?:million|billion|M|B)\b`)
// PERCENT IS WRITTEN BOTH WAYS and a solicitation picks either, often in the same document. The first
// version matched only the symbol, so "Travel is estimated at 15 percent" scored zero travel, and an
// amendment moving it to 40 percent produced NO detected change. A staffing tool that misses a travel
// change misprices the role.
percentRe = regexp.MustCompile(`(?i)travel(?:\s+\w+){0,4}?\s+(?:up\s+to\s+)?(\d{1,3})\s?(?:%|percent)|(\d{1,3})\s?(?:%|percent)\s+travel`)
// QUANTITY sits in front of a LABOR CATEGORY, and a labor category is usually several words:
// "four Cyber Systems Operations specialists". The first version allowed only a couple of fixed
// qualifiers between the number and the noun, so that phrase matched NOTHING and an amendment raising
// the headcount from four to six produced no detected change. For a cleared staffing product that is
// the single most important number on the page, so the gap between the count and the noun is widened to
// the length of a realistic labor category and no further.
qtyRe = regexp.MustCompile(`(?i)\b(?:(\d{1,3})|one|two|three|four|five|six|seven|eight|nine|ten)\s*(?:\(\s*\d{1,3}\s*\)\s*)?(?:full\s+time\s+)?(?:[A-Za-z][A-Za-z/+.-]*\s+){0,5}?(?:FTEs?|personnel|positions?|engineers?|analysts?|developers?|architects?|scientists?|specialists?)\b`)
yearsRe = regexp.MustCompile(`(?i)\b(?:minimum\s+of\s+)?(\d{1,2})\+?\s*years?\b`)
dateRe = regexp.MustCompile(`(?i)\b\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{4}\b|\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b`)
// Certifications appear both as symbols and spelled out ("Security+", "Security Plus", "Sec+"), and a
// solicitation writer picks either, so both spellings are matched rather than assuming the symbol form.
certRe = regexp.MustCompile(`(?i)(Security\s?\+|Security Plus|Sec\+|Network\s?\+|Network Plus|CISSP|CISM|CISA|CEH|PMP|ITIL|CCNA|CCNP|AWS Certified[\w \-]*|Azure [\w \-]*Certified|CompTIA [\w+]+|Splunk[\w \-]*|Cloud Practitioner)`)
remoteRe = regexp.MustCompile(`(?i)\b(remote|telework|work from home|hybrid)\b`)
pageBreak = regexp.MustCompile(`\f|(?im)^\s*page\s+(\d{1,4})\s+of\s+\d{1,4}\s*$`)
numberWord = map[string]int{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10}
)
// ExtractDeterministic splits source text into citable spans. It prefers numbered clauses, which is how a
// statement of work is actually organized, and falls back to paragraphs so an unstructured document still
// yields evidence rather than nothing.
func ExtractDeterministic(text string) []Chunk {
if strings.TrimSpace(text) == "" {
return nil
}
page := 1
if m := pageBreak.FindStringSubmatch(text); len(m) > 1 && m[1] != "" {
if n, err := strconv.Atoi(m[1]); err == nil {
page = n
}
}
locs := clauseRe.FindAllStringSubmatchIndex(text, -1)
out := make([]Chunk, 0, len(locs)+8)
if len(locs) > 0 {
for i, loc := range locs {
label := strings.TrimSpace(text[loc[2]:loc[3]])
start := loc[1]
end := len(text)
if i+1 < len(locs) {
end = locs[i+1][0]
}
body := strings.TrimSpace(text[start:end])
if body == "" {
continue
}
out = append(out, newChunk(label, page, body, "numbered_clause"))
}
return out
}
// No numbered clauses: fall back to headings and paragraphs so the pass still produces citable evidence.
label := ""
for _, para := range strings.Split(text, "\n\n") {
p := strings.TrimSpace(para)
if p == "" {
continue
}
if m := headingRe.FindStringSubmatch(p); len(m) > 1 && len(p) == len(strings.TrimSpace(m[1])) {
label = strings.TrimSpace(m[1])
continue
}
out = append(out, newChunk(label, page, p, "paragraph"))
}
return out
}
func newChunk(label string, page int, body, method string) Chunk {
sum := sha256.Sum256([]byte(normalizeWS(body)))
return Chunk{
SectionLabel: label,
PageNumber: page,
Text: body,
ContentHash: hex.EncodeToString(sum[:]),
Method: method,
// Deterministic extraction copies a literal span, so it is certain about what it copied. The
// confidence of an INTERPRETATION of that span is a separate question the reviewer answers.
Confidence: 1.0,
}
}
// ExtractSignals pulls the structured facts a solicitation states outright. Every field is literal: a zero
// value means the text did not say, never that the answer is none.
func ExtractSignals(text string) Signals {
var s Signals
low := strings.ToLower(text)
// Clearance, most specific first so "top secret sci" is not read as "secret".
switch {
case strings.Contains(low, "ts/sci"), strings.Contains(low, "ts sci"), strings.Contains(low, "top secret/sci"),
strings.Contains(low, "sci eligibility"), strings.Contains(low, "top secret sci"):
s.ClearanceLevel = "ts_sci"
case strings.Contains(low, "top secret"):
s.ClearanceLevel = "top_secret"
case strings.Contains(low, "secret"):
s.ClearanceLevel = "secret"
case strings.Contains(low, "confidential"):
s.ClearanceLevel = "confidential"
case strings.Contains(low, "public trust"), strings.Contains(low, "no clearance"):
s.ClearanceLevel = "none"
}
switch {
case strings.Contains(low, "full scope poly"), strings.Contains(low, "full-scope poly"), strings.Contains(low, "fspoly"):
s.Polygraph = "full_scope"
case strings.Contains(low, "ci poly"), strings.Contains(low, "counterintelligence poly"):
s.Polygraph = "ci"
}
if m := percentRe.FindStringSubmatch(text); m != nil {
for _, g := range m[1:] {
if g != "" {
if n, err := strconv.Atoi(g); err == nil && n >= 0 && n <= 100 {
s.TravelPercent = n
break
}
}
}
}
if m := qtyRe.FindStringSubmatch(text); m != nil {
if m[1] != "" {
if n, err := strconv.Atoi(m[1]); err == nil {
s.Quantity = n
}
} else {
word := strings.ToLower(strings.Fields(m[0])[0])
s.Quantity = numberWord[word]
}
}
if m := yearsRe.FindStringSubmatch(text); m != nil {
if n, err := strconv.Atoi(m[1]); err == nil && n < 60 {
s.MinimumYears = n
}
}
s.Money = dedupeStrings(moneyRe.FindAllString(text, -1))
s.Dates = dedupeStrings(dateRe.FindAllString(text, -1))
s.Certifications = dedupeStrings(certRe.FindAllString(text, -1))
s.RemoteAllowed = remoteRe.MatchString(text)
return s
}
func dedupeStrings(in []string) []string {
if len(in) == 0 {
return nil
}
seen := make(map[string]bool, len(in))
out := make([]string, 0, len(in))
for _, v := range in {
v = strings.TrimSpace(v)
key := strings.ToLower(v)
if v == "" || seen[key] {
continue
}
seen[key] = true
out = append(out, v)
}
return out
}