Janady13/missionmatch / internal/platform/extraction/deterministic.go

1package extraction 2 3import ( 4 "context" 5 "regexp" 6 "sort" 7 "strconv" 8 "strings" 9) 10 11// deterministic is the offline rules plus gazetteer extractor. It is fully deterministic and needs no 12// model, key, or network, so it is the production default for the closed vocabulary fields and is the 13// only extractor in this build. Every value it emits is a self reported mention, never a verified fact. 14type deterministic struct{} 15 16// NewDeterministic builds the offline extractor. 17func NewDeterministic() Extractor { return deterministic{} } 18 19// ExtractResume parses a resume into proposed candidate fields. 20func (deterministic) ExtractResume(_ context.Context, text string) (RawExtraction, error) { 21 lower := strings.ToLower(text) 22 level := detectClearanceLevel(lower) 23 rx := RawExtraction{ 24 FullName: detectName(text), 25 ClearanceLevel: level, 26 ClearancePoly: detectPoly(lower), 27 ClearanceActive: level != "" && !reNegativeClearance.MatchString(lower), 28 Skills: detectFromGazetteer(lower, skillMatchers), 29 Certifications: detectFromGazetteer(lower, certMatchers), 30 Agencies: detectFromGazetteer(lower, agencyMatchers), 31 YearsExperience: detectYears(lower), 32 Location: detectLocation(text), 33 } 34 return rx, nil 35} 36 37// ExtractRequirement parses a pasted job description into proposed requirement fields. The clearance is 38// the requirement's STATED required clearance (active is not meaningful for a requirement), and the 39// skills, certifications, and agency become the required set the candidate is scored against. 40func (deterministic) ExtractRequirement(_ context.Context, text string) (RawExtraction, error) { 41 lower := strings.ToLower(text) 42 rx := RawExtraction{ 43 ClearanceLevel: detectClearanceLevel(lower), 44 ClearancePoly: detectPoly(lower), 45 Skills: detectFromGazetteer(lower, skillMatchers), 46 Certifications: detectFromGazetteer(lower, certMatchers), 47 Agencies: detectFromGazetteer(lower, agencyMatchers), 48 YearsExperience: detectYears(lower), 49 Location: detectLocation(text), 50 } 51 return rx, nil 52} 53 54// ---------- clearance ---------- 55 56var ( 57 // TS/SCI in its common written forms. "sci" alone, or any of the ts sci spellings, implies TS/SCI. 58 reTSSCI = regexp.MustCompile(`(?i)\b(ts\s*/?\s*sci|tssci|sci)\b`) 59 reTopSecret = regexp.MustCompile(`(?i)\btop\s+secret\b`) 60 reSecret = regexp.MustCompile(`(?i)\bsecret\b`) 61 reConfid = regexp.MustCompile(`(?i)\bconfidential\b`) 62 reFullScope = regexp.MustCompile(`(?i)\b(full[\s-]*scope|fs\s*poly|lifestyle\s*poly)\b`) 63 reCIPoly = regexp.MustCompile(`(?i)\b(ci\s*poly|counter\s*intelligence\s*poly|counterintelligence)\b`) 64 reNegativeCle = regexp.MustCompile(`(?i)\b(expired|inactive|lapsed|former\s+clearance|no\s+longer)\b`) 65 reYears = regexp.MustCompile(`(?i)(\d{1,2})\s*\+?\s*years?`) 66 reCityState = regexp.MustCompile(`\b([A-Z][a-zA-Z]+(?: [A-Z][a-zA-Z]+){0,2}),\s*([A-Z]{2})\b`) 67) 68 69// reNegativeClearance is aliased so ExtractResume reads clearly. 70var reNegativeClearance = reNegativeCle 71 72// detectClearanceLevel returns the highest specificity clearance code found, empty when none. Order is 73// load bearing: top secret is checked before secret (secret is a substring of top secret), and TS/SCI 74// before top secret, so a resume that says "TS/SCI" is never downgraded to Secret. 75func detectClearanceLevel(lower string) string { 76 switch { 77 case reTSSCI.MatchString(lower) && (reTopSecret.MatchString(lower) || strings.Contains(lower, "sci")): 78 return "ts_sci" 79 case reTopSecret.MatchString(lower): 80 return "top_secret" 81 case reSecret.MatchString(lower): 82 return "secret" 83 case reConfid.MatchString(lower): 84 return "confidential" 85 default: 86 return "" 87 } 88} 89 90func detectPoly(lower string) string { 91 switch { 92 case reFullScope.MatchString(lower): 93 return "full_scope" 94 case reCIPoly.MatchString(lower): 95 return "ci" 96 default: 97 return "" 98 } 99} 100 101func detectYears(lower string) int { 102 best := 0 103 for _, m := range reYears.FindAllStringSubmatch(lower, -1) { 104 n, err := strconv.Atoi(m[1]) 105 if err != nil { 106 continue 107 } 108 if n > 50 { // implausible for a single career; treat as noise 109 continue 110 } 111 if n > best { 112 best = n 113 } 114 } 115 return best 116} 117 118// detectLocation returns the first "City, ST" mention whose two letter code is a real USPS state or 119// territory, from the original (cased) text. Validating the code rejects job title and org fragments 120// like "Manager, HR" or "Director, VP" that share the City, ST shape, while keeping HI, NM, VA, DC. 121func detectLocation(text string) string { 122 for _, m := range reCityState.FindAllStringSubmatch(text, -1) { 123 if usStates[m[2]] { 124 return strings.TrimSpace(m[1]) + ", " + m[2] 125 } 126 } 127 return "" 128} 129 130// usStates is the set of USPS state, district, and territory codes used to validate a detected location. 131var usStates = map[string]bool{ 132 "AL": true, "AK": true, "AZ": true, "AR": true, "CA": true, "CO": true, "CT": true, "DE": true, 133 "FL": true, "GA": true, "HI": true, "ID": true, "IL": true, "IN": true, "IA": true, "KS": true, 134 "KY": true, "LA": true, "ME": true, "MD": true, "MA": true, "MI": true, "MN": true, "MS": true, 135 "MO": true, "MT": true, "NE": true, "NV": true, "NH": true, "NJ": true, "NM": true, "NY": true, 136 "NC": true, "ND": true, "OH": true, "OK": true, "OR": true, "PA": true, "RI": true, "SC": true, 137 "SD": true, "TN": true, "TX": true, "UT": true, "VT": true, "VA": true, "WA": true, "WV": true, 138 "WI": true, "WY": true, "DC": true, "PR": true, "GU": true, "VI": true, "AS": true, "MP": true, 139} 140 141// ---------- name (resume only, best effort, used solely for the Save Candidate prefill) ---------- 142 143var headerWords = map[string]bool{ 144 "resume": true, "curriculum": true, "vitae": true, "profile": true, "summary": true, 145 "objective": true, "experience": true, "education": true, "skills": true, "contact": true, 146} 147 148// detectName returns the first line that looks like a person name: two to four capitalized alphabetic 149// tokens, no digits, not a section header. Empty when nothing qualifies (the form then asks for it). 150func detectName(text string) string { 151 for _, line := range strings.Split(text, "\n") { 152 line = strings.TrimSpace(line) 153 if line == "" || strings.ContainsAny(line, "0123456789@") { 154 continue 155 } 156 fields := strings.Fields(line) 157 if len(fields) < 2 || len(fields) > 4 { 158 continue 159 } 160 ok := true 161 for _, f := range fields { 162 if headerWords[strings.ToLower(strings.Trim(f, ".,"))] { 163 ok = false 164 break 165 } 166 r := []rune(f) 167 if len(r) == 0 || r[0] < 'A' || r[0] > 'Z' { 168 ok = false 169 break 170 } 171 for _, c := range r { 172 if !(c >= 'A' && c <= 'Z') && !(c >= 'a' && c <= 'z') && c != '.' && c != '-' && c != '\'' { 173 ok = false 174 break 175 } 176 } 177 if !ok { 178 break 179 } 180 } 181 if ok { 182 return line 183 } 184 } 185 return "" 186} 187 188// ---------- gazetteers (canonical label plus the lowercased mentions that resolve to it) ---------- 189 190type gazEntry struct { 191 canonical string 192 mentions []string 193} 194 195// mentionMatcher matches one gazetteer mention. A wordlike mention (letters, digits, spaces) is matched 196// on a word boundary so a short token like "java" or "ato" matches "Java," or "ATO" but NOT the inside 197// of "javascript" or "potato". A mention carrying a regex special character (for example "c++", 198// "security+") falls back to a plain substring match, which is safe because those are distinctive. 199type mentionMatcher struct { 200 re *regexp.Regexp // nil means use the substring path 201 contains string 202} 203 204func (m mentionMatcher) match(lower string) bool { 205 if m.re != nil { 206 return m.re.MatchString(lower) 207 } 208 return strings.Contains(lower, m.contains) 209} 210 211// compiledGaz is a gazetteer entry with its mentions compiled to matchers, built once in init. 212type compiledGaz struct { 213 canonical string 214 matchers []mentionMatcher 215} 216 217var reWordlike = regexp.MustCompile(`^[a-z0-9][a-z0-9 ]*$`) 218 219func compileMentions(mentions []string) []mentionMatcher { 220 out := make([]mentionMatcher, 0, len(mentions)) 221 for _, m := range mentions { 222 m = strings.TrimSpace(m) 223 if m == "" { 224 continue 225 } 226 if reWordlike.MatchString(m) { 227 out = append(out, mentionMatcher{re: regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(m) + `\b`)}) 228 } else { 229 out = append(out, mentionMatcher{contains: m}) 230 } 231 } 232 return out 233} 234 235func compileGaz(g []gazEntry) []compiledGaz { 236 out := make([]compiledGaz, len(g)) 237 for i, e := range g { 238 out[i] = compiledGaz{canonical: e.canonical, matchers: compileMentions(e.mentions)} 239 } 240 return out 241} 242 243var ( 244 skillMatchers []compiledGaz 245 certMatchers []compiledGaz 246 agencyMatchers []compiledGaz 247) 248 249func init() { 250 skillMatchers = compileGaz(skillGazetteer) 251 certMatchers = compileGaz(certGazetteer) 252 agencyMatchers = compileGaz(agencyGazetteer) 253} 254 255// detectFromGazetteer returns the canonical labels whose mentions appear in the lowercased text, 256// deduped and sorted so the output is deterministic regardless of order of appearance. 257func detectFromGazetteer(lower string, g []compiledGaz) []string { 258 var out []string 259 seen := map[string]bool{} 260 for _, e := range g { 261 for _, m := range e.matchers { 262 if m.match(lower) { 263 if !seen[e.canonical] { 264 seen[e.canonical] = true 265 out = append(out, e.canonical) 266 } 267 break 268 } 269 } 270 } 271 sort.Strings(out) 272 return out 273} 274 275// skillGazetteer maps common cleared IT and cyber skills to their mentions. Canonical labels are chosen 276// to match the requirement skill ontology by name (the engine matches skills case insensitively by 277// name), so an extracted skill scores against a requirement that lists the same skill. 278var skillGazetteer = []gazEntry{ 279 {"Python", []string{"python"}}, 280 {"Java", []string{"java"}}, 281 {"JavaScript", []string{"javascript", "typescript"}}, 282 {"Go", []string{"golang"}}, 283 {"C++", []string{"c++"}}, 284 {"SQL", []string{"sql", "postgresql", "postgres", "mysql"}}, 285 {"Linux", []string{"linux", "rhel", "red hat enterprise"}}, 286 {"AWS", []string{"aws", "amazon web services"}}, 287 {"Azure", []string{"azure"}}, 288 {"Kubernetes", []string{"kubernetes", "k8s"}}, 289 {"Docker", []string{"docker", "containerization"}}, 290 {"Terraform", []string{"terraform"}}, 291 {"Ansible", []string{"ansible"}}, 292 {"Splunk", []string{"splunk"}}, 293 {"SIEM", []string{"siem", "security information and event"}}, 294 {"Incident Response", []string{"incident response", "incident handling"}}, 295 {"Penetration Testing", []string{"penetration testing", "pen testing", "pentest"}}, 296 {"Vulnerability Management", []string{"vulnerability management", "vulnerability assessment"}}, 297 {"Network Security", []string{"network security", "firewall", "intrusion detection"}}, 298 {"Cloud Security", []string{"cloud security"}}, 299 {"DevSecOps", []string{"devsecops", "devops"}}, 300 {"RMF", []string{"rmf", "risk management framework"}}, 301 {"ATO", []string{"authority to operate", "ato"}}, 302 {"NIST", []string{"nist 800", "nist sp 800"}}, 303 {"Threat Intelligence", []string{"threat intelligence", "threat intel"}}, 304 {"Active Directory", []string{"active directory"}}, 305 {"VMware", []string{"vmware"}}, 306 {"Machine Learning", []string{"machine learning", "deep learning"}}, 307 {"Data Analysis", []string{"data analysis", "data analytics"}}, 308 {"Cryptography", []string{"cryptography", "encryption"}}, 309 {"Cybersecurity", []string{"cybersecurity", "cyber security"}}, 310 {"Systems Engineering", []string{"systems engineering", "system engineering"}}, 311 {"Software Engineering", []string{"software engineering", "software development"}}, 312 {"Intelligence Analysis", []string{"intelligence analysis", "all source", "sigint", "geoint"}}, 313} 314 315// certGazetteer maps common GovCon IT and cyber certifications to their mentions. 316var certGazetteer = []gazEntry{ 317 {"CISSP", []string{"cissp"}}, 318 {"CISM", []string{"cism"}}, 319 {"CISA", []string{"cisa"}}, 320 {"CompTIA Security+", []string{"security+", "security plus", "comptia security"}}, 321 {"CompTIA Network+", []string{"network+", "network plus"}}, 322 {"CASP+", []string{"casp+", "comptia advanced security"}}, 323 {"CEH", []string{"certified ethical hacker", "ceh certification"}}, 324 {"CCNA", []string{"ccna"}}, 325 {"CCNP", []string{"ccnp"}}, 326 {"PMP", []string{"pmp", "project management professional"}}, 327 {"ITIL", []string{"itil"}}, 328 {"GSEC", []string{"gsec"}}, 329 {"GCIH", []string{"gcih"}}, 330 {"OSCP", []string{"oscp", "offensive security certified"}}, 331 {"AWS Certified Solutions Architect", []string{"aws certified solutions architect", "aws solutions architect"}}, 332} 333 334// agencyGazetteer maps federal customers to their mentions, weighted toward the cleared defense space. 335var agencyGazetteer = []gazEntry{ 336 {"Air Force", []string{"air force", "usaf"}}, 337 {"Space Force", []string{"space force", "ussf"}}, 338 {"Army", []string{"u.s. army", "united states army", "department of the army"}}, 339 {"Navy", []string{"u.s. navy", "united states navy", "department of the navy"}}, 340 {"Marine Corps", []string{"marine corps", "usmc"}}, 341 {"DoD", []string{"department of defense", "dod", "d.o.d"}}, 342 {"DISA", []string{"disa", "defense information systems agency"}}, 343 {"NSA", []string{"national security agency", "nsa"}}, 344 {"CIA", []string{"central intelligence agency"}}, 345 {"DIA", []string{"defense intelligence agency"}}, 346 {"NGA", []string{"national geospatial"}}, 347 {"DHS", []string{"homeland security", "dhs"}}, 348 {"FBI", []string{"federal bureau of investigation", "fbi"}}, 349 {"DARPA", []string{"darpa"}}, 350 {"CENTCOM", []string{"centcom", "central command"}}, 351 {"SOCOM", []string{"socom", "special operations command"}}, 352 {"INDOPACOM", []string{"indopacom", "pacom", "indo-pacific command"}}, 353} 354