Janady13/missionmatch / internal/platform/benchimport/resume_source.go

1package benchimport 2 3import ( 4 "context" 5 "os" 6 "path/filepath" 7 "regexp" 8 "sort" 9 "strings" 10 11 "github.com/missionworkforce/missionmatch/internal/platform/docextract" 12 "github.com/missionworkforce/missionmatch/internal/platform/extraction" 13) 14 15// resumeSource is the FileSource backed by the pure Go docextract reader and the deterministic 16// extraction.Extractor. It walks a directory of resume files, converts each to plain text, runs the 17// offline extractor, and maps the structured RawExtraction onto a BenchRecord. No provider key, no 18// network, no browser. 19type resumeSource struct { 20 ext extraction.Extractor 21} 22 23// NewResumeSource builds a resume FileSource over the given extractor. Pass extraction.NewDeterministic() 24// for the offline default. 25func NewResumeSource(ext extraction.Extractor) FileSource { 26 return resumeSource{ext: ext} 27} 28 29// resumeExtensions are the file types the docextract reader handles. Other files in the directory are 30// skipped rather than mined, so a stray archive or image never produces a garbage record. 31var resumeExtensions = map[string]bool{ 32 ".pdf": true, ".docx": true, ".htm": true, ".html": true, ".txt": true, 33} 34 35// Resumes walks dir (non recursively, then one level of subdirectories so a per person folder works), 36// reads each resume file, extracts its text, runs the extractor, and yields one BenchRecord per readable 37// file. A file that fails to read, fails to extract, or yields no scoring signal is skipped and counted, 38// never aborts the batch. Profiles is the structured path; this is the unstructured resume path. 39func (s resumeSource) Resumes(ctx context.Context, dir string, yield func(rec BenchRecord, file string) error) (BatchResult, error) { 40 var res BatchResult 41 walkErr := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { 42 if err != nil { 43 return err 44 } 45 if d.IsDir() { 46 return nil 47 } 48 ext := strings.ToLower(filepath.Ext(path)) 49 if !resumeExtensions[ext] { 50 return nil 51 } 52 res.Read++ 53 data, err := os.ReadFile(path) 54 if err != nil { 55 res.Skipped++ 56 return nil 57 } 58 text, err := docextract.ExtractText(ctx, path, data) 59 if err != nil || strings.TrimSpace(text) == "" { 60 res.Skipped++ 61 return nil 62 } 63 rec, ok := s.mapResume(ctx, text) 64 if !ok { 65 res.Skipped++ 66 return nil 67 } 68 // Count a record as mapped only after yield accepts it, so a batch cap does not inflate the count. 69 if err := yield(rec, path); err != nil { 70 return err 71 } 72 res.Mapped++ 73 return nil 74 }) 75 if walkErr != nil { 76 return res, walkErr 77 } 78 return res, nil 79} 80 81// Profiles is the structured roster path and is not the resume reader's job. A resume source delegates 82// the CSV and JSONL decode to the structured source. 83func (s resumeSource) Profiles(ctx context.Context, path string, yield func(rec BenchRecord, batch string) error) (BatchResult, error) { 84 return structuredSource{}.Profiles(ctx, path, yield) 85} 86 87// mapResume turns extracted resume text into a BenchRecord, honoring invariant 5. The extractor yields a 88// RawExtraction whose FullName is a best effort person name and whose Skills, Certifications, Agencies, 89// ClearanceLevel, YearsExperience, and Location are structured tokens. The mapping places FullName ONLY on 90// BenchRecord.FullName (which lands on candidates.full_name and is never embedded), derives JobTitle ONLY 91// from a sanitized title gazetteer over the text (never a raw span, never a name), pulls the email ONLY 92// into the dedup contact field, and carries the clearance code for triage only. A record with no scoring 93// signal is rejected (ok is false) so an empty or garbage upload is not written. 94func (s resumeSource) mapResume(ctx context.Context, text string) (BenchRecord, bool) { 95 rx, err := s.ext.ExtractResume(ctx, text) 96 if err != nil { 97 return BenchRecord{}, false 98 } 99 if !rx.HasSignal() { 100 return BenchRecord{}, false 101 } 102 rec := BenchRecord{ 103 FullName: sanitizeName(rx.FullName), 104 Location: rx.Location, 105 JobTitle: detectTitle(text), 106 Skills: append([]string(nil), rx.Skills...), 107 Certs: append([]string(nil), rx.Certifications...), 108 Agencies: append([]string(nil), rx.Agencies...), 109 Years: rx.YearsExperience, 110 ClearanceCode: rx.ClearanceLevel, 111 Email: detectEmail(text), 112 Phone: detectPhone(text), 113 } 114 return rec, true 115} 116 117// sanitizeName trims a detected name. It is the ONLY field that may carry a person name; it lands on 118// candidates.full_name and is never embedded or scored. 119func sanitizeName(name string) string { return strings.TrimSpace(name) } 120 121// reEmail and rePhone find a contact email and phone for dedup ONLY. Neither value ever reaches a job 122// title, a role, a headline, or any field the embed profile composer reads (invariant 5). 123var ( 124 reEmail = regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`) 125 rePhone = regexp.MustCompile(`\b(?:\+?1[\s.\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b`) 126) 127 128func detectEmail(text string) string { 129 if m := reEmail.FindString(text); m != "" { 130 return strings.ToLower(m) 131 } 132 return "" 133} 134 135func detectPhone(text string) string { return rePhone.FindString(text) } 136 137// titleGazetteer maps a sanitized canonical job title to the lowercased mentions that resolve to it. A 138// JobTitle on a BenchRecord can ONLY be one of these canonical labels, so it is a closed sanitized 139// vocabulary: a raw resume span, a name, or an email can never become a title (invariant 5). The labels 140// are chosen to be useful headline anchors for the name free embed profile. 141var titleGazetteer = []struct { 142 canonical string 143 mentions []string 144}{ 145 {"Intelligence Analyst", []string{"intelligence analyst", "intel analyst", "all source analyst", "sigint analyst", "geoint analyst"}}, 146 {"Cyber Security Engineer", []string{"cyber security engineer", "cybersecurity engineer", "security engineer", "cyber engineer"}}, 147 {"Information System Security Officer", []string{"information system security officer", "isso", "information systems security officer"}}, 148 {"Information System Security Manager", []string{"information system security manager", "issm"}}, 149 {"Systems Engineer", []string{"systems engineer", "system engineer"}}, 150 {"Software Engineer", []string{"software engineer", "software developer"}}, 151 {"Network Engineer", []string{"network engineer"}}, 152 {"Cloud Architect", []string{"cloud architect", "solutions architect"}}, 153 {"Data Scientist", []string{"data scientist"}}, 154 {"Data Engineer", []string{"data engineer"}}, 155 {"Penetration Tester", []string{"penetration tester", "pen tester"}}, 156 {"Incident Responder", []string{"incident responder", "incident response analyst"}}, 157 {"Program Manager", []string{"program manager"}}, 158 {"Project Manager", []string{"project manager"}}, 159 {"DevSecOps Engineer", []string{"devsecops engineer", "devops engineer"}}, 160 {"COMSEC Custodian", []string{"comsec custodian"}}, 161} 162 163// detectTitle returns the first canonical title whose mention appears in the lowercased text, sanitized to 164// the closed vocabulary above, empty when none matches. A word boundary check keeps a short mention from 165// matching inside a larger word. Because the output is always a canonical label, the title is sanitized by 166// construction: a name, an email, or a raw span can never become a title. 167func detectTitle(text string) string { 168 lower := strings.ToLower(text) 169 var hits []string 170 for _, g := range titleGazetteer { 171 for _, m := range g.mentions { 172 if wordBoundaryContains(lower, m) { 173 hits = append(hits, g.canonical) 174 break 175 } 176 } 177 } 178 if len(hits) == 0 { 179 return "" 180 } 181 // Sort so the chosen title is deterministic regardless of order of appearance. 182 sort.Strings(hits) 183 return hits[0] 184} 185 186// wordBoundaryContains reports whether mention appears in lower on a word boundary, so "isso" matches 187// "ISSO" but not the inside of a longer token. mention is a lowercased gazetteer phrase. 188func wordBoundaryContains(lower, mention string) bool { 189 re := regexp.MustCompile(`\b` + regexp.QuoteMeta(mention) + `\b`) 190 return re.MatchString(lower) 191} 192