Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / internal/platform/benchimport/infer.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | internal/platform/benchimport/infer.go |
| Original SHA-256 | a64b85858966bc47b053070d52943c18b560550022bd5347d418cb95d4c2e35e |
| Git blob SHA | 4e691724a4d5f76a8e0985f3ef4087974b8279f4 |
| Reading-copy SHA-256 | a64b85858966bc47b053070d52943c18b560550022bd5347d418cb95d4c2e35e |
Recovered source transcript
Download complete source text ↓211 lines · 7.5 KB
package benchimport
import (
"strings"
)
// COLUMN INFERENCE FROM THE VALUES, not the heading.
//
// The alias table handles a heading someone has seen before. This handles the one nobody has: a column called
// "Eligibility Status" holding TS/SCI and Secret is a clearance column whatever it is named, and a column of
// things shaped like name@place.tld is an email column even if the heading is blank. A recruiter should not
// have to know what we call it.
//
// IT IS DETERMINISTIC AND IT USES NO MODEL. Every classifier below is a closed rule over the cell contents,
// so the same file infers the same way on every run, on every machine, forever. That matters more here than
// cleverness would: an import that guessed differently on Tuesday than it did on Monday is an import nobody
// can trust with their bench, and a wrong guess puts the wrong human in front of a customer.
//
// IT PROPOSES, IT DOES NOT DECIDE. Inference returns a confidence and the caller reports it. Identity columns
// (name, email, phone) are deliberately held to a higher bar than descriptive ones, because a mis mapped
// clearance is a bad filter while a mis mapped name is the wrong person.
// Inference is one proposed column mapping with the evidence behind it.
type Inference struct {
Header string // the recruiter's original heading, so they recognise it
Canonical string // what we believe it is
Confidence float64 // fraction of non empty sampled values that fit
Sampled int // how many non empty values were looked at
Basis string // a short human sentence, shown in the preview
}
// Applied reports whether this inference is confident enough to use without a person confirming it. The bar is
// higher for identity fields, and nothing is ever applied on a single value.
func (i Inference) Applied() bool {
if i.Sampled < 3 || i.Canonical == "" {
return false
}
switch i.Canonical {
case "name", "email", "phone", "external_id":
return i.Confidence >= 0.95
default:
return i.Confidence >= 0.80
}
}
// clearanceVocab is the closed set a clearance cell can say. It is a vocabulary rather than a pattern because
// clearance is the one field where a false positive is expensive: a column of arbitrary statuses must NOT be
// read as clearance, and only these words mean what they appear to mean.
var clearanceVocab = map[string]bool{
"ts/sci": true, "ts sci": true, "tssci": true, "top secret/sci": true, "top secret sci": true,
"top secret": true, "ts": true, "secret": true, "s": true, "confidential": true,
"public trust": true, "public": true, "none": true, "no clearance": true, "not cleared": true,
"interim secret": true, "interim ts": true, "interim top secret": true,
"ts/sci w/ poly": true, "ts/sci with polygraph": true, "ts/sci ci poly": true,
"ts/sci full scope": true, "ts/sci fs poly": true, "ci poly": true, "full scope poly": true,
"dod secret": true, "dod top secret": true, "q": true, "l": true,
}
var usStates = map[string]bool{
"al": true, "ak": true, "az": true, "ar": true, "ca": true, "co": true, "ct": true, "de": true,
"fl": true, "ga": true, "hi": true, "id": true, "il": true, "in": true, "ia": true, "ks": true,
"ky": true, "la": true, "me": true, "md": true, "ma": true, "mi": true, "mn": true, "ms": true,
"mo": true, "mt": true, "ne": true, "nv": true, "nh": true, "nj": true, "nm": true, "ny": true,
"nc": true, "nd": true, "oh": true, "ok": true, "or": true, "pa": true, "ri": true, "sc": true,
"sd": true, "tn": true, "tx": true, "ut": true, "vt": true, "va": true, "wa": true, "wv": true,
"wi": true, "wy": true, "dc": true,
}
// InferColumn looks at up to a few dozen cell values and proposes what the column is. An empty proposal means
// nothing fit well enough, which is a perfectly good answer and better than a guess.
func InferColumn(header string, values []string) Inference {
sample := make([]string, 0, 40)
for _, v := range values {
v = strings.TrimSpace(v)
if v != "" {
sample = append(sample, v)
}
if len(sample) == 40 {
break
}
}
if len(sample) == 0 {
return Inference{Header: header}
}
best := Inference{Header: header, Sampled: len(sample)}
consider := func(canon string, hits int, basis string) {
c := float64(hits) / float64(len(sample))
if c > best.Confidence {
best.Canonical, best.Confidence, best.Basis = canon, c, basis
}
}
var email, phone, clearance, years, state, listy, personName int
for _, v := range sample {
lv := strings.ToLower(v)
if looksEmail(v) {
email++
}
if looksPhone(v) {
phone++
}
if clearanceVocab[lv] {
clearance++
}
if n, ok := smallInt(v); ok && n >= 0 && n <= 60 {
years++
}
if looksCityState(v) {
state++
}
if strings.Count(v, ";") >= 1 || strings.Count(v, ",") >= 2 {
listy++
}
if looksPersonName(v) {
personName++
}
}
consider("email", email, "the values are email addresses")
consider("phone", phone, "the values are phone numbers")
consider("clearance", clearance, "the values are clearance levels")
consider("location", state, "the values look like City, ST")
consider("skills", listy, "the cells hold several values separated by a semicolon or commas")
// Years is checked LAST among numerics and deliberately weakened: almost any small number column would
// match, so it only wins when the heading does not already suggest something else entirely.
if !strings.Contains(strings.ToLower(header), "salary") && !strings.Contains(strings.ToLower(header), "rate") {
consider("years", years, "the values are small whole numbers in a plausible experience range")
}
// A name is proposed only when nothing else fit, because two capitalised words is the least specific
// signal here and the most costly to get wrong.
if best.Canonical == "" {
consider("name", personName, "the values look like people's names")
}
return best
}
func looksEmail(v string) bool {
at := strings.IndexByte(v, '@')
if at <= 0 || at == len(v)-1 || strings.ContainsAny(v, " \t") {
return false
}
return strings.Contains(v[at+1:], ".")
}
func looksPhone(v string) bool {
d := 0
for _, r := range v {
switch {
case r >= '0' && r <= '9':
d++
case r == ' ', r == '-', r == '(', r == ')', r == '+', r == '.', r == 'x', r == 'X':
default:
return false
}
}
return d >= 10 && d <= 15
}
func smallInt(v string) (int, bool) {
n := 0
if v == "" {
return 0, false
}
for _, r := range v {
if r < '0' || r > '9' {
return 0, false
}
n = n*10 + int(r-'0')
if n > 1000 {
return 0, false
}
}
return n, true
}
func looksCityState(v string) bool {
i := strings.LastIndexByte(v, ',')
tail := v
if i >= 0 {
tail = v[i+1:]
}
tail = strings.TrimSpace(strings.ToLower(tail))
// The LAST token, not the first. "Columbia MD" has no comma, and taking the first token checked
// "columbia" against the state list and found nothing, so every comma free location silently failed.
if len(tail) > 2 {
if sp := strings.Fields(tail); len(sp) > 0 {
tail = sp[len(sp)-1]
}
}
return len(tail) == 2 && usStates[tail]
}
// looksPersonName is deliberately strict. Two or three words, letters and the punctuation names carry, no
// digits, and not a single word, because one capitalised word is a city, a title, an agency, or anything else.
func looksPersonName(v string) bool {
if strings.ContainsAny(v, "0123456789@;/") {
return false
}
f := strings.Fields(v)
if len(f) < 2 || len(f) > 4 {
return false
}
for _, w := range f {
for _, r := range w {
if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && r != '\'' && r != '-' && r != '.' && r != ',' {
return false
}
}
}
return true
}