Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / internal/platform/benchimport/structured_source_test.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | internal/platform/benchimport/structured_source_test.go |
| Original SHA-256 | 671b4a064fcd16dbac13eefc06ed294fff0bb0d13c915de8ca8e22104fad0213 |
| Git blob SHA | ba1a8301acc129c66a0a0644bb9eb4b299ff2c71 |
| Reading-copy SHA-256 | 671b4a064fcd16dbac13eefc06ed294fff0bb0d13c915de8ca8e22104fad0213 |
Recovered source transcript
Download complete source text ↓347 lines · 13.9 KB
package benchimport
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
// TestCSVColumnMapping proves the CSV decoder maps the standard bench columns by header name onto a
// BenchRecord, splits the skill, cert, and agency lists, and routes the email and phone only to the dedup
// contact fields, never to the title.
func TestCSVColumnMapping(t *testing.T) {
csv := strings.Join([]string{
"name,location,title,skills,certs,agencies,years,clearance,email,phone,external_id",
`John Smith,"Honolulu, HI",Cyber Security Engineer,Python;Splunk;RMF,CISSP;Security+,Air Force;NSA,12,secret,john.smith@example.com,808-555-0100,bench-1`,
}, "\n")
dir := t.TempDir()
path := filepath.Join(dir, "roster.csv")
if err := os.WriteFile(path, []byte(csv), 0o644); err != nil {
t.Fatalf("write csv: %v", err)
}
var got []BenchRecord
res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error {
got = append(got, rec)
if batch != "roster.csv" {
t.Fatalf("expected batch roster.csv, got %q", batch)
}
return nil
})
if err != nil {
t.Fatalf("Profiles csv: %v", err)
}
if res.Mapped != 1 || len(got) != 1 {
t.Fatalf("expected 1 mapped row, got mapped=%d records=%d", res.Mapped, len(got))
}
r := got[0]
if r.FullName != "John Smith" {
t.Fatalf("name mapping: got %q", r.FullName)
}
if r.JobTitle != "Cyber Security Engineer" {
t.Fatalf("title mapping: got %q", r.JobTitle)
}
if r.Location != "Honolulu, HI" {
t.Fatalf("location mapping: got %q", r.Location)
}
if !containsStr(r.Skills, "Python") || !containsStr(r.Skills, "RMF") {
t.Fatalf("skill list mapping: got %v", r.Skills)
}
if !containsStr(r.Certs, "CISSP") {
t.Fatalf("cert list mapping: got %v", r.Certs)
}
if !containsStr(r.Agencies, "Air Force") {
t.Fatalf("agency list mapping: got %v", r.Agencies)
}
if r.Years != 12 {
t.Fatalf("years mapping: got %d", r.Years)
}
if r.ClearanceCode != "secret" {
t.Fatalf("clearance triage code mapping: got %q", r.ClearanceCode)
}
if r.Email != "john.smith@example.com" {
t.Fatalf("email dedup field mapping: got %q", r.Email)
}
// The email and phone never ride into the title.
if strings.Contains(r.JobTitle, "@") || strings.Contains(r.JobTitle, "555") {
t.Fatalf("contact info leaked into the title: %q", r.JobTitle)
}
}
// TestCSVMalformedRowSkip proves a malformed row (here a bare quote in a non quoted field) is skipped and
// counted, while the well formed rows around it still map, so one bad row never aborts the batch.
func TestCSVMalformedRowSkip(t *testing.T) {
csv := strings.Join([]string{
"name,skills,clearance",
"Good One,Python;SIEM,secret",
`Bad"Row,x,secret`,
"Good Two,RMF;ATO,top_secret",
}, "\n")
dir := t.TempDir()
path := filepath.Join(dir, "roster.csv")
if err := os.WriteFile(path, []byte(csv), 0o644); err != nil {
t.Fatalf("write csv: %v", err)
}
var mapped []BenchRecord
res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error {
mapped = append(mapped, rec)
return nil
})
if err != nil {
t.Fatalf("Profiles csv: %v", err)
}
if res.Skipped < 1 {
t.Fatalf("expected at least one skipped malformed row, got skipped=%d", res.Skipped)
}
if len(mapped) < 2 {
t.Fatalf("expected the two good rows to map around the bad row, got %d", len(mapped))
}
for _, r := range mapped {
if r.FullName == "" {
t.Fatalf("a mapped row lost its name: %+v", r)
}
}
}
// TestCSVEmptyRowSkip proves a row with no scoring signal (no skills, no clearance, no certs, no years) is
// skipped so a header only or blank profile is not imported.
func TestCSVEmptyRowSkip(t *testing.T) {
csv := strings.Join([]string{
"name,skills,clearance",
"Signal Person,Python,secret",
"No Signal Person,,",
}, "\n")
dir := t.TempDir()
path := filepath.Join(dir, "roster.csv")
if err := os.WriteFile(path, []byte(csv), 0o644); err != nil {
t.Fatalf("write csv: %v", err)
}
var mapped int
res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error {
mapped++
return nil
})
if err != nil {
t.Fatalf("Profiles csv: %v", err)
}
if mapped != 1 {
t.Fatalf("expected only the signal row to map, got %d", mapped)
}
if res.Skipped != 1 {
t.Fatalf("expected the no signal row skipped, got skipped=%d", res.Skipped)
}
}
// TestJSONLMapping proves the JSONL decoder maps one object per line, folds the alias keys, and skips a
// line that does not parse.
func TestJSONLMapping(t *testing.T) {
jsonl := strings.Join([]string{
`{"name":"Ana Reyes","location":"Albuquerque, NM","title":"Data Scientist","skills":["Python","Machine Learning"],"certs":["CISSP"],"years":7,"clearance":"ts_sci","email":"ana@example.com","external_id":"b2"}`,
`{ this is not valid json `,
`{"full_name":"Lee Park","skills":["RMF","ATO"],"clearance_level":"secret"}`,
}, "\n")
dir := t.TempDir()
path := filepath.Join(dir, "roster.jsonl")
if err := os.WriteFile(path, []byte(jsonl), 0o644); err != nil {
t.Fatalf("write jsonl: %v", err)
}
var got []BenchRecord
res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error {
got = append(got, rec)
return nil
})
if err != nil {
t.Fatalf("Profiles jsonl: %v", err)
}
if res.Skipped < 1 {
t.Fatalf("expected the malformed json line skipped, got skipped=%d", res.Skipped)
}
if len(got) != 2 {
t.Fatalf("expected 2 mapped rows, got %d", len(got))
}
if got[0].FullName != "Ana Reyes" || got[0].JobTitle != "Data Scientist" {
t.Fatalf("first jsonl row mapping: %+v", got[0])
}
if got[0].ClearanceCode != "ts_sci" {
t.Fatalf("first jsonl clearance triage code: %q", got[0].ClearanceCode)
}
if got[1].FullName != "Lee Park" || !containsStr(got[1].Skills, "RMF") {
t.Fatalf("second jsonl alias mapping: %+v", got[1])
}
}
// A REAL EXPORT FROM A REAL ATS, which is the file a recruiter actually has. None of these headings match the
// canonical names, and before the alias table every one of these rows imported as nothing: the parser was
// correct and the product was unusable, which is the worst combination because nothing looks broken.
func TestRecruiterExportHeadersMapWithoutAnyoneEditingTheFile(t *testing.T) {
csvText := "Candidate Name,E-Mail,Mobile Phone,Current Title,City,Security Clearance,Years of Experience,Technical Skills,Customer,ATS ID\n" +
"Dana Reyes,DANA@example.invalid,555-0100,Systems Engineer,Columbia MD,TS/SCI,9,\"Kubernetes; AWS\",DIA,BH-4471\n"
var got []BenchRecord
res, err := decodeCSV(context.Background(), strings.NewReader(csvText), "ats.csv",
func(r BenchRecord, _ string) error { got = append(got, r); return nil })
if err != nil {
t.Fatalf("decode: %v", err)
}
if len(got) != 1 {
t.Fatalf("want 1 mapped record from a real ATS export, got %d (read %d, skipped %d)", len(got), res.Read, res.Skipped)
}
r := got[0]
// Each of these came from a heading the canonical list does not contain.
if r.FullName != "Dana Reyes" {
t.Errorf(`"Candidate Name" did not map to the name, got %q`, r.FullName)
}
if r.Email != "dana@example.invalid" {
t.Errorf(`"E-Mail" did not map and lowercase, got %q`, r.Email)
}
if r.Phone != "555-0100" {
t.Errorf(`"Mobile Phone" did not map to the phone, got %q`, r.Phone)
}
if r.JobTitle != "Systems Engineer" {
t.Errorf(`"Current Title" did not map to the job title, got %q`, r.JobTitle)
}
if r.Location != "Columbia MD" {
t.Errorf(`"City" did not map to the location, got %q`, r.Location)
}
if r.ClearanceCode != "TS/SCI" {
t.Errorf(`"Security Clearance" did not map, got %q. This is the column that decides whether a bench is usable.`, r.ClearanceCode)
}
if r.Years != 9 {
t.Errorf(`"Years of Experience" did not map, got %d`, r.Years)
}
if len(r.Skills) != 2 {
t.Errorf(`"Technical Skills" did not split, got %v`, r.Skills)
}
if len(r.Agencies) != 1 || r.Agencies[0] != "DIA" {
t.Errorf(`"Customer" did not map to agencies, got %v`, r.Agencies)
}
if r.ExternalID != "BH-4471" {
t.Errorf(`"ATS ID" did not map to the external id, so re importing would duplicate everyone, got %q`, r.ExternalID)
}
}
// AN UNREADABLE COLUMN MUST BE NAMED, not silently dropped. This is what turns "only some of my people came
// in" into a sentence the recruiter can act on.
func TestUnmappedHeadersAreReportedByName(t *testing.T) {
csvText := "Name,Willingness To Relocate,Desired Salary\nPat Lee,Yes,150000\n"
res, err := decodeCSV(context.Background(), strings.NewReader(csvText), "x.csv",
func(BenchRecord, string) error { return nil })
if err != nil {
t.Fatalf("decode: %v", err)
}
if len(res.UnmappedHeaders) != 2 {
t.Fatalf("want the 2 unreadable headings reported, got %v", res.UnmappedHeaders)
}
joined := strings.Join(res.UnmappedHeaders, ",")
if !strings.Contains(joined, "Willingness To Relocate") || !strings.Contains(joined, "Desired Salary") {
t.Errorf("the reported headings must be the recruiter's ORIGINAL text so they recognise them, got %v", res.UnmappedHeaders)
}
// And a recognised column must NOT be reported as unmapped, or the report becomes noise nobody reads.
if strings.Contains(joined, "Name") {
t.Error("a mapped column was reported as unmapped, which would train people to ignore the report")
}
}
// A CONTROL ON THE ALIAS TABLE ITSELF. A prefix strip that is too eager would map a manager's name onto the
// candidate, putting the wrong human in the bench, which is worse than importing nothing.
func TestAliasingDoesNotInventMappings(t *testing.T) {
if got := canonHeader("Hiring Manager Name"); got == "name" {
t.Errorf(`"Hiring Manager Name" must NOT become the candidate name, got %q`, got)
}
if got := canonHeader("Referred By Email"); got == "email" {
t.Errorf(`"Referred By Email" must NOT become the candidate email, got %q`, got)
}
if got := canonHeader("Candidate Name"); got != "name" {
t.Errorf(`"Candidate Name" should map to name, got %q`, got)
}
}
// AUTO SORT. A file whose headings nobody has ever seen must still land in the right columns, because the
// recruiter should not have to know our vocabulary. Every heading below is deliberately unrecognisable.
func TestAutoSortPlacesColumnsByTheirContents(t *testing.T) {
csvText := "Full Name,Eligibility Status,Reach Them At,Best Number,Based Out Of,Tenure\n" +
"Dana Reyes,TS/SCI,dana@example.invalid,555-010-0100,Columbia MD,9\n" +
"Sam Cole,Secret,sam@example.invalid,(555) 010-0200,Denver CO,4\n" +
"Alex Kim,None,alex@example.invalid,555.010.0300,Austin TX,12\n" +
"Jo Park,Top Secret,jo@example.invalid,5550100400,Reston VA,7\n"
var got []BenchRecord
res, err := decodeCSV(context.Background(), strings.NewReader(csvText), "unknown.csv",
func(r BenchRecord, _ string) error { got = append(got, r); return nil })
if err != nil {
t.Fatalf("decode: %v", err)
}
if len(got) != 4 {
t.Fatalf("want 4 records, got %d", len(got))
}
// None of these headings is in the alias table. They landed by their VALUES.
if got[0].ClearanceCode != "TS/SCI" {
t.Errorf(`"Eligibility Status" was not auto sorted to clearance, got %q`, got[0].ClearanceCode)
}
if got[0].Email != "dana@example.invalid" {
t.Errorf(`"Reach Them At" was not auto sorted to email, got %q`, got[0].Email)
}
if got[0].Phone != "555-010-0100" {
t.Errorf(`"Best Number" was not auto sorted to phone, got %q`, got[0].Phone)
}
if got[0].Location != "Columbia MD" {
t.Errorf(`"Based Out Of" was not auto sorted to location, got %q`, got[0].Location)
}
if got[0].Years != 9 {
t.Errorf(`"Tenure" was not auto sorted to years, got %d`, got[0].Years)
}
// And every inference is REPORTED, so nothing was sorted silently.
if len(res.Inferred) < 5 {
t.Errorf("want each inference reported for the preview, got %d: %+v", len(res.Inferred), res.Inferred)
}
for _, inf := range res.Inferred {
if inf.Basis == "" {
t.Errorf("inference %q carries no human readable basis, so a preview could not explain it", inf.Header)
}
}
}
// THE CONTROL. Auto sort must REFUSE what it cannot tell, or it is just a confident guesser. A column of
// arbitrary statuses must not become clearance, and salary must not become years just because it is numeric.
func TestAutoSortRefusesWhatItCannotTell(t *testing.T) {
inf := InferColumn("Availability", []string{"Immediately", "Two weeks", "Negotiable", "Unknown", "Ask"})
if inf.Applied() {
t.Errorf("an arbitrary status column was auto sorted to %q at %.2f, it should have been left alone",
inf.Canonical, inf.Confidence)
}
sal := InferColumn("Desired Salary", []string{"150000", "165000", "140000", "175000"})
if sal.Applied() && sal.Canonical == "years" {
t.Error("a salary column was auto sorted to years, which would put 150000 years of experience on a candidate")
}
// A single value is never enough, whatever it looks like.
one := InferColumn("Mystery", []string{"dana@example.invalid"})
if one.Applied() {
t.Error("a single sampled value was enough to auto sort, which is a guess wearing a confidence score")
}
}
// A REAL HEADING ALWAYS BEATS A GUESS. If the file says Email in a way we recognise, an inferred email column
// must not overwrite it.
func TestRecognisedHeadingWinsOverInference(t *testing.T) {
// A clearance column is present because hasSignal requires real workforce signal: a row carrying only a
// name and an email is deliberately NOT a bench record, and every row here would otherwise be skipped.
csvText := "Name,Email,Backup Contact,Clearance\n" +
"Dana Reyes,primary@example.invalid,other@example.invalid,TS/SCI\n" +
"Sam Cole,sam@example.invalid,sam2@example.invalid,Secret\n" +
"Jo Park,jo@example.invalid,jo2@example.invalid,Secret\n" +
"Alex Kim,alex@example.invalid,alex2@example.invalid,None\n"
var got []BenchRecord
if _, err := decodeCSV(context.Background(), strings.NewReader(csvText), "x.csv",
func(r BenchRecord, _ string) error { got = append(got, r); return nil }); err != nil {
t.Fatalf("decode: %v", err)
}
if len(got) == 0 {
t.Fatal("no records mapped")
}
if got[0].Email != "primary@example.invalid" {
t.Errorf("an inferred column overwrote the properly named Email column, got %q", got[0].Email)
}
}