package benchimport import ( "context" "os" "path/filepath" "strings" "testing" ) // TestCSVColumnMapping proves the CSV decoder maps the standard bench columns by header name onto a // BenchRecord, splits the skill, cert, and agency lists, and routes the email and phone only to the dedup // contact fields, never to the title. func TestCSVColumnMapping(t *testing.T) { csv := strings.Join([]string{ "name,location,title,skills,certs,agencies,years,clearance,email,phone,external_id", `John Smith,"Honolulu, HI",Cyber Security Engineer,Python;Splunk;RMF,CISSP;Security+,Air Force;NSA,12,secret,john.smith@example.com,808-555-0100,bench-1`, }, "\n") dir := t.TempDir() path := filepath.Join(dir, "roster.csv") if err := os.WriteFile(path, []byte(csv), 0o644); err != nil { t.Fatalf("write csv: %v", err) } var got []BenchRecord res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error { got = append(got, rec) if batch != "roster.csv" { t.Fatalf("expected batch roster.csv, got %q", batch) } return nil }) if err != nil { t.Fatalf("Profiles csv: %v", err) } if res.Mapped != 1 || len(got) != 1 { t.Fatalf("expected 1 mapped row, got mapped=%d records=%d", res.Mapped, len(got)) } r := got[0] if r.FullName != "John Smith" { t.Fatalf("name mapping: got %q", r.FullName) } if r.JobTitle != "Cyber Security Engineer" { t.Fatalf("title mapping: got %q", r.JobTitle) } if r.Location != "Honolulu, HI" { t.Fatalf("location mapping: got %q", r.Location) } if !containsStr(r.Skills, "Python") || !containsStr(r.Skills, "RMF") { t.Fatalf("skill list mapping: got %v", r.Skills) } if !containsStr(r.Certs, "CISSP") { t.Fatalf("cert list mapping: got %v", r.Certs) } if !containsStr(r.Agencies, "Air Force") { t.Fatalf("agency list mapping: got %v", r.Agencies) } if r.Years != 12 { t.Fatalf("years mapping: got %d", r.Years) } if r.ClearanceCode != "secret" { t.Fatalf("clearance triage code mapping: got %q", r.ClearanceCode) } if r.Email != "john.smith@example.com" { t.Fatalf("email dedup field mapping: got %q", r.Email) } // The email and phone never ride into the title. if strings.Contains(r.JobTitle, "@") || strings.Contains(r.JobTitle, "555") { t.Fatalf("contact info leaked into the title: %q", r.JobTitle) } } // TestCSVMalformedRowSkip proves a malformed row (here a bare quote in a non quoted field) is skipped and // counted, while the well formed rows around it still map, so one bad row never aborts the batch. func TestCSVMalformedRowSkip(t *testing.T) { csv := strings.Join([]string{ "name,skills,clearance", "Good One,Python;SIEM,secret", `Bad"Row,x,secret`, "Good Two,RMF;ATO,top_secret", }, "\n") dir := t.TempDir() path := filepath.Join(dir, "roster.csv") if err := os.WriteFile(path, []byte(csv), 0o644); err != nil { t.Fatalf("write csv: %v", err) } var mapped []BenchRecord res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error { mapped = append(mapped, rec) return nil }) if err != nil { t.Fatalf("Profiles csv: %v", err) } if res.Skipped < 1 { t.Fatalf("expected at least one skipped malformed row, got skipped=%d", res.Skipped) } if len(mapped) < 2 { t.Fatalf("expected the two good rows to map around the bad row, got %d", len(mapped)) } for _, r := range mapped { if r.FullName == "" { t.Fatalf("a mapped row lost its name: %+v", r) } } } // TestCSVEmptyRowSkip proves a row with no scoring signal (no skills, no clearance, no certs, no years) is // skipped so a header only or blank profile is not imported. func TestCSVEmptyRowSkip(t *testing.T) { csv := strings.Join([]string{ "name,skills,clearance", "Signal Person,Python,secret", "No Signal Person,,", }, "\n") dir := t.TempDir() path := filepath.Join(dir, "roster.csv") if err := os.WriteFile(path, []byte(csv), 0o644); err != nil { t.Fatalf("write csv: %v", err) } var mapped int res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error { mapped++ return nil }) if err != nil { t.Fatalf("Profiles csv: %v", err) } if mapped != 1 { t.Fatalf("expected only the signal row to map, got %d", mapped) } if res.Skipped != 1 { t.Fatalf("expected the no signal row skipped, got skipped=%d", res.Skipped) } } // TestJSONLMapping proves the JSONL decoder maps one object per line, folds the alias keys, and skips a // line that does not parse. func TestJSONLMapping(t *testing.T) { jsonl := strings.Join([]string{ `{"name":"Ana Reyes","location":"Albuquerque, NM","title":"Data Scientist","skills":["Python","Machine Learning"],"certs":["CISSP"],"years":7,"clearance":"ts_sci","email":"ana@example.com","external_id":"b2"}`, `{ this is not valid json `, `{"full_name":"Lee Park","skills":["RMF","ATO"],"clearance_level":"secret"}`, }, "\n") dir := t.TempDir() path := filepath.Join(dir, "roster.jsonl") if err := os.WriteFile(path, []byte(jsonl), 0o644); err != nil { t.Fatalf("write jsonl: %v", err) } var got []BenchRecord res, err := NewStructuredSource().Profiles(context.Background(), path, func(rec BenchRecord, batch string) error { got = append(got, rec) return nil }) if err != nil { t.Fatalf("Profiles jsonl: %v", err) } if res.Skipped < 1 { t.Fatalf("expected the malformed json line skipped, got skipped=%d", res.Skipped) } if len(got) != 2 { t.Fatalf("expected 2 mapped rows, got %d", len(got)) } if got[0].FullName != "Ana Reyes" || got[0].JobTitle != "Data Scientist" { t.Fatalf("first jsonl row mapping: %+v", got[0]) } if got[0].ClearanceCode != "ts_sci" { t.Fatalf("first jsonl clearance triage code: %q", got[0].ClearanceCode) } if got[1].FullName != "Lee Park" || !containsStr(got[1].Skills, "RMF") { t.Fatalf("second jsonl alias mapping: %+v", got[1]) } }