package adapters import ( "context" "testing" "github.com/missionworkforce/missionmatch/internal/modules/ontology" "github.com/missionworkforce/missionmatch/internal/platform/extraction" ) type stubText struct{ blobs []string } func (s stubText) Texts(_ context.Context, yield func(string) error) error { for _, b := range s.blobs { if err := yield(b); err != nil { return err } } return nil } type stubExtractor struct{ perText map[string][]string } func (s stubExtractor) ExtractResume(_ context.Context, _ string) (extraction.RawExtraction, error) { return extraction.RawExtraction{}, nil } func (s stubExtractor) ExtractRequirement(_ context.Context, text string) (extraction.RawExtraction, error) { return extraction.RawExtraction{Skills: s.perText[text]}, nil } func TestCorpusYieldsDedupedSkills(t *testing.T) { blob1 := "Seeking a SIGINT analyst with Splunk and AWS experience." blob2 := "Cloud engineer with AWS and Kubernetes." ex := stubExtractor{perText: map[string][]string{ blob1: {"AWS", "Splunk"}, blob2: {"AWS", "Kubernetes"}, }} src := NewCorpusSourceWith(stubText{blobs: []string{blob1, blob2}}, ex) seen := map[string]int{} err := src.Fetch(context.Background(), func(r ontology.VocabRecord) error { if r.Kind != ontology.KindSkill { t.Fatalf("unexpected kind %q", r.Kind) } seen[r.Term]++ return nil }) if err != nil { t.Fatalf("fetch: %v", err) } if seen["AWS"] != 1 { t.Fatalf("AWS should be deduped to 1, got %d", seen["AWS"]) } if seen["Splunk"] != 1 || seen["Kubernetes"] != 1 { t.Fatalf("expected Splunk and Kubernetes once each, got %v", seen) } }