package adapters import ( "context" "strings" "github.com/jackc/pgx/v5/pgxpool" "github.com/missionworkforce/missionmatch/internal/modules/ontology" "github.com/missionworkforce/missionmatch/internal/platform/extraction" ) // TextReader streams free text blobs from the retained federal corpus. The pool backed implementation // reads description and title columns; tests use a stub. type TextReader interface { Texts(ctx context.Context, yield func(string) error) error } // CorpusSource mines the GovCon vocabulary from text we already retain (no external API). It runs the // deterministic extractor over each blob and yields one KindSkill record per distinct skill mention, so the // ontology is grounded in the words Air Force and Space Force buyers actually use. These are global public // reference tables, so no PII or tenant data is read. type CorpusSource struct { reader TextReader extract extraction.Extractor } // NewCorpusSource builds the adapter using the deterministic offline extractor. func NewCorpusSource(reader TextReader) *CorpusSource { return &CorpusSource{reader: reader, extract: extraction.NewDeterministic()} } // NewCorpusSourceWith builds the adapter with an injected extractor, for tests. func NewCorpusSourceWith(reader TextReader, extract extraction.Extractor) *CorpusSource { return &CorpusSource{reader: reader, extract: extract} } // Name is the provenance label. func (s *CorpusSource) Name() string { return "Corpus Mining" } // Fetch runs the extractor over each retained text blob and yields each distinct skill once. func (s *CorpusSource) Fetch(ctx context.Context, yield func(ontology.VocabRecord) error) error { seen := map[string]bool{} return s.reader.Texts(ctx, func(text string) error { rx, err := s.extract.ExtractRequirement(ctx, text) if err != nil { return err } for _, skill := range rx.Skills { key := strings.ToLower(strings.TrimSpace(skill)) if key == "" || seen[key] { continue } seen[key] = true if err := yield(ontology.VocabRecord{ SourceName: "Corpus Mining", SourceRecordID: key, Kind: ontology.KindSkill, Term: skill, Category: "mined", }); err != nil { return err } } return nil }) } // PoolTextReader reads minable free text from the retained federal tables: USASpending award descriptions // (the richest), and SAM and unified opportunity titles. These are global public reference tables. type PoolTextReader struct{ pool *pgxpool.Pool } // NewPoolText builds a pool backed text reader. func NewPoolText(pool *pgxpool.Pool) *PoolTextReader { return &PoolTextReader{pool: pool} } // Texts streams each non empty text blob from the corpus tables. func (r *PoolTextReader) Texts(ctx context.Context, yield func(string) error) error { const q = ` SELECT description FROM usaspending_awards WHERE description <> '' UNION ALL SELECT title FROM sam_opportunities WHERE title <> '' UNION ALL SELECT title FROM opportunity_records WHERE title <> ''` rows, err := r.pool.Query(ctx, q) if err != nil { return err } defer rows.Close() for rows.Next() { var text string if err := rows.Scan(&text); err != nil { return err } if err := yield(text); err != nil { return err } } return rows.Err() }