Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/modules/ontology/adapters/corpus.go
Original SHA-256	84d800383fc0cfeed061442c1bcad792748211caadba0c85b9de8837e192a879
Git blob SHA	aecb8fe40e245d8be203f1c98860d6908d5e96db
Reading-copy SHA-256	84d800383fc0cfeed061442c1bcad792748211caadba0c85b9de8837e192a879

Original source ↗

Every recorded occurrence

These 20 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/modules/ontology/adapters/corpus.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓100 lines · 3.2 KB

package adapters

import (
	"context"
	"strings"

	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/modules/ontology"
	"github.com/missionworkforce/missionmatch/internal/platform/extraction"
)

// TextReader streams free text blobs from the retained federal corpus. The pool backed implementation
// reads description and title columns; tests use a stub.
type TextReader interface {
	Texts(ctx context.Context, yield func(string) error) error
}

// CorpusSource mines the GovCon vocabulary from text we already retain (no external API). It runs the
// deterministic extractor over each blob and yields one KindSkill record per distinct skill mention, so the
// ontology is grounded in the words Air Force and Space Force buyers actually use. These are global public
// reference tables, so no PII or tenant data is read.
type CorpusSource struct {
	reader  TextReader
	extract extraction.Extractor
}

// NewCorpusSource builds the adapter using the deterministic offline extractor.
func NewCorpusSource(reader TextReader) *CorpusSource {
	return &CorpusSource{reader: reader, extract: extraction.NewDeterministic()}
}

// NewCorpusSourceWith builds the adapter with an injected extractor, for tests.
func NewCorpusSourceWith(reader TextReader, extract extraction.Extractor) *CorpusSource {
	return &CorpusSource{reader: reader, extract: extract}
}

// Name is the provenance label.
func (s *CorpusSource) Name() string { return "Corpus Mining" }

// Fetch runs the extractor over each retained text blob and yields each distinct skill once.
func (s *CorpusSource) Fetch(ctx context.Context, yield func(ontology.VocabRecord) error) error {
	seen := map[string]bool{}
	return s.reader.Texts(ctx, func(text string) error {
		rx, err := s.extract.ExtractRequirement(ctx, text)
		if err != nil {
			return err
		}
		for _, skill := range rx.Skills {
			key := strings.ToLower(strings.TrimSpace(skill))
			if key == "" || seen[key] {
				continue
			}
			seen[key] = true
			if err := yield(ontology.VocabRecord{
				SourceName:     "Corpus Mining",
				SourceRecordID: key,
				Kind:           ontology.KindSkill,
				Term:           skill,
				Category:       "mined",
			}); err != nil {
				return err
			}
		}
		return nil
	})
}

// PoolTextReader reads minable free text from the retained federal tables: USASpending award descriptions
// (the richest), and SAM and unified opportunity titles. These are global public reference tables.
type PoolTextReader struct{ pool *pgxpool.Pool }

// NewPoolText builds a pool backed text reader.
func NewPoolText(pool *pgxpool.Pool) *PoolTextReader { return &PoolTextReader{pool: pool} }

// Texts streams each non empty text blob from the corpus tables.
func (r *PoolTextReader) Texts(ctx context.Context, yield func(string) error) error {
	const q = `
SELECT description FROM usaspending_awards WHERE description <> ''
UNION ALL
SELECT title FROM sam_opportunities WHERE title <> ''
UNION ALL
SELECT title FROM opportunity_records WHERE title <> ''`
	rows, err := r.pool.Query(ctx, q)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var text string
		if err := rows.Scan(&text); err != nil {
			return err
		}
		if err := yield(text); err != nil {
			return err
		}
	}
	return rows.Err()
}