Janady13/missionmatch / internal/modules/ontology/ingest.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/ontology/ingest.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/modules/ontology/ingest.go
Original SHA-256	0b91b73e5b097f2016dbd156653e8a21c20ec08fb373b635110b7685e296efad
Git blob SHA	95787799e013be75d64af823bc38b43cc5e4d5b7
Reading-copy SHA-256	0b91b73e5b097f2016dbd156653e8a21c20ec08fb373b635110b7685e296efad

Original source ↗

Every recorded occurrence

These 20 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/ontology/ingest.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/modules/ontology/ingest.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓129 lines · 4.0 KB

package ontology

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"

	"github.com/missionworkforce/missionmatch/internal/platform/audit"
	"github.com/missionworkforce/missionmatch/internal/platform/storage"
)

// RecordWriter persists a normalized record. The pool backed implementation (NewDBWriter) upserts into
// ontology_terms; tests use a fake. rawObjectKey and rawSHA256 are the storage pointer and integrity stamp.
type RecordWriter interface {
	Upsert(ctx context.Context, rec VocabRecord, rawObjectKey, rawSHA256 string) error
}

// Auditor records the platform ingest event. The concrete audit.Writer satisfies it through System.
type Auditor interface {
	System(ctx context.Context, e audit.Event) error
}

// IngestResult is the real, counted outcome of one source pull. Every count comes from records actually
// retained and written, never a fabricated number.
type IngestResult struct {
	Source   string         `json:"source"`
	Fetched  int            `json:"fetched"`
	Upserted int            `json:"upserted"`
	ByKind   map[string]int `json:"by_kind"`
}

// Ingester pulls a source, retains each raw payload in object storage, upserts the normalized record, and
// writes one audit event for the run. It is the single ingestion path shared by every ontology source.
type Ingester struct {
	store   storage.Storer
	writer  RecordWriter
	auditor Auditor
}

// NewIngester builds an ingester. All three dependencies are required.
func NewIngester(store storage.Storer, writer RecordWriter, auditor Auditor) *Ingester {
	return &Ingester{store: store, writer: writer, auditor: auditor}
}

// Ingest pulls every record from src, retaining and upserting each, then writes the ingest audit event. A
// storage or database failure aborts the pull fail closed. A provider rate limit is a soft stop. The audit
// write is fail closed, and because every upsert is idempotent a rerun is safe.
func (i *Ingester) Ingest(ctx context.Context, src OntologySource) (IngestResult, error) {
	res := IngestResult{Source: src.Name(), ByKind: map[string]int{}}
	err := src.Fetch(ctx, func(rec VocabRecord) error {
		if rec.SourceName == "" {
			rec.SourceName = src.Name()
		}
		if !rec.Kind.Valid() {
			return fmt.Errorf("ontology: record %q from %s has invalid kind %q", rec.SourceRecordID, res.Source, rec.Kind)
		}
		if rec.SourceRecordID == "" {
			return fmt.Errorf("ontology: record from %s has an empty source record id", res.Source)
		}
		key, sha, serr := i.retainRaw(ctx, rec)
		if serr != nil {
			return serr
		}
		if werr := i.writer.Upsert(ctx, rec, key, sha); werr != nil {
			return werr
		}
		res.Fetched++
		res.Upserted++
		res.ByKind[string(rec.Kind)]++
		return nil
	})
	if err != nil && !errors.Is(err, ErrThrottled) {
		return res, err
	}
	if aerr := i.audit(ctx, res); aerr != nil {
		return res, aerr
	}
	return res, nil
}

func (i *Ingester) retainRaw(ctx context.Context, rec VocabRecord) (string, string, error) {
	payload := rec.Raw
	if len(payload) == 0 {
		payload = json.RawMessage("{}")
	}
	key := rawKey(rec.SourceName, rec.SourceRecordID)
	sha := storage.SHA256Hex(payload)
	if err := i.store.Put(ctx, key, payload); err != nil {
		return "", "", fmt.Errorf("ontology: retain raw: %w", err)
	}
	return key, sha, nil
}

func (i *Ingester) audit(ctx context.Context, res IngestResult) error {
	body, err := json.Marshal(res)
	if err != nil {
		return fmt.Errorf("ontology: marshal ingest audit: %w", err)
	}
	return i.auditor.System(ctx, audit.Event{
		Action:     "ontology.ingest",
		EntityType: "ontology_source",
		EntityID:   res.Source,
		NewValues:  body,
	})
}

func rawKey(source, id string) string {
	return "ontology_terms/" + slug(source) + "/" + slug(id) + ".json"
}

func slug(s string) string {
	out := make([]rune, 0, len(s))
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '_':
			out = append(out, r)
		case r >= 'A' && r <= 'Z':
			out = append(out, r+('a'-'A'))
		default:
			out = append(out, '_')
		}
	}
	if len(out) == 0 {
		return "unknown"
	}
	return string(out)
}