Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/modules/opportunity_intelligence/ingest.go
Original SHA-256	9ee85adb3bc91235ade8f61a18f338a0e7ab34c3d7c41155eac1a0d4a05f5969
Git blob SHA	ed29ca06dc61e3fb1352e76f3a9c81713d3cf16e
Reading-copy SHA-256	9ee85adb3bc91235ade8f61a18f338a0e7ab34c3d7c41155eac1a0d4a05f5969

Original source ↗

Every recorded occurrence

These 35 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / internal/modules/opportunity_intelligence/ingest.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓142 lines · 5.0 KB

package oppintel

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"

	"github.com/missionworkforce/missionmatch/internal/platform/audit"
	"github.com/missionworkforce/missionmatch/internal/platform/storage"
)

// RecordWriter persists a normalized record. The dbgen backed implementation (NewDBWriter) upserts into
// opportunity_records; tests use a fake. rawObjectKey and rawSHA256 are the storage pointer and the
// integrity stamp for the retained payload.
type RecordWriter interface {
	Upsert(ctx context.Context, rec OpportunityRecord, rawObjectKey, rawSHA256 string) error
}

// Auditor records the platform ingest event. The concrete audit.Writer satisfies it through System;
// tests use a fake. It is an interface so the ingester can be unit tested without a database.
type Auditor interface {
	System(ctx context.Context, e audit.Event) error
}

// IngestResult is the real, counted outcome of one source pull. Every count comes from records actually
// fetched and written, never a fabricated number.
type IngestResult struct {
	Source   string         `json:"source"`
	Fetched  int            `json:"fetched"`
	Upserted int            `json:"upserted"`
	ByKind   map[string]int `json:"by_kind"`
}

// Ingester pulls a source, retains each raw payload in object storage, upserts the normalized record,
// and writes one audit event for the run. It is the single ingestion path shared by every feed.
type Ingester struct {
	store   storage.Storer
	writer  RecordWriter
	auditor Auditor
}

// NewIngester builds an ingester. All three dependencies are required.
func NewIngester(store storage.Storer, writer RecordWriter, auditor Auditor) *Ingester {
	return &Ingester{store: store, writer: writer, auditor: auditor}
}

// Ingest pulls every record from src, retaining and upserting each, then writes the ingest audit event.
// A storage or database failure aborts the pull fail closed. A provider rate limit is a soft stop: the
// records pulled before the limit are kept and the run is still audited. The audit write is fail closed:
// a run the trail cannot record returns an error, and because every upsert is idempotent a rerun is
// safe.
func (i *Ingester) Ingest(ctx context.Context, src OpportunitySource) (IngestResult, error) {
	res := IngestResult{Source: src.Name(), ByKind: map[string]int{}}
	err := src.Fetch(ctx, func(rec OpportunityRecord) error {
		if rec.SourceName == "" {
			rec.SourceName = src.Name()
		}
		if !rec.Kind.Valid() {
			return fmt.Errorf("oppintel: record %q from %s has invalid kind %q", rec.SourceRecordID, res.Source, rec.Kind)
		}
		if rec.SourceRecordID == "" {
			return fmt.Errorf("oppintel: record from %s has an empty source record id", res.Source)
		}
		key, sha, serr := i.retainRaw(ctx, rec)
		if serr != nil {
			return serr
		}
		if werr := i.writer.Upsert(ctx, rec, key, sha); werr != nil {
			return werr
		}
		// Count only after a record is retained and written, so the reported Fetched and Upserted are
		// honest: a record that fails validation or a write aborts the run and is never counted.
		res.Fetched++
		res.Upserted++
		res.ByKind[string(rec.Kind)]++
		return nil
	})
	if err != nil && !errors.Is(err, ErrThrottled) {
		return res, err
	}
	if aerr := i.audit(ctx, res); aerr != nil {
		return res, aerr
	}
	return res, nil
}

// retainRaw writes the raw payload to object storage under a deterministic per record key and returns
// the key and the payload sha256. An empty payload is stored as an empty JSON object so the key is
// always valid and the integrity stamp is stable.
func (i *Ingester) retainRaw(ctx context.Context, rec OpportunityRecord) (string, string, error) {
	payload := rec.Raw
	if len(payload) == 0 {
		payload = json.RawMessage("{}")
	}
	key := rawKey(rec.SourceName, rec.SourceRecordID)
	sha := storage.SHA256Hex(payload)
	if err := i.store.Put(ctx, key, payload); err != nil {
		return "", "", fmt.Errorf("oppintel: retain raw: %w", err)
	}
	return key, sha, nil
}

func (i *Ingester) audit(ctx context.Context, res IngestResult) error {
	body, err := json.Marshal(res)
	if err != nil {
		return fmt.Errorf("oppintel: marshal ingest audit: %w", err)
	}
	return i.auditor.System(ctx, audit.Event{
		Action:     "opportunity.ingest",
		EntityType: "opportunity_source",
		EntityID:   res.Source,
		NewValues:  body,
	})
}

// rawKey is the object storage key for a record's retained payload. Source and id are reduced to a safe
// slug so a source supplied id can never traverse the store.
func rawKey(source, id string) string {
	return "opportunity_records/" + slug(source) + "/" + slug(id) + ".json"
}

// slug reduces a string to a storage safe token: lower case, with anything outside a to z, 0 to 9, dot,
// or underscore replaced by underscore. Empty input becomes "unknown".
func slug(s string) string {
	out := make([]rune, 0, len(s))
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '_':
			out = append(out, r)
		case r >= 'A' && r <= 'Z':
			out = append(out, r+('a'-'A'))
		default:
			out = append(out, '_')
		}
	}
	if len(out) == 0 {
		return "unknown"
	}
	return string(out)
}