package oppintel import ( "context" "encoding/json" "errors" "fmt" "github.com/missionworkforce/missionmatch/internal/platform/audit" "github.com/missionworkforce/missionmatch/internal/platform/storage" ) // RecordWriter persists a normalized record. The dbgen backed implementation (NewDBWriter) upserts into // opportunity_records; tests use a fake. rawObjectKey and rawSHA256 are the storage pointer and the // integrity stamp for the retained payload. type RecordWriter interface { Upsert(ctx context.Context, rec OpportunityRecord, rawObjectKey, rawSHA256 string) error } // Auditor records the platform ingest event. The concrete audit.Writer satisfies it through System; // tests use a fake. It is an interface so the ingester can be unit tested without a database. type Auditor interface { System(ctx context.Context, e audit.Event) error } // IngestResult is the real, counted outcome of one source pull. Every count comes from records actually // fetched and written, never a fabricated number. type IngestResult struct { Source string `json:"source"` Fetched int `json:"fetched"` Upserted int `json:"upserted"` ByKind map[string]int `json:"by_kind"` } // Ingester pulls a source, retains each raw payload in object storage, upserts the normalized record, // and writes one audit event for the run. It is the single ingestion path shared by every feed. type Ingester struct { store storage.Storer writer RecordWriter auditor Auditor } // NewIngester builds an ingester. All three dependencies are required. func NewIngester(store storage.Storer, writer RecordWriter, auditor Auditor) *Ingester { return &Ingester{store: store, writer: writer, auditor: auditor} } // Ingest pulls every record from src, retaining and upserting each, then writes the ingest audit event. // A storage or database failure aborts the pull fail closed. A provider rate limit is a soft stop: the // records pulled before the limit are kept and the run is still audited. The audit write is fail closed: // a run the trail cannot record returns an error, and because every upsert is idempotent a rerun is // safe. func (i *Ingester) Ingest(ctx context.Context, src OpportunitySource) (IngestResult, error) { res := IngestResult{Source: src.Name(), ByKind: map[string]int{}} err := src.Fetch(ctx, func(rec OpportunityRecord) error { if rec.SourceName == "" { rec.SourceName = src.Name() } if !rec.Kind.Valid() { return fmt.Errorf("oppintel: record %q from %s has invalid kind %q", rec.SourceRecordID, res.Source, rec.Kind) } if rec.SourceRecordID == "" { return fmt.Errorf("oppintel: record from %s has an empty source record id", res.Source) } key, sha, serr := i.retainRaw(ctx, rec) if serr != nil { return serr } if werr := i.writer.Upsert(ctx, rec, key, sha); werr != nil { return werr } // Count only after a record is retained and written, so the reported Fetched and Upserted are // honest: a record that fails validation or a write aborts the run and is never counted. res.Fetched++ res.Upserted++ res.ByKind[string(rec.Kind)]++ return nil }) if err != nil && !errors.Is(err, ErrThrottled) { return res, err } if aerr := i.audit(ctx, res); aerr != nil { return res, aerr } return res, nil } // retainRaw writes the raw payload to object storage under a deterministic per record key and returns // the key and the payload sha256. An empty payload is stored as an empty JSON object so the key is // always valid and the integrity stamp is stable. func (i *Ingester) retainRaw(ctx context.Context, rec OpportunityRecord) (string, string, error) { payload := rec.Raw if len(payload) == 0 { payload = json.RawMessage("{}") } key := rawKey(rec.SourceName, rec.SourceRecordID) sha := storage.SHA256Hex(payload) if err := i.store.Put(ctx, key, payload); err != nil { return "", "", fmt.Errorf("oppintel: retain raw: %w", err) } return key, sha, nil } func (i *Ingester) audit(ctx context.Context, res IngestResult) error { body, err := json.Marshal(res) if err != nil { return fmt.Errorf("oppintel: marshal ingest audit: %w", err) } return i.auditor.System(ctx, audit.Event{ Action: "opportunity.ingest", EntityType: "opportunity_source", EntityID: res.Source, NewValues: body, }) } // rawKey is the object storage key for a record's retained payload. Source and id are reduced to a safe // slug so a source supplied id can never traverse the store. func rawKey(source, id string) string { return "opportunity_records/" + slug(source) + "/" + slug(id) + ".json" } // slug reduces a string to a storage safe token: lower case, with anything outside a to z, 0 to 9, dot, // or underscore replaced by underscore. Empty input becomes "unknown". func slug(s string) string { out := make([]rune, 0, len(s)) for _, r := range s { switch { case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '_': out = append(out, r) case r >= 'A' && r <= 'Z': out = append(out, r+('a'-'A')) default: out = append(out, '_') } } if len(out) == 0 { return "unknown" } return string(out) }