Janady13/missionmatch / internal/platform/scrub/scrub.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/platform/scrub/scrub.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/platform/scrub/scrub.go
Original SHA-256	c7d8d48f7b263d9544b8b0412167f54b3938431e773cf068023559694d57c08a
Git blob SHA	69e1fed9ffe43ab9ab8ad73bc60edc950a438a7c
Reading-copy SHA-256	c7d8d48f7b263d9544b8b0412167f54b3938431e773cf068023559694d57c08a
Original source ↗
Recovered source transcript

Download complete source text ↓312 lines · 15.7 KB
// Package scrub is the contents scrubbed teaching boundary between a tenant and the shared, stateless
// MEGAMIND inference plane. It exists to enforce one rule from the architecture: MEGAMIND may learn only
// the CRAFT of the work (abstract patterns), never any client facts, and nothing one client learns may
// ever come from another client's contents.
//
// The boundary has two halves, both enforced here so the rule holds by construction rather than by
// reviewer vigilance:
//
//  1. TeachingSignal is a closed struct of ONLY abstract, contents free fields (counts, ratios, enum
//     categories, booleans). It carries no id, no name, no email, no free text, no org, and no raw
//     value. A craft pattern such as "reviewers approved an adjacent skill substitution N of M times at
//     similarity band X" fits; a candidate, an org, or a resume does not.
//  2. Validate is a defense in depth scan that REJECTS any serialized payload still carrying a client
//     fact marker (a UUID, an email, or an overlong free text run). Both the tenant side (before send)
//     and the gateway side (on receive) call it, so a leak is refused at two independent layers.
//
// If a future change adds a free text field to TeachingSignal, Validate will start failing the tests,
// which is the intended tripwire.
package scrub

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
)

// ErrClientFactLeak is returned when a teaching payload still contains a client fact marker.
var ErrClientFactLeak = errors.New("scrub: payload contains a client fact and cannot cross the teaching boundary")

// SimilarityBand is a coarse, contents free bucket of a semantic similarity, so a craft signal can speak
// about "a near match" without carrying the underlying text or score precision that could fingerprint a
// tenant's data.
type SimilarityBand string

const (
	// These are a coarse, public quantization of a match's cosine strength for a teaching signal; they
	// are NOT the engine's met/unmet boundary. That boundary is engine.DefaultSemanticThreshold (a
	// calibrated 0.62 today), which can sit inside the low band, so a band edge must never be read as
	// the decision cutoff.
	BandExact   SimilarityBand = "exact"   // 1.00
	BandHigh    SimilarityBand = "high"    // 0.90 to 0.99
	BandMedium  SimilarityBand = "medium"  // 0.80 to 0.89
	BandLow     SimilarityBand = "low"     // below 0.80
	BandUnknown SimilarityBand = "unknown" // no signal
)

// Outcome is the human decision a craft signal generalizes over, taken verbatim from the existing
// slate_members.decision vocabulary so the signal is interpretable without any candidate context.
type Outcome string

const (
	OutcomeApproved Outcome = "approved"
	OutcomeRejected Outcome = "rejected"
	OutcomePending  Outcome = "pending"
)

// TeachingSignal is the ONLY thing the tenant side may send to MEGAMIND's teaching channel. Every field
// is an abstraction of the work, never a fact about a person, an org, or a document. There is
// deliberately no string field that can hold free text, no identifier, and no raw score.
type TeachingSignal struct {
	// Pattern names the craft observation in a fixed, enumerated vocabulary (not free text). Example:
	// "adjacent_skill_substitution_outcome". Callers must use a constant from Patterns; Validate rejects
	// anything outside it.
	Pattern string `json:"pattern"`
	// Band buckets the semantic similarity involved (no raw score, no text).
	Band SimilarityBand `json:"band"`
	// Outcome is the human decision being generalized.
	Outcome Outcome `json:"outcome"`
	// Observations is the count of events this signal aggregates (an integer, never a row id).
	Observations int `json:"observations"`
	// Successes is how many of Observations matched the outcome (an integer).
	Successes int `json:"successes"`
	// ModelVersion is the deterministic engine model version the signal sits on top of (public, not a
	// client fact), so a craft pattern is always tied to the scoring contract that produced it.
	ModelVersion string `json:"model_version"`
}

// Patterns is the closed vocabulary of craft observations. A signal whose Pattern is not here is refused,
// so the teaching channel cannot be repurposed to smuggle a free text field.
var Patterns = map[string]bool{
	"adjacent_skill_substitution_outcome": true,
	"clearance_gap_override_outcome":      true,
	"experience_shortfall_outcome":        true,
	"reranker_label_agreement":            true,
}

// registeredModelVersions is the allow list of engine model versions a craft signal may sit on. It is
// empty by default and is populated by callers through RegisterModelVersion, never inside this package.
// scrub is internal/platform and MUST NOT import the engine module or any internal/modules package: that
// would invert the dependency direction. The producer (which legitimately imports engine) registers
// engine.ModelVersion at init so the boundary accepts the live version and refuses a drifted one. The
// effect is the same tripwire as an unknown Pattern: a model_version that nobody registered is a sign the
// scoring contract changed underneath the signal, so the signal is refused rather than silently folded
// into a different version's aggregate.
var registeredModelVersions = map[string]bool{}

// RegisterModelVersion records an engine model version the teaching boundary will accept. Callers invoke
// it at init with engine.ModelVersion. Registering the same version twice is a no op. It is the only way
// a model version becomes valid, so the allow list cannot be widened from outside this seam.
func RegisterModelVersion(v string) {
	if v != "" {
		registeredModelVersions[v] = true
	}
}

// uuidRe and emailRe are the unambiguous client fact markers. A teaching payload that still contains
// either has not been scrubbed and must not cross the boundary.
var (
	uuidRe  = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
	emailRe = regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`)
)

// maxFieldLen caps any string field so a craft enum can never be used as a free text smuggling channel.
const maxFieldLen = 64

// Validate is the boundary check. It runs on the tenant side before send AND on the gateway side on
// receive. It refuses a payload that is not a well formed TeachingSignal, that uses a Pattern outside the
// closed vocabulary, that carries a count that does not add up, or whose serialized form still contains a
// UUID or an email. It returns ErrClientFactLeak (wrapped) on any client fact marker.
func Validate(raw []byte) error {
	// Marker scan first: this catches a leak even if the struct shape changes underneath us.
	if uuidRe.Match(raw) {
		return fmt.Errorf("%w: uuid", ErrClientFactLeak)
	}
	if emailRe.Match(raw) {
		return fmt.Errorf("%w: email", ErrClientFactLeak)
	}

	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.DisallowUnknownFields() // a stray extra field (a smuggled fact) is refused
	var s TeachingSignal
	if err := dec.Decode(&s); err != nil {
		return fmt.Errorf("scrub: not a teaching signal: %w", err)
	}
	if !Patterns[s.Pattern] {
		return fmt.Errorf("scrub: pattern %q is not in the craft vocabulary", clip(s.Pattern))
	}
	if len(s.Pattern) > maxFieldLen || len(s.Band) > maxFieldLen || len(s.Outcome) > maxFieldLen || len(s.ModelVersion) > maxFieldLen {
		return errors.New("scrub: a field exceeds the abstract length cap")
	}
	// A craft pattern is always tied to the scoring contract that produced it, so an empty model_version
	// is meaningless and refused. A non empty but unregistered version is refused the same way an unknown
	// Pattern is: it signals the scoring contract drifted out from under the signal.
	if s.ModelVersion == "" {
		return errors.New("scrub: model_version is required, a craft signal must name its scoring contract")
	}
	if !registeredModelVersions[s.ModelVersion] {
		return fmt.Errorf("scrub: model_version %q is not a registered scoring contract", clip(s.ModelVersion))
	}
	if s.Observations < 0 || s.Successes < 0 || s.Successes > s.Observations {
		return errors.New("scrub: counts are not a valid aggregate")
	}
	switch s.Band {
	case BandExact, BandHigh, BandMedium, BandLow, BandUnknown:
	default:
		return fmt.Errorf("scrub: band %q is not a valid bucket", clip(string(s.Band)))
	}
	switch s.Outcome {
	case OutcomeApproved, OutcomeRejected, OutcomePending:
	default:
		return fmt.Errorf("scrub: outcome %q is not a valid decision", clip(string(s.Outcome)))
	}
	return nil
}

// Marshal serializes a TeachingSignal and refuses to emit one that would fail the boundary, so a caller
// cannot accidentally hand raw bytes to the transport.
func Marshal(s TeachingSignal) ([]byte, error) {
	b, err := json.Marshal(s)
	if err != nil {
		return nil, err
	}
	if err := Validate(b); err != nil {
		return nil, err
	}
	return b, nil
}

func clip(s string) string {
	if len(s) > maxFieldLen {
		return s[:maxFieldLen]
	}
	return s
}

// The outbound egress boundary. Where the teaching boundary above governs what MEGAMIND may LEARN,
// this governs what may LEAVE the local plane for an external inference backend such as OpenAI. It
// enforces the rule from the architecture: only public solicitation and knowledge text may reach an
// external model, and it fails closed to the local plane. Private tenant, candidate, resume, scoring,
// slate, and EEO data never leaves, always.

// EgressClass names the ONLY payload kinds that may cross to an external inference backend. A payload
// whose class is not one of these is refused, so the external lane cannot be repurposed to carry
// private data.
type EgressClass string

const (
	ClassPublicKnowledge      EgressClass = "PUBLIC_KNOWLEDGE"
	ClassPublicDataSummary    EgressClass = "PUBLIC_DATA_SUMMARY"
	ClassOntologySuggest      EgressClass = "ONTOLOGY_SUGGEST"
	ClassRequirementNormalize EgressClass = "REQUIREMENT_NORMALIZE"
)

// ErrEgressForbidden is returned when a payload under a permitted class still carries a private data
// marker, so an accidental leak is caught at the boundary rather than at the external backend.
var ErrEgressForbidden = errors.New("scrub: payload carries a private data marker and may not leave the local plane")

// ErrEgressClassUnknown is returned when the egress class is not one of the public allow listed
// classes. The gate fails closed: an unrecognized class never leaves the machine.
var ErrEgressClassUnknown = errors.New("scrub: egress class is not in the public allow list")

// phoneRe is a defense in depth marker for a north american phone number, one more unambiguous
// private data shape alongside a uuid and an email.
var phoneRe = regexp.MustCompile(`\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}`)

// Provenance is WHERE a payload came from, which is the property that actually decides whether it may
// leave. Gate 2 fixtures proved why the class and a marker scan are not enough on their own: a candidate
// row with the uuid, email, and phone already stripped is still one named cleared human with a clearance,
// a location, and an availability, and it sailed through; a customer's own solicitation package excerpt
// contains no personal data at all and sailed through too. The first is private because of WHAT IT IS and
// the second because of WHERE IT CAME FROM, and no amount of pattern matching on the bytes can recover
// either fact once the caller has flattened it into a string.
//
// So the caller must now DECLARE the source, from a closed vocabulary, and only two sources may ever
// leave. This is the same shape as the closed intent registry in the AI layer and for the same reason:
// the safe set is enumerated, everything else is refused, and a new source has to be added deliberately by
// someone who has thought about it.
type Provenance string

const (
	// ProvenancePublicCache is data retrieved from a public external source and recorded in
	// ingest_manifest, which is the SAM.gov notice cache. This is what REQUIREMENT_NORMALIZE exists for.
	ProvenancePublicCache Provenance = "public_cache"
	// ProvenanceCuratedCard is a knowledge card curated by an operator and cited to a primary source.
	ProvenanceCuratedCard Provenance = "curated_card"
	// ProvenanceTenantUpload is a document a tenant uploaded. NEVER leaves.
	ProvenanceTenantUpload Provenance = "tenant_upload"
	// ProvenanceTenantRecord is anything read from a tenant table: candidates, requirements, slates,
	// screenings, applications. NEVER leaves.
	ProvenanceTenantRecord Provenance = "tenant_record"
)

// ErrEgressProvenanceForbidden is returned when the payload's declared source is one that may never cross
// the boundary, or is not in the vocabulary at all. It is a distinct error from the marker scan because it
// is a distinct failure: the caller correctly told us where the data came from and the answer is no.
var ErrEgressProvenanceForbidden = errors.New("scrub: payload provenance may not cross to an external plane")

// ErrEgressNoProvenance is returned when a payload is offered with NO element provenance at all. It is a
// separate error from a forbidden provenance because it is a separate mistake: a caller that composed a
// payload without tracking where any of it came from. Silence is not permission.
var ErrEgressNoProvenance = errors.New("scrub: no element provenance supplied, so nothing proves this payload may leave")

// AllowEgress is the outbound boundary check for external inference.
//
// PROVENANCE IS NOT DECLARED AT CALL TIME. The provs argument is the set of provenance values READ FROM
// THE STORED ROWS that every element of the payload was composed from, and callers obtain it by selecting
// the column, never by naming a constant. Migration 0081 makes that column born with the row and immutable:
// a default supplies it, a CHECK bounds it, and a trigger overwrites any value on insert and refuses any
// change on update. So fabricating provenance requires shipping a migration, which is a named reviewable
// location, rather than typing a constant at a call site.
//
// FOUR conditions, all required, all failing closed:
//
//  1. at least one element provenance is present. An empty set is refused, because a payload nobody can
//     account for is exactly the payload that should not leave.
//  2. EVERY element's provenance is one of the two that may leave. One tenant element poisons the whole
//     payload, which is the correct semantics: a public notice with one paragraph of a customer's PWS
//     pasted into it is a customer's PWS.
//  3. the CLASS is one of the four public classes.
//  4. the payload carries no private data marker. This is a BACKSTOP against a composition bug, not the
//     main guarantee, and its known limit is asserted by name in the tests.
func AllowEgress(provs []Provenance, class EgressClass, payload []byte) error {
	if len(provs) == 0 {
		return ErrEgressNoProvenance
	}
	for _, prov := range provs {
		switch prov {
		case ProvenancePublicCache, ProvenanceCuratedCard:
		case ProvenanceTenantUpload, ProvenanceTenantRecord:
			return fmt.Errorf("%w: %s stays on the local plane", ErrEgressProvenanceForbidden, prov)
		default:
			return fmt.Errorf("%w: unknown provenance %q", ErrEgressProvenanceForbidden, clip(string(prov)))
		}
	}
	switch class {
	case ClassPublicKnowledge, ClassPublicDataSummary, ClassOntologySuggest, ClassRequirementNormalize:
	default:
		return fmt.Errorf("%w: %q", ErrEgressClassUnknown, clip(string(class)))
	}
	if uuidRe.Match(payload) {
		return fmt.Errorf("%w: uuid", ErrEgressForbidden)
	}
	if emailRe.Match(payload) {
		return fmt.Errorf("%w: email", ErrEgressForbidden)
	}
	if phoneRe.Match(payload) {
		return fmt.Errorf("%w: phone", ErrEgressForbidden)
	}
	return nil
}

// ContentHash is the stable fingerprint of an external bound payload, used to audit a call by hash and
// byte count without ever recording the content itself.
func ContentHash(payload []byte) string {
	sum := sha256.Sum256(payload)
	return hex.EncodeToString(sum[:])
}