Janady13/missionmatch / internal/platform/scrub/scrub.go

1// Package scrub is the contents scrubbed teaching boundary between a tenant and the shared, stateless 2// MEGAMIND inference plane. It exists to enforce one rule from the architecture: MEGAMIND may learn only 3// the CRAFT of the work (abstract patterns), never any client facts, and nothing one client learns may 4// ever come from another client's contents. 5// 6// The boundary has two halves, both enforced here so the rule holds by construction rather than by 7// reviewer vigilance: 8// 9// 1. TeachingSignal is a closed struct of ONLY abstract, contents free fields (counts, ratios, enum 10// categories, booleans). It carries no id, no name, no email, no free text, no org, and no raw 11// value. A craft pattern such as "reviewers approved an adjacent skill substitution N of M times at 12// similarity band X" fits; a candidate, an org, or a resume does not. 13// 2. Validate is a defense in depth scan that REJECTS any serialized payload still carrying a client 14// fact marker (a UUID, an email, or an overlong free text run). Both the tenant side (before send) 15// and the gateway side (on receive) call it, so a leak is refused at two independent layers. 16// 17// If a future change adds a free text field to TeachingSignal, Validate will start failing the tests, 18// which is the intended tripwire. 19package scrub 20 21import ( 22 "bytes" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "regexp" 27) 28 29// ErrClientFactLeak is returned when a teaching payload still contains a client fact marker. 30var ErrClientFactLeak = errors.New("scrub: payload contains a client fact and cannot cross the teaching boundary") 31 32// SimilarityBand is a coarse, contents free bucket of a semantic similarity, so a craft signal can speak 33// about "a near match" without carrying the underlying text or score precision that could fingerprint a 34// tenant's data. 35type SimilarityBand string 36 37const ( 38 // These are a coarse, public quantization of a match's cosine strength for a teaching signal; they 39 // are NOT the engine's met/unmet boundary. That boundary is engine.DefaultSemanticThreshold (a 40 // calibrated 0.62 today), which can sit inside the low band, so a band edge must never be read as 41 // the decision cutoff. 42 BandExact SimilarityBand = "exact" // 1.00 43 BandHigh SimilarityBand = "high" // 0.90 to 0.99 44 BandMedium SimilarityBand = "medium" // 0.80 to 0.89 45 BandLow SimilarityBand = "low" // below 0.80 46 BandUnknown SimilarityBand = "unknown" // no signal 47) 48 49// Outcome is the human decision a craft signal generalizes over, taken verbatim from the existing 50// slate_members.decision vocabulary so the signal is interpretable without any candidate context. 51type Outcome string 52 53const ( 54 OutcomeApproved Outcome = "approved" 55 OutcomeRejected Outcome = "rejected" 56 OutcomePending Outcome = "pending" 57) 58 59// TeachingSignal is the ONLY thing the tenant side may send to MEGAMIND's teaching channel. Every field 60// is an abstraction of the work, never a fact about a person, an org, or a document. There is 61// deliberately no string field that can hold free text, no identifier, and no raw score. 62type TeachingSignal struct { 63 // Pattern names the craft observation in a fixed, enumerated vocabulary (not free text). Example: 64 // "adjacent_skill_substitution_outcome". Callers must use a constant from Patterns; Validate rejects 65 // anything outside it. 66 Pattern string `json:"pattern"` 67 // Band buckets the semantic similarity involved (no raw score, no text). 68 Band SimilarityBand `json:"band"` 69 // Outcome is the human decision being generalized. 70 Outcome Outcome `json:"outcome"` 71 // Observations is the count of events this signal aggregates (an integer, never a row id). 72 Observations int `json:"observations"` 73 // Successes is how many of Observations matched the outcome (an integer). 74 Successes int `json:"successes"` 75 // ModelVersion is the deterministic engine model version the signal sits on top of (public, not a 76 // client fact), so a craft pattern is always tied to the scoring contract that produced it. 77 ModelVersion string `json:"model_version"` 78} 79 80// Patterns is the closed vocabulary of craft observations. A signal whose Pattern is not here is refused, 81// so the teaching channel cannot be repurposed to smuggle a free text field. 82var Patterns = map[string]bool{ 83 "adjacent_skill_substitution_outcome": true, 84 "clearance_gap_override_outcome": true, 85 "experience_shortfall_outcome": true, 86 "reranker_label_agreement": true, 87} 88 89// registeredModelVersions is the allow list of engine model versions a craft signal may sit on. It is 90// empty by default and is populated by callers through RegisterModelVersion, never inside this package. 91// scrub is internal/platform and MUST NOT import the engine module or any internal/modules package: that 92// would invert the dependency direction. The producer (which legitimately imports engine) registers 93// engine.ModelVersion at init so the boundary accepts the live version and refuses a drifted one. The 94// effect is the same tripwire as an unknown Pattern: a model_version that nobody registered is a sign the 95// scoring contract changed underneath the signal, so the signal is refused rather than silently folded 96// into a different version's aggregate. 97var registeredModelVersions = map[string]bool{} 98 99// RegisterModelVersion records an engine model version the teaching boundary will accept. Callers invoke 100// it at init with engine.ModelVersion. Registering the same version twice is a no op. It is the only way 101// a model version becomes valid, so the allow list cannot be widened from outside this seam. 102func RegisterModelVersion(v string) { 103 if v != "" { 104 registeredModelVersions[v] = true 105 } 106} 107 108// uuidRe and emailRe are the unambiguous client fact markers. A teaching payload that still contains 109// either has not been scrubbed and must not cross the boundary. 110var ( 111 uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) 112 emailRe = regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`) 113) 114 115// maxFieldLen caps any string field so a craft enum can never be used as a free text smuggling channel. 116const maxFieldLen = 64 117 118// Validate is the boundary check. It runs on the tenant side before send AND on the gateway side on 119// receive. It refuses a payload that is not a well formed TeachingSignal, that uses a Pattern outside the 120// closed vocabulary, that carries a count that does not add up, or whose serialized form still contains a 121// UUID or an email. It returns ErrClientFactLeak (wrapped) on any client fact marker. 122func Validate(raw []byte) error { 123 // Marker scan first: this catches a leak even if the struct shape changes underneath us. 124 if uuidRe.Match(raw) { 125 return fmt.Errorf("%w: uuid", ErrClientFactLeak) 126 } 127 if emailRe.Match(raw) { 128 return fmt.Errorf("%w: email", ErrClientFactLeak) 129 } 130 131 dec := json.NewDecoder(bytes.NewReader(raw)) 132 dec.DisallowUnknownFields() // a stray extra field (a smuggled fact) is refused 133 var s TeachingSignal 134 if err := dec.Decode(&s); err != nil { 135 return fmt.Errorf("scrub: not a teaching signal: %w", err) 136 } 137 if !Patterns[s.Pattern] { 138 return fmt.Errorf("scrub: pattern %q is not in the craft vocabulary", clip(s.Pattern)) 139 } 140 if len(s.Pattern) > maxFieldLen || len(s.Band) > maxFieldLen || len(s.Outcome) > maxFieldLen || len(s.ModelVersion) > maxFieldLen { 141 return errors.New("scrub: a field exceeds the abstract length cap") 142 } 143 // A craft pattern is always tied to the scoring contract that produced it, so an empty model_version 144 // is meaningless and refused. A non empty but unregistered version is refused the same way an unknown 145 // Pattern is: it signals the scoring contract drifted out from under the signal. 146 if s.ModelVersion == "" { 147 return errors.New("scrub: model_version is required, a craft signal must name its scoring contract") 148 } 149 if !registeredModelVersions[s.ModelVersion] { 150 return fmt.Errorf("scrub: model_version %q is not a registered scoring contract", clip(s.ModelVersion)) 151 } 152 if s.Observations < 0 || s.Successes < 0 || s.Successes > s.Observations { 153 return errors.New("scrub: counts are not a valid aggregate") 154 } 155 switch s.Band { 156 case BandExact, BandHigh, BandMedium, BandLow, BandUnknown: 157 default: 158 return fmt.Errorf("scrub: band %q is not a valid bucket", clip(string(s.Band))) 159 } 160 switch s.Outcome { 161 case OutcomeApproved, OutcomeRejected, OutcomePending: 162 default: 163 return fmt.Errorf("scrub: outcome %q is not a valid decision", clip(string(s.Outcome))) 164 } 165 return nil 166} 167 168// Marshal serializes a TeachingSignal and refuses to emit one that would fail the boundary, so a caller 169// cannot accidentally hand raw bytes to the transport. 170func Marshal(s TeachingSignal) ([]byte, error) { 171 b, err := json.Marshal(s) 172 if err != nil { 173 return nil, err 174 } 175 if err := Validate(b); err != nil { 176 return nil, err 177 } 178 return b, nil 179} 180 181func clip(s string) string { 182 if len(s) > maxFieldLen { 183 return s[:maxFieldLen] 184 } 185 return s 186} 187