package magstore

import (
	"context"
	"errors"

	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/learn"
)

// Quality gate tolerances and the metric bin count.
//
// Ranking (AUC) is the primary discrimination metric: a challenger must not rank materially worse than the
// champion (AUCTolerance absorbs floating point noise and a tiny held out split). Calibration (ECE) is guarded
// two ways so the gate is honest against a DEGENERATE champion: the prior only champion is a CONSTANT predictor
// (AUC near 0.5), and a constant predictor's ECE is a lucky baseline (it is just the gap between the constant
// and the overall base rate), not a meaningful bar. So the challenger's calibration passes when it is either not
// materially worse than the champion (ECETolerance, the mature champion versus challenger case) OR within an
// absolute cold start bound (ECECeiling, so a genuine discriminating challenger is not blocked by a constant
// champion's deceptively low ECE, while a truly miscalibrated challenger past the ceiling still fails). A later
// isotonic recalibration tightens ECE further.
const (
	ECETolerance = 0.03
	ECECeiling   = 0.15
	AUCTolerance = 0.02
	MetricBins   = 10
)

// QualityResult is the held out comparison of a champion and a challenger: the split size, each model's ranking
// (AUC) and calibration (ECE), and whether the challenger clears the quality gate.
type QualityResult struct {
	HeldOut  int
	ChampAUC float64
	ChalAUC  float64
	ChampECE float64
	ChalECE  float64
	Pass     bool
}

// EvaluateHeldOut reads the pooled decided bids and holds out a fraction as a test split, taking the tail of
// the read's stable id order so the split is deterministic and reproducible (the same ledger always yields the
// same held out set), then scores the split under both models and computes AUC and ECE for each. The challenger
// passes when it ranks at least as well as the champion (AUC within tolerance) and calibrates at least as well
// (ECE within tolerance). It is pure over the stored models and the label read; the fairness gate is a separate
// check the caller runs through the compliance pool.
func EvaluateHeldOut(ctx context.Context, pool *pgxpool.Pool, champ, chal Model, holdout float64) (QualityResult, error) {
	rows, err := ReadLabeledBids(ctx, pool)
	if err != nil {
		return QualityResult{}, err
	}
	decided := make([]learn.LabeledRow, 0, len(rows))
	for _, r := range rows {
		if learn.IsDecided(r.Result) {
			decided = append(decided, r)
		}
	}
	if len(decided) == 0 {
		return QualityResult{}, errors.New("magstore: no decided bids to evaluate")
	}
	if holdout <= 0 || holdout >= 1 {
		holdout = 0.30
	}
	cut := len(decided) - int(float64(len(decided))*holdout)
	if cut >= len(decided) {
		cut = len(decided) - 1 // always hold out at least one row
	}
	if cut < 0 {
		cut = 0
	}
	test := decided[cut:] // the tail of the stable id order, a deterministic held out split

	champProbs := make([]float64, len(test))
	chalProbs := make([]float64, len(test))
	y := make([]int, len(test))
	for i, r := range test {
		champProbs[i] = champ.Score(r.Features)
		chalProbs[i] = chal.Score(r.Features)
		if r.Result == learn.ResultWin {
			y[i] = 1
		}
	}
	res := QualityResult{
		HeldOut:  len(test),
		ChampAUC: learn.AUC(champProbs, y),
		ChalAUC:  learn.AUC(chalProbs, y),
		ChampECE: learn.ECE(champProbs, y, MetricBins),
		ChalECE:  learn.ECE(chalProbs, y, MetricBins),
	}
	aucNotWorse := res.ChalAUC >= res.ChampAUC-AUCTolerance
	eceNotWorse := res.ChalECE <= res.ChampECE+ECETolerance // relative: not worse than the champion
	eceAcceptable := res.ChalECE <= ECECeiling              // absolute: within the cold start calibration bound
	res.Pass = aucNotWorse && (eceNotWorse || eceAcceptable)
	return res, nil
}