Janady13/missionmatch / internal/integration/analytics_integration_test.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/integration/analytics_integration_test.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/integration/analytics_integration_test.go
Original SHA-256	2a82aff38305bd2998951c832130edb8f7f75ef50ff72f3ac320ea106aa27045
Git blob SHA	2cf4457383ed9b78d9b35f3097be0aafdee314b1
Reading-copy SHA-256	2a82aff38305bd2998951c832130edb8f7f75ef50ff72f3ac320ea106aa27045
Original source ↗
Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.
Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/integration/analytics_integration_test.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/integration/analytics_integration_test.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Recovered source transcript

Download complete source text ↓323 lines · 12.0 KB
//go:build integration

package integration

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/google/uuid"

	"github.com/missionworkforce/missionmatch/internal/modules/recruiting"
)

// WARNING. Point MM_APP_DSN and MM_MIGRATOR_DSN at a scratch or staging database, never at production.

// backdateRequisition moves a requisition's opened_at so a cohort can span real durations without the
// test waiting. Done through the migrator because opened_at is not an application writable field.
func backdateRequisition(t *testing.T, org, req uuid.UUID, daysAgo int) {
	t.Helper()
	if _, err := migrator.Exec(context.Background(),
		// make_interval takes an integer directly. Concatenating the number into a string and casting it
		// forces pgx to encode an int as text, which it refuses, and the failure looks like a driver bug
		// rather than a query one.
		`UPDATE requisitions SET opened_at = now() - make_interval(days => $3)
		  WHERE organization_id = $1 AND id = $2`, org, req, daysAgo); err != nil {
		t.Fatalf("backdate: %v", err)
	}
}

// TestAnalyticsCorrectsSurvivorshipBias is the point of this phase.
//
// Five orders filled quickly and five still open far longer. The naive figure, which every other tool
// reports, sees only the five that filled and says the desk is fast. The corrected figure keeps the open
// ones in as lower bounds and tells the truth.
func TestAnalyticsCorrectsSurvivorshipBias(t *testing.T) {
	org := seedOrg(t)
	_, ctx := seedRecruiter(t, org, "Analytics Recruiter")
	svc := reqSvc()

	// Five that filled fast: opened 40 days ago, placed 10 days after opening.
	for i := 0; i < 5; i++ {
		r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Fast role", Openings: 1})
		if err != nil {
			t.Fatalf("create: %v", err)
		}
		req := uuid.MustParse(r.ID)
		backdateRequisition(t, org, req, 40)
		cand := seedPipelineCandidate(t, org, "Fast cand")
		a, err := svc.Submit(ctx, req, recruiting.CreateApplicationInput{CandidateID: cand.String()})
		if err != nil {
			t.Fatalf("submit: %v", err)
		}
		start := time.Now().UTC().AddDate(0, 0, -30).Format("2006-01-02") // 10 days after opening
		if _, err := svc.CreatePlacement(ctx, uuid.MustParse(a.ID), recruiting.CreatePlacementInput{
			EmploymentType: "contract", StartDate: start, BillRate: "150", PayRate: "100"}); err != nil {
			t.Fatalf("place: %v", err)
		}
	}
	// Seven opened 80 days ago and still open, so each is known to have taken at least 80 days.
	//
	// Seven rather than five deliberately. With an even five and five, survival falls to exactly one half
	// at the fill time and 9.83 days IS the correct median, so that cohort demonstrates nothing. With
	// fewer than half filled the curve never reaches one half at all, and the corrected answer becomes
	// the honest one: not yet known. The naive figure still cheerfully reports ten days.
	for i := 0; i < 7; i++ {
		r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Stuck role", Openings: 1})
		if err != nil {
			t.Fatalf("create: %v", err)
		}
		backdateRequisition(t, org, uuid.MustParse(r.ID), 80)
	}

	v, err := svc.Analytics(ctx, time.Now().UTC().AddDate(0, 0, -120).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics: %v", err)
	}
	ttf := v.TimeToStart
	if ttf.Cohort != 12 || ttf.FilledN != 5 || ttf.OpenN != 7 {
		t.Fatalf("cohort accounting: %+v", ttf)
	}
	// The naive figure sees only the fast five and reports about ten days.
	if ttf.NaiveMedian == "" {
		t.Fatal("no naive median was computed")
	}
	naive := analyticsFloat(t, ttf.NaiveMedian)
	if naive > 12 {
		t.Errorf("naive median = %s days, expected roughly 10 from the fast cohort", ttf.NaiveMedian)
	}
	// The corrected figure keeps the seven stuck orders in the calculation. Only five of twelve have
	// filled, so survival never falls to one half and there is no median yet. Reporting the naive ten
	// days here would be the fabrication this whole phase exists to prevent.
	if ttf.MedianReached {
		t.Errorf("a median of %s was reported when only %d of %d orders have filled",
			ttf.Median, ttf.FilledN, ttf.Cohort)
	}
	if ttf.Median != "" {
		t.Errorf("median = %q, want it absent and distinguishable from zero", ttf.Median)
	}
	// The still open orders are visible in the report rather than silently dropped, which is what lets a
	// reader see why the median is unavailable.
	if v.Aging == nil || v.OldestOpenDays < 75 {
		t.Errorf("oldest open = %d days, want the 80 day cohort to be visible", v.OldestOpenDays)
	}
	// The curve must be a real survival function.
	if len(ttf.Curve) == 0 {
		t.Error("no survival curve was produced")
	}
	prev := 1.0
	for _, p := range ttf.Curve {
		s := analyticsFloat(t, p.Surviving)
		if s > prev+1e-9 {
			t.Errorf("survival rose from %v to %v", prev, s)
		}
		prev = s
	}
}

// analyticsFloat parses a reported decimal string. Named distinctly because the MAG suite already has a
// mustFloat with different semantics.
func analyticsFloat(t *testing.T, s string) float64 {
	t.Helper()
	var f float64
	if _, err := fmt.Sscanf(s, "%g", &f); err != nil {
		t.Fatalf("value %q is not a number: %v", s, err)
	}
	return f
}

// TestAnalyticsReportsBothTimeToFillClocks. The industry does not agree on whether time to fill ends at
// the accepted offer or the start date, so reporting one silently is how two tools describe the same
// desk differently.
func TestAnalyticsReportsBothClocks(t *testing.T) {
	org := seedOrg(t)
	_, ctx := seedRecruiter(t, org, "Two Clocks")
	svc := reqSvc()

	r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Both clocks", Openings: 1})
	if err != nil {
		t.Fatalf("create: %v", err)
	}
	req := uuid.MustParse(r.ID)
	backdateRequisition(t, org, req, 30)
	cand := seedPipelineCandidate(t, org, "Cand")
	a, _ := svc.Submit(ctx, req, recruiting.CreateApplicationInput{CandidateID: cand.String()})
	o, err := svc.ExtendOffer(ctx, uuid.MustParse(a.ID), recruiting.CreateOfferInput{
		BillRate: "150", PayRate: "100"})
	if err != nil {
		t.Fatalf("offer: %v", err)
	}
	if _, err := svc.RespondToOffer(ctx, uuid.MustParse(o.ID), recruiting.OfferStatusInput{
		ExpectedStatus: "extended", Status: "accepted"}); err != nil {
		t.Fatalf("accept: %v", err)
	}
	if _, err := svc.CreatePlacement(ctx, uuid.MustParse(a.ID), recruiting.CreatePlacementInput{
		EmploymentType: "contract", StartDate: time.Now().UTC().AddDate(0, 0, 14).Format("2006-01-02"),
		BillRate: "150", PayRate: "100"}); err != nil {
		t.Fatalf("place: %v", err)
	}

	v, err := svc.Analytics(ctx, time.Now().UTC().AddDate(0, 0, -60).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics: %v", err)
	}
	if v.TimeToOffer.FilledN != 1 {
		t.Errorf("time to offer counted %d filled, want 1", v.TimeToOffer.FilledN)
	}
	if v.TimeToStart.FilledN != 1 {
		t.Errorf("time to start counted %d filled, want 1", v.TimeToStart.FilledN)
	}
	// The start is later than the offer, so the second clock must read longer than the first.
	toOffer := analyticsFloat(t, v.TimeToOffer.Median)
	toStart := analyticsFloat(t, v.TimeToStart.Median)
	if toStart <= toOffer {
		t.Errorf("time to start %v is not after time to offer %v", toStart, toOffer)
	}
	if v.TimeZone == "" {
		t.Error("the report does not say which timezone its days are counted in")
	}
}

// TestAnalyticsSuppressesMeaninglessRates. One filled requisition is a hundred percent fill rate and it
// means nothing. Counts are always shown; only the ratio is withheld.
func TestAnalyticsSuppressesMeaninglessRates(t *testing.T) {
	org := seedOrg(t)
	_, ctx := seedRecruiter(t, org, "Tiny Cohort")
	svc := reqSvc()
	r, _ := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Only one", Openings: 1})
	rr, _ := svc.GetRequisition(ctx, uuid.MustParse(r.ID))
	if _, err := svc.ChangeStatus(ctx, uuid.MustParse(r.ID), recruiting.ChangeStatusInput{
		ExpectedUpdatedAt: rr.UpdatedAt, Status: "filled"}); err != nil {
		t.Fatalf("fill: %v", err)
	}

	v, err := svc.Analytics(ctx, time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics: %v", err)
	}
	if !v.FillRate.Suppressed {
		t.Error("a fill rate was reported for a cohort of one")
	}
	if v.FillRate.OfResolved != "" || v.FillRate.OfDecided != "" {
		t.Errorf("rates leaked despite suppression: %+v", v.FillRate)
	}
	if v.FillRate.Filled != 1 {
		t.Error("suppressing the rate must not suppress the counts")
	}
}

// TestAnalyticsEmptyOrgIsComplete. A new client's first login must render the whole report as zeros
// rather than a crash, a divide by zero, or a missing section.
func TestAnalyticsEmptyOrgIsComplete(t *testing.T) {
	org := seedOrg(t)
	_, ctx := seedRecruiter(t, org, "Brand New")
	svc := reqSvc()

	v, err := svc.Analytics(ctx, "")
	if err != nil {
		t.Fatalf("analytics on an empty org: %v", err)
	}
	if v.TimeToOffer.Cohort != 0 || v.TimeToStart.MedianReached {
		t.Errorf("an empty org reported a time to fill: %+v", v.TimeToStart)
	}
	if v.Funnel == nil || v.Aging == nil || v.Sources == nil || v.Recruiters == nil || v.Conversion == nil {
		t.Error("report slices must be empty, never nil, so the JSON is [] and not null")
	}
	if len(v.Funnel) != 6 {
		t.Errorf("funnel has %d rungs on an empty org, want 6 so the chart shape is stable", len(v.Funnel))
	}
	if len(v.Aging) != 5 {
		t.Errorf("aging has %d bands, want 5", len(v.Aging))
	}
	if v.TimeZone == "" {
		t.Error("no timezone reported")
	}
}

// TestAnalyticsFunnelIsMonotoneOnRealData. The pure test proves the arithmetic; this proves the query
// feeding it yields marks with the same property.
func TestAnalyticsFunnelIsMonotone(t *testing.T) {
	org := seedOrg(t)
	_, ctx := seedRecruiter(t, org, "Funnel Owner")
	svc := reqSvc()
	r, _ := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Funnel role", Openings: 6})
	req := uuid.MustParse(r.ID)

	ladder := []string{"screened", "submitted", "interviewing", "offered", "placed"}
	for i := 0; i < 6; i++ {
		cand := seedPipelineCandidate(t, org, "C")
		a, err := svc.Submit(ctx, req, recruiting.CreateApplicationInput{
			CandidateID: cand.String(), SourceTouch: "referral"})
		if err != nil {
			t.Fatalf("submit: %v", err)
		}
		cur := "sourced"
		for _, step := range ladder[:i] {
			if _, err := svc.MoveStage(ctx, uuid.MustParse(a.ID), recruiting.MoveStageInput{
				ExpectedStage: cur, ToStage: step}); err != nil {
				t.Fatalf("advance: %v", err)
			}
			cur = step
		}
	}

	v, err := svc.Analytics(ctx, time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics: %v", err)
	}
	for i := 0; i+1 < len(v.Funnel); i++ {
		if v.Funnel[i+1].Count > v.Funnel[i].Count {
			t.Errorf("funnel widened: %s=%d then %s=%d",
				v.Funnel[i].Stage, v.Funnel[i].Count, v.Funnel[i+1].Stage, v.Funnel[i+1].Count)
		}
	}
	if v.Funnel[0].Count != 6 {
		t.Errorf("first rung = %d, want 6", v.Funnel[0].Count)
	}
	// Source effectiveness is attributed on first touch.
	found := false
	for _, s := range v.Sources {
		if s.Source == "referral" {
			found = true
			if s.Applications != 6 {
				t.Errorf("referral shows %d applications, want 6", s.Applications)
			}
		}
	}
	if !found {
		t.Error("the referral source did not appear in source effectiveness")
	}
}

// TestAnalyticsTenantIsolation, with a control.
func TestAnalyticsTenantIsolation(t *testing.T) {
	orgA := seedOrg(t)
	_, ctxA := seedRecruiter(t, orgA, "Tenant A")
	svc := reqSvc()
	for i := 0; i < 3; i++ {
		if _, err := svc.CreateRequisition(ctxA, recruiting.CreateRequisitionInput{Title: "A role"}); err != nil {
			t.Fatalf("create in A: %v", err)
		}
	}
	orgB := seedOrg(t)
	_, ctxB := seedRecruiter(t, orgB, "Tenant B")

	vB, err := svc.Analytics(ctxB, time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics in B: %v", err)
	}
	if vB.TimeToStart.Cohort != 0 {
		t.Fatalf("tenant B's report counts %d requisitions it does not own", vB.TimeToStart.Cohort)
	}
	// Control: A sees its own three.
	vA, err := svc.Analytics(ctxA, time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339))
	if err != nil {
		t.Fatalf("analytics in A: %v", err)
	}
	if vA.TimeToStart.Cohort != 3 {
		t.Errorf("tenant A's cohort = %d, want 3", vA.TimeToStart.Cohort)
	}
}