Janady13/missionmatch / internal/integration/recruitmatch_stress_test.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/integration/recruitmatch_stress_test.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	internal/integration/recruitmatch_stress_test.go
Original SHA-256	7b66fdf8183073f032a6cc5a3abaf7491aa9b1f515ba916ccab52c5951b84cb7
Git blob SHA	6c09b3e706edc791a38da5eb9b83ccfdbc28988c
Reading-copy SHA-256	7b66fdf8183073f032a6cc5a3abaf7491aa9b1f515ba916ccab52c5951b84cb7

Original source ↗

Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/integration/recruitmatch_stress_test.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / internal/integration/recruitmatch_stress_test.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original

Recovered source transcript

Download complete source text ↓177 lines · 6.0 KB

//go:build integration

// WARNING: these tests SEED REAL ROWS. seedOrg and seedCandidate insert organizations and candidates
// that persist after the run. Point MM_APP_DSN and MM_MIGRATOR_DSN at a disposable database, never at
// production. Running the suite against production leaves org-<uuid> rows behind that have to be purged
// by hand, which is exactly the pollution pattern this database was already cleaned of once.

package integration

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/google/uuid"

	"github.com/missionworkforce/missionmatch/internal/modules/recruitmatch"
	"github.com/missionworkforce/missionmatch/internal/platform/tenant"
)

// TestRecruitDashboardConcurrentTenantIsolation is the load bearing safety proof for this module.
//
// recruitMATCH builds per request TEMP tables inside one transaction on a POOLED connection. That design
// is only safe if two things hold under contention: the app.current_org_id GUC is transaction local so it
// cannot survive onto the next borrower of the connection, and the temp tables are dropped ON COMMIT so a
// later request on the same backend cannot observe a previous tenant's rows.
//
// Sequential tests cannot prove either property, because a single request never overlaps another. This
// test hammers the same pool from many goroutines across several organizations at once, each holding a
// different, known bench size, and asserts every response reports exactly its own. A leak shows up as a
// count belonging to a different organization.
func TestRecruitDashboardConcurrentTenantIsolation(t *testing.T) {
	const (
		orgCount           = 4
		requestsPerOrg     = 12
		candidatesPerOrgAt = 2 // org i gets (i+1)*candidatesPerOrgAt candidates, so every org differs
	)

	type orgFixture struct {
		id   uuid.UUID
		want int64
	}

	fixtures := make([]orgFixture, 0, orgCount)
	for i := 0; i < orgCount; i++ {
		org := seedOrg(t)
		n := (i + 1) * candidatesPerOrgAt
		for c := 0; c < n; c++ {
			seedCandidate(t, org, "secret", "active", true)
		}
		fixtures = append(fixtures, orgFixture{id: org, want: int64(n)})
	}

	svc := recruitmatch.NewService(appDB)
	asOf := time.Date(2026, 7, 20, 0, 0, 0, 0, time.UTC)

	var wg sync.WaitGroup
	errs := make(chan string, orgCount*requestsPerOrg)

	for _, f := range fixtures {
		for r := 0; r < requestsPerOrg; r++ {
			wg.Add(1)
			go func(f orgFixture, attempt int) {
				defer wg.Done()
				ctx := tenant.WithOrg(context.Background(), f.id.String())
				v, err := svc.RecruiterDashboard(ctx, asOf)
				if err != nil {
					errs <- fmt.Sprintf("org %s attempt %d: %v", f.id, attempt, err)
					return
				}
				if v.OrganizationID != f.id.String() {
					errs <- fmt.Sprintf("org %s attempt %d: view reports org %s", f.id, attempt, v.OrganizationID)
				}
				if v.Headline.Identified != f.want {
					errs <- fmt.Sprintf("LEAK: org %s attempt %d saw %d candidates, owns %d",
						f.id, attempt, v.Headline.Identified, f.want)
				}
				// The funnel must stay monotonic under contention as well, since a partially visible temp
				// table would show up here before it showed up in a count.
				h := v.Headline
				if !(h.Identified >= h.Screened && h.Screened >= h.Submitted && h.Submitted >= h.Approved) {
					errs <- fmt.Sprintf("org %s attempt %d: funnel not monotonic %+v", f.id, attempt, h)
				}
			}(f, r)
		}
	}

	wg.Wait()
	close(errs)

	failures := 0
	for e := range errs {
		if failures < 12 {
			t.Error(e)
		}
		failures++
	}
	if failures > 0 {
		t.Fatalf("%d of %d concurrent requests failed isolation", failures, orgCount*requestsPerOrg)
	}
}

// TestRecruitDashboardRepeatedCallsAreStableUnderConcurrency proves the view is byte stable for a fixed
// asOf even when many goroutines drive the same organization at once. A temp table that survived a commit
// or a GUC that drifted would surface as a differing payload between two concurrent reads.
func TestRecruitDashboardRepeatedCallsAreStableUnderConcurrency(t *testing.T) {
	org := seedOrg(t)
	for i := 0; i < 5; i++ {
		seedCandidate(t, org, "ts_sci", "active", true)
	}

	svc := recruitmatch.NewService(appDB)
	asOf := time.Date(2026, 7, 20, 0, 0, 0, 0, time.UTC)
	ctx := tenant.WithOrg(context.Background(), org.String())

	baseline, err := svc.RecruiterDashboard(ctx, asOf)
	if err != nil {
		t.Fatalf("baseline: %v", err)
	}

	const n = 24
	var wg sync.WaitGroup
	bad := make(chan string, n)
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func(attempt int) {
			defer wg.Done()
			v, err := svc.RecruiterDashboard(ctx, asOf)
			if err != nil {
				bad <- fmt.Sprintf("attempt %d: %v", attempt, err)
				return
			}
			if v.Headline != baseline.Headline {
				bad <- fmt.Sprintf("attempt %d headline drift: %+v vs %+v", attempt, v.Headline, baseline.Headline)
			}
			if v.GeneratedAt != baseline.GeneratedAt {
				bad <- fmt.Sprintf("attempt %d generated_at drift: %s vs %s", attempt, v.GeneratedAt, baseline.GeneratedAt)
			}
			if len(v.Clearance) != len(baseline.Clearance) {
				bad <- fmt.Sprintf("attempt %d clearance length drift", attempt)
				return
			}
			for j := range v.Clearance {
				if v.Clearance[j] != baseline.Clearance[j] {
					bad <- fmt.Sprintf("attempt %d clearance row %d drift", attempt, j)
				}
			}
		}(i)
	}
	wg.Wait()
	close(bad)

	for e := range bad {
		t.Error(e)
	}
}

// TestRecruitDashboardTempTablesDoNotCollideWithCommandCenter proves the rm_ prefix choice was necessary
// and sufficient. Both dashboards build per request temp tables; if they ever ran against the same pooled
// backend in sequence and the names overlapped, the second would fail on an existing relation. Driving
// both repeatedly against the same pool exercises that path.
func TestRecruitDashboardTempTablesDoNotCollideWithCommandCenter(t *testing.T) {
	org := seedOrg(t)
	seedCandidate(t, org, "secret", "active", true)
	ctx := tenant.WithOrg(context.Background(), org.String())
	svc := recruitmatch.NewService(appDB)
	asOf := time.Date(2026, 7, 20, 0, 0, 0, 0, time.UTC)

	for i := 0; i < 15; i++ {
		if _, err := svc.RecruiterDashboard(ctx, asOf); err != nil {
			t.Fatalf("iteration %d: %v", i, err)
		}
	}
}