//go:build integration package integration import ( "context" "sync" "testing" "github.com/google/uuid" "github.com/missionworkforce/missionmatch/internal/modules/recruiting" ) // WARNING. These tests create organizations, users and job orders. Point MM_APP_DSN and MM_MIGRATOR_DSN // at a scratch or staging database, never at production. // TestConcurrentStatusChangesElectExactlyOneWinner is the race that matters most on this table, because // a status change moves three clocks and writes history. If two transitions both applied from the same // starting state, the row would end in one status while the history claimed two different paths, and // every duration derived from it afterwards would be wrong with no way to detect it. // // Sixteen callers hold the same token and all fire at once. Exactly one may win. func TestConcurrentStatusChangesElectExactlyOneWinner(t *testing.T) { org := seedOrg(t) _, ctx := seedRecruiter(t, org, "Race Recruiter") svc := reqSvc() r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Race Role"}) if err != nil { t.Fatalf("create: %v", err) } id := uuid.MustParse(r.ID) const n = 16 var wg sync.WaitGroup results := make([]error, n) targets := []string{"on_hold", "filled", "closed", "cancelled"} start := make(chan struct{}) for i := 0; i < n; i++ { wg.Add(1) go func(i int) { defer wg.Done() <-start // release them together so the writes genuinely overlap _, results[i] = svc.ChangeStatus(ctx, id, recruiting.ChangeStatusInput{ ExpectedUpdatedAt: r.UpdatedAt, Status: targets[i%len(targets)], }) }(i) } close(start) wg.Wait() won := 0 for _, e := range results { if e == nil { won++ } } if won != 1 { t.Fatalf("%d of %d concurrent transitions succeeded, want exactly 1", won, n) } // And the survivor must be internally consistent: one status, one matching clock state, and a history // with exactly one transition recorded on top of the opening event. d, err := svc.GetRequisition(ctx, id) if err != nil { t.Fatalf("get: %v", err) } if len(d.Events) != 2 { t.Errorf("history has %d events, want 2 (opened plus one winning transition)", len(d.Events)) } if (d.Status == "on_hold") != (d.OnHoldSince != "") { t.Errorf("status %q with on_hold_since=%q breaks the hold invariant", d.Status, d.OnHoldSince) } terminal := d.Status == "filled" || d.Status == "closed" || d.Status == "cancelled" if terminal != (d.ClosedAt != "") { t.Errorf("status %q with closed_at=%q breaks the closed_at invariant", d.Status, d.ClosedAt) } if d.WorkingDays < 0 || d.WorkingDays > d.AgeDays { t.Errorf("clocks are inconsistent after the race: working=%v age=%v", d.WorkingDays, d.AgeDays) } } // TestConcurrentClaimsElectExactlyOneOwner. Two recruiters clicking Claim at the same instant must not // both believe they hold the order. The row lock inside the transaction is what serialises them. func TestConcurrentClaimsElectExactlyOneOwner(t *testing.T) { org := seedOrg(t) _, creator := seedRecruiter(t, org, "Claim Creator") svc := reqSvc() r, err := svc.CreateRequisition(creator, recruiting.CreateRequisitionInput{Title: "Claim Race Role"}) if err != nil { t.Fatalf("create: %v", err) } id := uuid.MustParse(r.ID) if _, err := svc.ReleaseRequisition(creator, id); err != nil { t.Fatalf("release: %v", err) } // Eight DIFFERENT recruiters. Using one user repeatedly would not be a race at all, because a repeat // claim by the holder is a legitimate no op and every call would correctly succeed. const n = 8 ctxs := make([]context.Context, n) names := make([]string, n) for i := 0; i < n; i++ { names[i] = "Claimer " + string(rune('A'+i)) _, ctxs[i] = seedRecruiter(t, org, names[i]) } var wg sync.WaitGroup results := make([]error, n) start := make(chan struct{}) for i := 0; i < n; i++ { wg.Add(1) go func(i int) { defer wg.Done() <-start _, results[i] = svc.ClaimRequisition(ctxs[i], id) }(i) } close(start) wg.Wait() won := 0 for _, e := range results { if e == nil { won++ } } if won != 1 { t.Fatalf("%d of %d concurrent claims succeeded, want exactly 1", won, n) } got, err := svc.GetRequisition(ctxs[0], id) if err != nil { t.Fatalf("get: %v", err) } if got.OwnerUserID == "" { t.Fatal("after eight claims the order has no owner") } found := false for _, nm := range names { if got.OwnerName == nm { found = true } } if !found { t.Errorf("owner is %q, which is not one of the claimants", got.OwnerName) } } // TestConcurrentCreatesAllSucceedAndAreDistinct. Unlike a status change, opening several job orders at // once is legitimate and none of them should block. This is the control: it confirms the locking above // is confined to the row being transitioned rather than serialising the whole table. func TestConcurrentCreatesAllSucceedAndAreDistinct(t *testing.T) { org := seedOrg(t) _, ctx := seedRecruiter(t, org, "Bulk Recruiter") svc := reqSvc() const n = 24 var wg sync.WaitGroup ids := make([]string, n) errs := make([]error, n) start := make(chan struct{}) for i := 0; i < n; i++ { wg.Add(1) go func(i int) { defer wg.Done() <-start r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{ Title: "Bulk Role " + itoaTest(i)}) errs[i] = err if err == nil { ids[i] = r.ID } }(i) } close(start) wg.Wait() seen := map[string]bool{} for i, e := range errs { if e != nil { t.Fatalf("concurrent create %d failed: %v", i, e) } if seen[ids[i]] { t.Fatalf("two creates returned the same id %s", ids[i]) } seen[ids[i]] = true } list, err := svc.ListRequisitions(ctx, "", "", "", "", 200, 0) if err != nil { t.Fatalf("list: %v", err) } if list.Total != n { t.Errorf("after %d concurrent creates the org has %d orders", n, list.Total) } } func itoaTest(n int) string { if n == 0 { return "0" } var b []byte for n > 0 { b = append([]byte{byte('0' + n%10)}, b...) n /= 10 } return string(b) }