Janady13/missionmatch / internal/integration/requisition_stress_test.go

1//go:build integration 2 3package integration 4 5import ( 6 "context" 7 "sync" 8 "testing" 9 10 "github.com/google/uuid" 11 12 "github.com/missionworkforce/missionmatch/internal/modules/recruiting" 13) 14 15// WARNING. These tests create organizations, users and job orders. Point MM_APP_DSN and MM_MIGRATOR_DSN 16// at a scratch or staging database, never at production. 17 18// TestConcurrentStatusChangesElectExactlyOneWinner is the race that matters most on this table, because 19// a status change moves three clocks and writes history. If two transitions both applied from the same 20// starting state, the row would end in one status while the history claimed two different paths, and 21// every duration derived from it afterwards would be wrong with no way to detect it. 22// 23// Sixteen callers hold the same token and all fire at once. Exactly one may win. 24func TestConcurrentStatusChangesElectExactlyOneWinner(t *testing.T) { 25 org := seedOrg(t) 26 _, ctx := seedRecruiter(t, org, "Race Recruiter") 27 svc := reqSvc() 28 29 r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{Title: "Race Role"}) 30 if err != nil { 31 t.Fatalf("create: %v", err) 32 } 33 id := uuid.MustParse(r.ID) 34 35 const n = 16 36 var wg sync.WaitGroup 37 results := make([]error, n) 38 targets := []string{"on_hold", "filled", "closed", "cancelled"} 39 start := make(chan struct{}) 40 41 for i := 0; i < n; i++ { 42 wg.Add(1) 43 go func(i int) { 44 defer wg.Done() 45 <-start // release them together so the writes genuinely overlap 46 _, results[i] = svc.ChangeStatus(ctx, id, recruiting.ChangeStatusInput{ 47 ExpectedUpdatedAt: r.UpdatedAt, 48 Status: targets[i%len(targets)], 49 }) 50 }(i) 51 } 52 close(start) 53 wg.Wait() 54 55 won := 0 56 for _, e := range results { 57 if e == nil { 58 won++ 59 } 60 } 61 if won != 1 { 62 t.Fatalf("%d of %d concurrent transitions succeeded, want exactly 1", won, n) 63 } 64 65 // And the survivor must be internally consistent: one status, one matching clock state, and a history 66 // with exactly one transition recorded on top of the opening event. 67 d, err := svc.GetRequisition(ctx, id) 68 if err != nil { 69 t.Fatalf("get: %v", err) 70 } 71 if len(d.Events) != 2 { 72 t.Errorf("history has %d events, want 2 (opened plus one winning transition)", len(d.Events)) 73 } 74 if (d.Status == "on_hold") != (d.OnHoldSince != "") { 75 t.Errorf("status %q with on_hold_since=%q breaks the hold invariant", d.Status, d.OnHoldSince) 76 } 77 terminal := d.Status == "filled" || d.Status == "closed" || d.Status == "cancelled" 78 if terminal != (d.ClosedAt != "") { 79 t.Errorf("status %q with closed_at=%q breaks the closed_at invariant", d.Status, d.ClosedAt) 80 } 81 if d.WorkingDays < 0 || d.WorkingDays > d.AgeDays { 82 t.Errorf("clocks are inconsistent after the race: working=%v age=%v", d.WorkingDays, d.AgeDays) 83 } 84} 85 86// TestConcurrentClaimsElectExactlyOneOwner. Two recruiters clicking Claim at the same instant must not 87// both believe they hold the order. The row lock inside the transaction is what serialises them. 88func TestConcurrentClaimsElectExactlyOneOwner(t *testing.T) { 89 org := seedOrg(t) 90 _, creator := seedRecruiter(t, org, "Claim Creator") 91 svc := reqSvc() 92 93 r, err := svc.CreateRequisition(creator, recruiting.CreateRequisitionInput{Title: "Claim Race Role"}) 94 if err != nil { 95 t.Fatalf("create: %v", err) 96 } 97 id := uuid.MustParse(r.ID) 98 if _, err := svc.ReleaseRequisition(creator, id); err != nil { 99 t.Fatalf("release: %v", err) 100 } 101 102 // Eight DIFFERENT recruiters. Using one user repeatedly would not be a race at all, because a repeat 103 // claim by the holder is a legitimate no op and every call would correctly succeed. 104 const n = 8 105 ctxs := make([]context.Context, n) 106 names := make([]string, n) 107 for i := 0; i < n; i++ { 108 names[i] = "Claimer " + string(rune('A'+i)) 109 _, ctxs[i] = seedRecruiter(t, org, names[i]) 110 } 111 112 var wg sync.WaitGroup 113 results := make([]error, n) 114 start := make(chan struct{}) 115 for i := 0; i < n; i++ { 116 wg.Add(1) 117 go func(i int) { 118 defer wg.Done() 119 <-start 120 _, results[i] = svc.ClaimRequisition(ctxs[i], id) 121 }(i) 122 } 123 close(start) 124 wg.Wait() 125 126 won := 0 127 for _, e := range results { 128 if e == nil { 129 won++ 130 } 131 } 132 if won != 1 { 133 t.Fatalf("%d of %d concurrent claims succeeded, want exactly 1", won, n) 134 } 135 136 got, err := svc.GetRequisition(ctxs[0], id) 137 if err != nil { 138 t.Fatalf("get: %v", err) 139 } 140 if got.OwnerUserID == "" { 141 t.Fatal("after eight claims the order has no owner") 142 } 143 found := false 144 for _, nm := range names { 145 if got.OwnerName == nm { 146 found = true 147 } 148 } 149 if !found { 150 t.Errorf("owner is %q, which is not one of the claimants", got.OwnerName) 151 } 152} 153 154// TestConcurrentCreatesAllSucceedAndAreDistinct. Unlike a status change, opening several job orders at 155// once is legitimate and none of them should block. This is the control: it confirms the locking above 156// is confined to the row being transitioned rather than serialising the whole table. 157func TestConcurrentCreatesAllSucceedAndAreDistinct(t *testing.T) { 158 org := seedOrg(t) 159 _, ctx := seedRecruiter(t, org, "Bulk Recruiter") 160 svc := reqSvc() 161 162 const n = 24 163 var wg sync.WaitGroup 164 ids := make([]string, n) 165 errs := make([]error, n) 166 start := make(chan struct{}) 167 for i := 0; i < n; i++ { 168 wg.Add(1) 169 go func(i int) { 170 defer wg.Done() 171 <-start 172 r, err := svc.CreateRequisition(ctx, recruiting.CreateRequisitionInput{ 173 Title: "Bulk Role " + itoaTest(i)}) 174 errs[i] = err 175 if err == nil { 176 ids[i] = r.ID 177 } 178 }(i) 179 } 180 close(start) 181 wg.Wait() 182 183 seen := map[string]bool{} 184 for i, e := range errs { 185 if e != nil { 186 t.Fatalf("concurrent create %d failed: %v", i, e) 187 } 188 if seen[ids[i]] { 189 t.Fatalf("two creates returned the same id %s", ids[i]) 190 } 191 seen[ids[i]] = true 192 } 193 list, err := svc.ListRequisitions(ctx, "", "", "", "", 200, 0) 194 if err != nil { 195 t.Fatalf("list: %v", err) 196 } 197 if list.Total != n { 198 t.Errorf("after %d concurrent creates the org has %d orders", n, list.Total) 199 } 200} 201 202func itoaTest(n int) string { 203 if n == 0 { 204 return "0" 205 } 206 var b []byte 207 for n > 0 { 208 b = append([]byte{byte('0' + n%10)}, b...) 209 n /= 10 210 } 211 return string(b) 212} 213