// Package tenantpool is the bounded, lazy registry of per tenant connection pools for the database per // tenant model. After auth validates a request to an organization, the router asks this manager for that // tenant's pool; the manager resolves the tenant's database from the control plane directory and its // credential from the secrets port, opens a pool (lazily, once, through database.NewBoundedPool so the // pgvector codecs and the NOBYPASSRLS role assertion run on every tenant pool), verifies it with a Ping, // caches it, and hands back a request scoped tenant.DB plus a release function. // // The design follows the pgx pool per tenant guidance: a BOUNDED number of live pools (LRU evicted), each // small (so MaxPools times MaxConns stays under the server max_connections), a Ping before caching so an // unreachable tenant fails fast with a typed error instead of hanging, a pin or refcount so an in flight // transaction is never closed under it, singleflight so concurrent first callers open exactly one pool, a // nonzero connection lifetime jitter, whole pool idle eviction (distinct from pgx's per connection idle // close), and typed errors that NEVER fall back to another tenant. RLS stays forced inside each tenant // database as defense in depth; the per request org GUC is still set by RunInTenantTx on the tenant pool. package tenantpool import ( "context" "errors" "fmt" "sync" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgconn" "golang.org/x/sync/singleflight" "github.com/missionworkforce/missionmatch/internal/platform/database" "github.com/missionworkforce/missionmatch/internal/platform/directory" "github.com/missionworkforce/missionmatch/internal/platform/secrets" "github.com/missionworkforce/missionmatch/internal/platform/tenant" ) // ErrTenantUnavailable means the tenant exists in the directory but its status does not permit serving // (it is provisioning, migrating, suspended, or offline). It is a fail closed denial, never a fallback. var ErrTenantUnavailable = errors.New("tenantpool: tenant database is not available for serving") // ErrTenantUnreachable means the tenant's database could not be opened or verified (a bad credential, a // network failure, or the wrong role). The caller maps it to a typed 503, never a hang and never another // tenant's data. var ErrTenantUnreachable = errors.New("tenantpool: tenant database is unreachable") // ErrClosed means the manager has been shut down. var ErrClosed = errors.New("tenantpool: manager is closed") // Config bounds the registry. MaxPools caps the live pools (LRU evicted beyond it); MaxConnsPerPool keeps // each pool small; IdleEvict closes a whole pool after this long with no use. Role is the database role // each pool must authenticate as (empty defaults to the NOBYPASSRLS app role); the compliance registry // asserts the compliance role instead. SecretRefSuffix is appended to the directory secret reference // before resolving, so the compliance registry resolves a distinct compliance credential for the same // tenant database (the app and compliance roles never share a connection). type Config struct { MaxPools int MaxConnsPerPool int IdleEvict time.Duration Role string SecretRefSuffix string } // opened is a live tenant pool plus the function that closes it. close is separated from the pool so the // registry logic is unit testable without a real database. type opened struct { db *tenant.DB close func() ref string } type entry struct { o *opened pins int last time.Time org uuid.UUID } // Manager is the per tenant pool registry. lookup and build are fields so the registry mechanics can be // unit tested with fakes; New wires the real directory lookup and the real open and verify path. type Manager struct { lookup func(ctx context.Context, org uuid.UUID) (directory.TenantConn, error) build func(ctx context.Context, tc directory.TenantConn) (*opened, error) maxPools int idle time.Duration now func() time.Time mu sync.Mutex pools map[uuid.UUID]*entry sf singleflight.Group closed bool stop chan struct{} } // New builds a manager over the control plane directory (read with the app pool) and the secrets // resolver. It starts the idle sweep when IdleEvict is positive. func New(dir *directory.Directory, sec secrets.Resolver, cfg Config) *Manager { mp := cfg.MaxPools if mp < 1 { mp = 1 } role := cfg.Role if role == "" { role = "missionmatch_app" } m := &Manager{ lookup: dir.Lookup, maxPools: mp, idle: cfg.IdleEvict, now: time.Now, pools: map[uuid.UUID]*entry{}, stop: make(chan struct{}), } m.build = defaultBuild(sec, int32(cfg.MaxConnsPerPool), role, cfg.SecretRefSuffix) if m.idle > 0 { go m.sweepLoop() } return m } // Acquire returns the tenant's pool (opening it lazily and once) plus a release function the caller MUST // call when the request is done, so the pool is unpinned and may be evicted when idle. A request with an // unknown organization gets directory.ErrUnknownTenant; an inactive or unreachable tenant gets a typed // error; never a different tenant's pool. func (m *Manager) Acquire(ctx context.Context, org uuid.UUID) (*tenant.DB, func(), error) { m.mu.Lock() if m.closed { m.mu.Unlock() return nil, nil, ErrClosed } if e, ok := m.pools[org]; ok { e.pins++ e.last = m.now() db := e.o.db m.mu.Unlock() return db, m.releaser(e), nil } m.mu.Unlock() // Cold path: singleflight so concurrent first callers open exactly one pool for this tenant. v, err, _ := m.sf.Do(org.String(), func() (any, error) { m.mu.Lock() if e, ok := m.pools[org]; ok { // opened by a racing caller while we waited m.mu.Unlock() return e, nil } closed := m.closed m.mu.Unlock() if closed { return nil, ErrClosed } tc, err := m.lookup(ctx, org) if err != nil { return nil, err // directory.ErrUnknownTenant propagates unchanged } if tc.Status != directory.StatusActive { return nil, fmt.Errorf("%w: status %q", ErrTenantUnavailable, tc.Status) } o, err := m.build(ctx, tc) if err != nil { return nil, err } m.mu.Lock() if m.closed { // shut down while we were opening m.mu.Unlock() o.close() return nil, ErrClosed } e := &entry{o: o, last: m.now(), org: org} m.pools[org] = e victims := m.evictLocked(org) m.mu.Unlock() closeAll(victims) return e, nil }) if err != nil { return nil, nil, err } e := v.(*entry) m.mu.Lock() if m.closed { m.mu.Unlock() return nil, nil, ErrClosed } cur, ok := m.pools[org] if !ok || cur != e { // Evicted in the small window between open and pin (rare, the new entry is most recently used so // eviction targets others first). Retry the whole acquire. m.mu.Unlock() return m.Acquire(ctx, org) } e.pins++ e.last = m.now() db := e.o.db m.mu.Unlock() return db, m.releaser(e), nil } // releaser returns an idempotent unpin for one acquire. func (m *Manager) releaser(e *entry) func() { var once sync.Once return func() { once.Do(func() { m.mu.Lock() if e.pins > 0 { e.pins-- } e.last = m.now() m.mu.Unlock() }) } } // evictLocked closes the least recently used UNPINNED pools while over the cap. A pinned pool (a live // request) is never closed, and the just inserted pool (protect) is never closed even though it is not // pinned yet, otherwise the caller would find it gone and retry forever. If nothing else is evictable the // registry stays temporarily over cap. Caller holds m.mu and MUST close the returned victims outside the // lock. func (m *Manager) evictLocked(protect uuid.UUID) []*entry { var victims []*entry for len(m.pools) > m.maxPools { var lru *entry for org, e := range m.pools { if e.pins > 0 || org == protect { continue } if lru == nil || e.last.Before(lru.last) { lru = e } } if lru == nil { break // everything else is pinned or protected, accept temporary over cap } delete(m.pools, lru.org) victims = append(victims, lru) } return victims } // sweepOnce closes whole pools idle past IdleEvict (separate from pgx's per connection idle close, which // only shrinks a pool toward MinConns, never frees the pool object). func (m *Manager) sweepOnce() { now := m.now() m.mu.Lock() var victims []*entry for org, e := range m.pools { if e.pins == 0 && now.Sub(e.last) >= m.idle { delete(m.pools, org) victims = append(victims, e) } } m.mu.Unlock() closeAll(victims) } func (m *Manager) sweepLoop() { interval := m.idle / 2 if interval < time.Second { interval = time.Second } t := time.NewTicker(interval) defer t.Stop() for { select { case <-m.stop: return case <-t.C: m.sweepOnce() } } } // Len reports the number of live tenant pools (for tests and metrics). func (m *Manager) Len() int { m.mu.Lock() defer m.mu.Unlock() return len(m.pools) } // Close shuts the manager down and closes every live pool. It is idempotent. func (m *Manager) Close() { m.mu.Lock() if m.closed { m.mu.Unlock() return } m.closed = true close(m.stop) victims := make([]*entry, 0, len(m.pools)) for org, e := range m.pools { victims = append(victims, e) delete(m.pools, org) } m.mu.Unlock() closeAll(victims) } // closeAll closes evicted pools outside the registry lock (Close blocks until the pool drains, so it must // never run under m.mu). An evicted pool is unpinned, so its drain is fast. func closeAll(es []*entry) { for _, e := range es { e.o.close() } } // defaultBuild resolves the tenant credential and opens and verifies the pool. On an authentication // failure (the credential rotation window) it forgets the cached secret and retries once with a fresh // resolve, per the AWS rotation guidance, before returning a typed unreachable error. func defaultBuild(sec secrets.Resolver, maxConns int32, role, refSuffix string) func(context.Context, directory.TenantConn) (*opened, error) { return func(ctx context.Context, tc directory.TenantConn) (*opened, error) { ref := tc.SecretRef + refSuffix dsn, err := sec.ResolveDSN(ctx, ref) if err != nil { return nil, fmt.Errorf("%w: resolve credential: %v", ErrTenantUnreachable, err) } o, err := openVerify(ctx, dsn, maxConns, ref, role) if err != nil && isAuthError(err) { sec.Forget(ref) if dsn2, e2 := sec.ResolveDSN(ctx, ref); e2 == nil { o, err = openVerify(ctx, dsn2, maxConns, ref, role) } } if err != nil { return nil, fmt.Errorf("%w: %v", ErrTenantUnreachable, err) } return o, nil } } // openVerify opens a bounded pool, Pings it under a deadline (pgxpool.New returns BEFORE connecting, so an // unreachable database looks open until first use), and asserts the expected NOBYPASSRLS role (the app // role, or the compliance role for the eeo registry). Any failure closes the pool and returns the error // so a dead pool is never cached. func openVerify(ctx context.Context, dsn string, maxConns int32, ref, role string) (*opened, error) { pool, err := database.NewBoundedPool(ctx, dsn, maxConns) if err != nil { return nil, err } verifyCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if err := pool.Ping(verifyCtx); err != nil { pool.Close() return nil, err } if err := database.AssertRole(verifyCtx, pool, role); err != nil { pool.Close() return nil, err } return &opened{db: tenant.New(pool), close: pool.Close, ref: ref}, nil } // isAuthError reports a Postgres authentication failure (invalid password or invalid authorization), the // rotation window signal that warrants a secret refresh and retry. func isAuthError(err error) bool { var pg *pgconn.PgError if errors.As(err, &pg) { return pg.Code == "28P01" || pg.Code == "28000" } return false }