fix: keep the usage pool cleanup task alive across lock loss and Redis errors (#28834)

With WEBSOCKET_MANAGER=redis on a multi-node deployment, the usage pool cleanup task could stop permanently for the whole cluster. Nodes that lost the startup lock race gave up for good after three attempts, and the winner died on a single failed renew or on any Redis connection error, releasing the lock with nobody left to take it over. From then on expired entries accumulated in the usage pool until a node restarted, so /api/usage over-reported models in use and every disconnect handler walked an ever-growing pool.

The task now retries lock acquisition forever like the session pool cleanup does, and any error is logged and answered by releasing the lock and returning to acquisition, so a transient failure costs one cleanup cycle and every node stays a takeover candidate. The delete of an emptied model entry is KeyError-guarded because a disconnect handler on another node can remove the same key between the sweep's snapshot and its delete; unguarded, that race was a permanent task killer that needed nothing rarer than a chat finishing while its tab closed.
This commit is contained in:
Classic298 authored and GitHub committed 2026-08-20 12:59:38 -07:00
1 parent b0fdc00452
commit 5586964bb2
1 file changed
+35 -34
+35 -34
View File
@@ -218,47 +218,48 @@ async def periodic_session_pool_cleanup():
async def periodic_usage_pool_cleanup(): async def periodic_usage_pool_cleanup():
max_retries = 2
retry_delay = random.uniform(WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT) retry_delay = random.uniform(WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT)
for attempt in range(max_retries + 1): while True:
if aquire_func(): try:
break if not aquire_func():
else: log.debug('Usage cleanup lock held by another node. Retrying.')
if attempt < max_retries:
log.debug('Cleanup lock already exists. Retry %s after %ss...', attempt + 1, retry_delay)
await asyncio.sleep(retry_delay) await asyncio.sleep(retry_delay)
else: continue
log.warning('Failed to acquire cleanup lock after retries. Skipping cleanup.')
return
log.debug('Running periodic_cleanup') try:
try: while True:
while True: if not renew_func():
if not renew_func(): log.warning('Unable to renew usage cleanup lock. Retrying cleanup ownership.')
log.error('Unable to renew cleanup lock. Exiting usage pool cleanup.') break
raise Exception('Unable to renew usage pool cleanup lock.')
now = int(time.time()) now = int(time.time())
for model_id, connections in list(USAGE_POOL.items()): for model_id, connections in list(USAGE_POOL.items()):
# Creating a list of sids to remove if they have timed out expired_sids = [
expired_sids = [ sid
sid for sid, details in connections.items() if now - details['updated_at'] > TIMEOUT_DURATION for sid, details in connections.items()
] if now - details['updated_at'] > TIMEOUT_DURATION
]
if connections and not expired_sids: if connections and not expired_sids:
continue continue
for sid in expired_sids: for sid in expired_sids:
del connections[sid] del connections[sid]
if not connections: if not connections:
log.debug('Cleaning up model %s from usage pool', model_id) log.debug('Cleaning up model %s from usage pool', model_id)
del USAGE_POOL[model_id] try:
else: del USAGE_POOL[model_id]
USAGE_POOL[model_id] = connections except KeyError:
await asyncio.sleep(TIMEOUT_DURATION) pass
finally: else:
release_func() USAGE_POOL[model_id] = connections
await asyncio.sleep(TIMEOUT_DURATION)
finally:
release_func()
except Exception:
log.exception('Usage pool cleanup failed. Retrying.')
await asyncio.sleep(retry_delay)
app = socketio.ASGIApp( app = socketio.ASGIApp(