perf: halve Redis round trips on model resolution and socket pools (#27225)
When WEBSOCKET_MANAGER=redis, app.state.MODELS and the socket session/
usage pools are Redis-backed dicts, so every membership test and
getitem is a network round trip:
- generate_chat_completion checked `model_id not in models` (HEXISTS)
and then read `models[model_id]` (HGET) on every chat completion.
A single .get() now serves both, with the same not-found error.
- The direct-connection branch spread the pool with `{**MODELS, ...}`,
which iterates keys() then fetches each value — HKEYS plus one HGET
per model. dict(MODELS.items()) issues a single HGETALL instead.
- get_user_ids_from_room called SESSION_POOL.get(sid) twice per
session (once to filter, once for the value); the usage handler
checked membership then fetched the same key. Both now do one
lookup.
In non-Redis mode these are plain dicts and behavior is identical.
Claude-Session: https://claude.ai/code/session_01MHg5zs1VBjvRWQ54qHpfYD
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
429f2df50c
commit
656a848043
2 files changed
+15
-12
No files matched your search
@@ -286,14 +286,14 @@ def get_session_ids_from_room(room):
|
|||||||
def get_user_ids_from_room(room):
|
def get_user_ids_from_room(room):
|
||||||
active_session_ids = get_session_ids_from_room(room)
|
active_session_ids = get_session_ids_from_room(room)
|
||||||
|
|
||||||
|
# Single pool lookup per session (each .get is a Redis round trip
|
||||||
|
# when the session pool is Redis-backed).
|
||||||
active_user_ids = list(
|
active_user_ids = list(
|
||||||
set(
|
{
|
||||||
[
|
entry['id']
|
||||||
SESSION_POOL.get(session_id)['id']
|
for entry in (SESSION_POOL.get(session_id) for session_id in active_session_ids)
|
||||||
for session_id in active_session_ids
|
if entry is not None
|
||||||
if SESSION_POOL.get(session_id) is not None
|
}
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
return active_user_ids
|
return active_user_ids
|
||||||
|
|
||||||
@@ -357,7 +357,7 @@ async def usage(sid, data):
|
|||||||
|
|
||||||
# Store the new usage data and task
|
# Store the new usage data and task
|
||||||
USAGE_POOL[model_id] = {
|
USAGE_POOL[model_id] = {
|
||||||
**(USAGE_POOL[model_id] if model_id in USAGE_POOL else {}),
|
**(USAGE_POOL.get(model_id) or {}),
|
||||||
sid: {'updated_at': current_time},
|
sid: {'updated_at': current_time},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -179,8 +179,10 @@ async def generate_chat_completion(
|
|||||||
# Merge the direct connection model into server models so that
|
# Merge the direct connection model into server models so that
|
||||||
# task functions (title, tags, etc.) can resolve a server-side
|
# task functions (title, tags, etc.) can resolve a server-side
|
||||||
# task model while still having the direct model available.
|
# task model while still having the direct model available.
|
||||||
|
# dict(...items()) is one HGETALL on a Redis-backed pool; ``{**pool}``
|
||||||
|
# would issue HKEYS plus one HGET per model.
|
||||||
models = {
|
models = {
|
||||||
**request.app.state.MODELS,
|
**dict(request.app.state.MODELS.items()),
|
||||||
request.state.model['id']: request.state.model,
|
request.state.model['id']: request.state.model,
|
||||||
}
|
}
|
||||||
log.debug(f'direct connection to model: {request.state.model["id"]}')
|
log.debug(f'direct connection to model: {request.state.model["id"]}')
|
||||||
@@ -188,11 +190,12 @@ async def generate_chat_completion(
|
|||||||
models = request.app.state.MODELS
|
models = request.app.state.MODELS
|
||||||
|
|
||||||
model_id = form_data['model']
|
model_id = form_data['model']
|
||||||
if model_id not in models:
|
# Single lookup — membership check plus getitem would be two Redis
|
||||||
|
# round trips on a Redis-backed model pool.
|
||||||
|
model = models.get(model_id)
|
||||||
|
if model is None:
|
||||||
raise Exception('Model not found')
|
raise Exception('Model not found')
|
||||||
|
|
||||||
model = models[model_id]
|
|
||||||
|
|
||||||
if getattr(request.state, 'direct', False) and model_id == getattr(request.state, 'model', {}).get('id'):
|
if getattr(request.state, 'direct', False) and model_id == getattr(request.state, 'model', {}).get('id'):
|
||||||
return await generate_direct_chat_completion(request, form_data, user=user, models=models)
|
return await generate_direct_chat_completion(request, form_data, user=user, models=models)
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in new issue
Block a user