fix: reduce TTFT by caching model lookups in chat completion (#20886)
fix: reduce TTFT by caching model lookups in chat completion Skip expensive get_all_models() calls when models are already cached in app.state. This significantly reduces Time To First Token (TTFT) for chat completions and embeddings requests. Previously, every request called get_all_models() which fetches model lists from all configured backends. Now we check the cache first and only call get_all_models() on cache miss. Affected endpoints: - openai: generate_chat_completion, embeddings - ollama: embed, embeddings Fixes #20069 Co-authored-by: Michael <42099345+mickeytheseal@users.noreply.github.com>
This commit is contained in:
1 parent
a4281f6a7f
commit
efe5416f83
2 files changed
+23
-9
No files matched your search
@@ -1027,14 +1027,17 @@ async def embed(
|
||||
log.info(f"generate_ollama_batch_embeddings {form_data}")
|
||||
|
||||
if url_idx is None:
|
||||
await get_all_models(request, user=user)
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
|
||||
model = form_data.model
|
||||
|
||||
if ":" not in model:
|
||||
model = f"{model}:latest"
|
||||
|
||||
# Check if model is already in app state cache to avoid expensive get_all_models() call
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
if not models or model not in models:
|
||||
await get_all_models(request, user=user)
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
|
||||
if model in models:
|
||||
url_idx = random.choice(models[model]["urls"])
|
||||
else:
|
||||
@@ -1112,14 +1115,17 @@ async def embeddings(
|
||||
log.info(f"generate_ollama_embeddings {form_data}")
|
||||
|
||||
if url_idx is None:
|
||||
await get_all_models(request, user=user)
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
|
||||
model = form_data.model
|
||||
|
||||
if ":" not in model:
|
||||
model = f"{model}:latest"
|
||||
|
||||
# Check if model is already in app state cache to avoid expensive get_all_models() call
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
if not models or model not in models:
|
||||
await get_all_models(request, user=user)
|
||||
models = request.app.state.OLLAMA_MODELS
|
||||
|
||||
if model in models:
|
||||
url_idx = random.choice(models[model]["urls"])
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user