This commit is contained in:
Timothy Jaeryang Baek committed 2026-08-17 00:57:57 -07:00
1 parent 017075a2d7
commit 0b27fa5e87
2 files changed
+22 -45

No files matched your search

+19 -38
View File
@@ -235,6 +235,23 @@ def _resolve_token_threshold(global_threshold: int, global_cap: int, metadata: d
return min(configured_threshold or global_threshold, global_cap) return min(configured_threshold or global_threshold, global_cap)
def _usage_token_count(usage: dict) -> int:
prompt_tokens = int(usage.get('prompt_tokens') or usage.get('prompt_eval_count') or 0)
if not prompt_tokens and (usage.get('prompt_n') is not None or usage.get('cache_n') is not None):
prompt_tokens = int(usage.get('prompt_n') or 0) + int(usage.get('cache_n') or 0)
if not prompt_tokens:
prompt_tokens = int(usage.get('input_tokens') or 0)
completion_tokens = int(
usage.get('completion_tokens')
or usage.get('output_tokens')
or usage.get('eval_count')
or usage.get('predicted_n')
or 0
)
return prompt_tokens + completion_tokens
async def get_chat_context_usage(chat: Any, model_id: str | None = None) -> dict | None: async def get_chat_context_usage(chat: Any, model_id: str | None = None) -> dict | None:
chat_data = chat.chat or {} chat_data = chat.chat or {}
history = chat_data.get('history') or {} history = chat_data.get('history') or {}
@@ -263,25 +280,7 @@ async def get_chat_context_usage(chat: Any, model_id: str | None = None) -> dict
for idx in range(len(messages) - 1, -1, -1): for idx in range(len(messages) - 1, -1, -1):
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage') usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
if isinstance(usage, dict) and ( if isinstance(usage, dict) and (tokens := _usage_token_count(usage)):
tokens := (
int(
usage.get('prompt_tokens')
or usage.get('input_tokens')
or usage.get('prompt_eval_count')
or usage.get('prompt_n')
or 0
)
+ int(
usage.get('completion_tokens')
or usage.get('output_tokens')
or usage.get('eval_count')
or usage.get('predicted_n')
or 0
)
+ int(usage.get('cache_n') or 0)
)
):
tokens += _estimate_messages_tokens(messages[idx + 1 :]) tokens += _estimate_messages_tokens(messages[idx + 1 :])
return _build_context_usage(tokens, threshold) return _build_context_usage(tokens, threshold)
@@ -320,25 +319,7 @@ def _exceeds_token_threshold(messages: list[dict], system_prompt: str, summary:
for idx in range(len(messages) - 1, -1, -1): for idx in range(len(messages) - 1, -1, -1):
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage') usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
if isinstance(usage, dict) and ( if isinstance(usage, dict) and (tokens := _usage_token_count(usage)):
tokens := (
int(
usage.get('prompt_tokens')
or usage.get('input_tokens')
or usage.get('prompt_eval_count')
or usage.get('prompt_n')
or 0
)
+ int(
usage.get('completion_tokens')
or usage.get('output_tokens')
or usage.get('eval_count')
or usage.get('predicted_n')
or 0
)
+ int(usage.get('cache_n') or 0)
)
):
return tokens + _estimate_messages_tokens(messages[idx + 1 :]) > threshold return tokens + _estimate_messages_tokens(messages[idx + 1 :]) > threshold
estimated = _estimate_tokens(system_prompt) + _estimate_tokens(summary or '') + _estimate_messages_tokens(messages) estimated = _estimate_tokens(system_prompt) + _estimate_tokens(summary or '') + _estimate_messages_tokens(messages)
+3 -7
View File
@@ -24,13 +24,9 @@ def normalize_usage(usage: dict) -> dict:
return {} return {}
# Map various field names to standard names # Map various field names to standard names
input_tokens = ( input_tokens = usage.get('input_tokens') or usage.get('prompt_tokens') or usage.get('prompt_eval_count')
usage.get('input_tokens') # Already standard if input_tokens is None:
or usage.get('prompt_tokens') # OpenAI input_tokens = int(usage.get('prompt_n') or 0) + int(usage.get('cache_n') or 0)
or usage.get('prompt_eval_count') # Ollama
or usage.get('prompt_n') # llama.cpp
or 0
)
output_tokens = ( output_tokens = (
usage.get('output_tokens') # Already standard usage.get('output_tokens') # Already standard