refac
This commit is contained in:
1 parent
7e96c53a20
commit
e8f2c123e6
2 files changed
+41
-9
No files matched your search
@@ -259,9 +259,25 @@ async def get_chat_context_usage(chat: Any, model_id: str | None = None) -> dict
|
|||||||
|
|
||||||
for idx in range(len(messages) - 1, -1, -1):
|
for idx in range(len(messages) - 1, -1, -1):
|
||||||
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
|
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
|
||||||
input_tokens = (usage or {}).get('prompt_tokens') or (usage or {}).get('input_tokens')
|
if isinstance(usage, dict) and (
|
||||||
if isinstance(usage, dict) and input_tokens:
|
tokens := (
|
||||||
tokens = int(input_tokens or 0) + int(usage.get('completion_tokens') or usage.get('output_tokens') or 0)
|
int(
|
||||||
|
usage.get('prompt_tokens')
|
||||||
|
or usage.get('input_tokens')
|
||||||
|
or usage.get('prompt_eval_count')
|
||||||
|
or usage.get('prompt_n')
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
+ int(
|
||||||
|
usage.get('completion_tokens')
|
||||||
|
or usage.get('output_tokens')
|
||||||
|
or usage.get('eval_count')
|
||||||
|
or usage.get('predicted_n')
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
+ int(usage.get('cache_n') or 0)
|
||||||
|
)
|
||||||
|
):
|
||||||
tokens += _estimate_messages_tokens(messages[idx + 1 :])
|
tokens += _estimate_messages_tokens(messages[idx + 1 :])
|
||||||
return _build_context_usage(tokens, threshold)
|
return _build_context_usage(tokens, threshold)
|
||||||
|
|
||||||
@@ -300,11 +316,26 @@ def _exceeds_token_threshold(messages: list[dict], system_prompt: str, summary:
|
|||||||
|
|
||||||
for idx in range(len(messages) - 1, -1, -1):
|
for idx in range(len(messages) - 1, -1, -1):
|
||||||
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
|
usage = messages[idx].get('usage') or (messages[idx].get('info') or {}).get('usage')
|
||||||
if isinstance(usage, dict) and (usage.get('prompt_tokens') or usage.get('input_tokens')):
|
if isinstance(usage, dict) and (
|
||||||
total = int(usage.get('prompt_tokens') or usage.get('input_tokens') or 0) + int(
|
tokens := (
|
||||||
usage.get('completion_tokens') or usage.get('output_tokens') or 0
|
int(
|
||||||
|
usage.get('prompt_tokens')
|
||||||
|
or usage.get('input_tokens')
|
||||||
|
or usage.get('prompt_eval_count')
|
||||||
|
or usage.get('prompt_n')
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
+ int(
|
||||||
|
usage.get('completion_tokens')
|
||||||
|
or usage.get('output_tokens')
|
||||||
|
or usage.get('eval_count')
|
||||||
|
or usage.get('predicted_n')
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
+ int(usage.get('cache_n') or 0)
|
||||||
)
|
)
|
||||||
return total + _estimate_messages_tokens(messages[idx + 1 :]) > threshold
|
):
|
||||||
|
return tokens + _estimate_messages_tokens(messages[idx + 1 :]) > threshold
|
||||||
|
|
||||||
estimated = _estimate_tokens(system_prompt) + _estimate_tokens(summary or '') + _estimate_messages_tokens(messages)
|
estimated = _estimate_tokens(system_prompt) + _estimate_tokens(summary or '') + _estimate_messages_tokens(messages)
|
||||||
return estimated > threshold
|
return estimated > threshold
|
||||||
|
|||||||
@@ -2035,7 +2035,7 @@ async def load_messages_from_db(chat_id: str, message_id: str) -> Optional[list[
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
return [
|
return [
|
||||||
{k: v for k, v in msg.items() if k in ('id', 'role', 'content', 'output', 'files', 'contextSummary')}
|
{k: v for k, v in msg.items() if k in ('id', 'role', 'content', 'output', 'files', 'contextSummary', 'usage')}
|
||||||
for msg in db_messages
|
for msg in db_messages
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2095,6 +2095,7 @@ def strip_compaction_fields(messages: list[dict]) -> list[dict]:
|
|||||||
clean = dict(message)
|
clean = dict(message)
|
||||||
clean.pop('contextSummary', None)
|
clean.pop('contextSummary', None)
|
||||||
clean.pop('context_summary', None)
|
clean.pop('context_summary', None)
|
||||||
|
clean.pop('usage', None)
|
||||||
clean.pop('id', None)
|
clean.pop('id', None)
|
||||||
stripped.append(clean)
|
stripped.append(clean)
|
||||||
return stripped
|
return stripped
|
||||||
@@ -2290,7 +2291,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||||||
{
|
{
|
||||||
k: v
|
k: v
|
||||||
for k, v in assistant_message.items()
|
for k, v in assistant_message.items()
|
||||||
if k in ('id', 'role', 'content', 'output', 'files', 'contextSummary')
|
if k in ('id', 'role', 'content', 'output', 'files', 'contextSummary', 'usage')
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user