refac
This commit is contained in:
1 parent
7d77efe0f1
commit
ba556bd8f0
3 files changed
+34
-21
No files matched your search
@@ -4145,12 +4145,14 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
if delta_count >= delta_chunk_size:
|
if delta_count >= delta_chunk_size:
|
||||||
await flush_pending_delta_data(delta_chunk_size)
|
await flush_pending_delta_data(delta_chunk_size)
|
||||||
|
|
||||||
|
filter_extra_params = {'__body__': form_data, **extra_params} if filter_functions else None
|
||||||
|
|
||||||
async for line in response.body_iterator:
|
async for line in response.body_iterator:
|
||||||
line = line.decode('utf-8', 'replace') if isinstance(line, bytes) else line
|
line = line.decode('utf-8', 'replace') if isinstance(line, bytes) else line
|
||||||
data = line
|
data = line
|
||||||
|
|
||||||
# Skip empty lines
|
# Skip empty lines
|
||||||
if not data.strip():
|
if not data or data.isspace():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# "data:" is the prefix for each event
|
# "data:" is the prefix for each event
|
||||||
@@ -4178,20 +4180,21 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
pass
|
pass
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Remove the prefix
|
# Remove the "data:" prefix
|
||||||
data = data[len('data:') :].strip()
|
data = data[5:].strip()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = json.loads(data)
|
data = json.loads(data)
|
||||||
|
|
||||||
data, _ = await process_filter_functions(
|
if filter_functions:
|
||||||
request=request,
|
data, _ = await process_filter_functions(
|
||||||
filter_context=filter_context,
|
request=request,
|
||||||
filter_functions=filter_functions,
|
filter_context=filter_context,
|
||||||
filter_type='stream',
|
filter_functions=filter_functions,
|
||||||
form_data=data,
|
filter_type='stream',
|
||||||
extra_params={'__body__': form_data, **extra_params},
|
form_data=data,
|
||||||
)
|
extra_params=filter_extra_params,
|
||||||
|
)
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
if 'event' in data and not getattr(request.state, 'direct', False):
|
if 'event' in data and not getattr(request.state, 'direct', False):
|
||||||
@@ -4433,7 +4436,12 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
}
|
}
|
||||||
delta_type = 'tool_call'
|
delta_type = 'tool_call'
|
||||||
|
|
||||||
image_urls = await get_image_urls(delta.get('images', []), request, metadata, user)
|
delta_images = delta.get('images')
|
||||||
|
image_urls = (
|
||||||
|
await get_image_urls(delta_images, request, metadata, user)
|
||||||
|
if delta_images
|
||||||
|
else []
|
||||||
|
)
|
||||||
if image_urls:
|
if image_urls:
|
||||||
image_file_list = [{'type': 'image', 'url': url} for url in image_urls]
|
image_file_list = [{'type': 'image', 'url': url} for url in image_urls]
|
||||||
message_files = image_file_list
|
message_files = image_file_list
|
||||||
|
|||||||
@@ -663,9 +663,9 @@ def strip_empty_content_blocks(messages: list[dict]) -> list[dict]:
|
|||||||
return messages
|
return messages
|
||||||
|
|
||||||
|
|
||||||
def openai_chat_message_template(model: str):
|
def openai_chat_message_template(model: str, message_id: str | None = None):
|
||||||
return {
|
return {
|
||||||
'id': f'{model}-{str(uuid.uuid4())}',
|
'id': message_id if message_id else f'{model}-{str(uuid.uuid4())}',
|
||||||
'created': int(time.time()),
|
'created': int(time.time()),
|
||||||
'model': model,
|
'model': model,
|
||||||
'choices': [{'index': 0, 'logprobs': None, 'finish_reason': None}],
|
'choices': [{'index': 0, 'logprobs': None, 'finish_reason': None}],
|
||||||
@@ -678,8 +678,9 @@ def openai_chat_chunk_message_template(
|
|||||||
reasoning_content: str | None = None,
|
reasoning_content: str | None = None,
|
||||||
tool_calls: list[dict | None] = None,
|
tool_calls: list[dict | None] = None,
|
||||||
usage: dict | None = None,
|
usage: dict | None = None,
|
||||||
|
message_id: str | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
template = openai_chat_message_template(model)
|
template = openai_chat_message_template(model, message_id)
|
||||||
template['object'] = 'chat.completion.chunk'
|
template['object'] = 'chat.completion.chunk'
|
||||||
|
|
||||||
template['choices'][0]['index'] = 0
|
template['choices'][0]['index'] = 0
|
||||||
|
|||||||
@@ -66,6 +66,8 @@ USAGE_COST_KEYS = {
|
|||||||
'completion_cost',
|
'completion_cost',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
USAGE_SUMMABLE_KEYS = USAGE_TOKEN_KEYS | USAGE_COST_KEYS
|
||||||
|
|
||||||
USAGE_DETAIL_KEYS = {
|
USAGE_DETAIL_KEYS = {
|
||||||
'prompt_tokens_details',
|
'prompt_tokens_details',
|
||||||
'completion_tokens_details',
|
'completion_tokens_details',
|
||||||
@@ -115,7 +117,7 @@ def merge_usage(current: dict | None, incoming: dict | None) -> dict:
|
|||||||
|
|
||||||
result = {**current_usage, **incoming_usage}
|
result = {**current_usage, **incoming_usage}
|
||||||
|
|
||||||
for key in USAGE_TOKEN_KEYS | USAGE_COST_KEYS:
|
for key in USAGE_SUMMABLE_KEYS:
|
||||||
if key in current_usage or key in incoming_usage:
|
if key in current_usage or key in incoming_usage:
|
||||||
current_value = current_usage.get(key, 0)
|
current_value = current_usage.get(key, 0)
|
||||||
incoming_value = incoming_usage.get(key, 0)
|
incoming_value = incoming_usage.get(key, 0)
|
||||||
@@ -238,9 +240,10 @@ async def convert_streaming_response_ollama_to_openai(ollama_streaming_response)
|
|||||||
data = json.loads(data)
|
data = json.loads(data)
|
||||||
|
|
||||||
model = data.get('model', 'ollama')
|
model = data.get('model', 'ollama')
|
||||||
message_content = data.get('message', {}).get('content', None)
|
message = data.get('message') or {}
|
||||||
reasoning_content = data.get('message', {}).get('thinking', None)
|
message_content = message.get('content', None)
|
||||||
tool_calls = data.get('message', {}).get('tool_calls', None)
|
reasoning_content = message.get('thinking', None)
|
||||||
|
tool_calls = message.get('tool_calls', None)
|
||||||
openai_tool_calls = None
|
openai_tool_calls = None
|
||||||
|
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
@@ -253,8 +256,9 @@ async def convert_streaming_response_ollama_to_openai(ollama_streaming_response)
|
|||||||
if done:
|
if done:
|
||||||
usage = convert_ollama_usage_to_openai(data)
|
usage = convert_ollama_usage_to_openai(data)
|
||||||
|
|
||||||
data = openai_chat_chunk_message_template(model, message_content, reasoning_content, openai_tool_calls, usage)
|
data = openai_chat_chunk_message_template(
|
||||||
data['id'] = completion_id
|
model, message_content, reasoning_content, openai_tool_calls, usage, message_id=completion_id
|
||||||
|
)
|
||||||
|
|
||||||
# First chunk must carry delta.role (OpenAI spec).
|
# First chunk must carry delta.role (OpenAI spec).
|
||||||
if first:
|
if first:
|
||||||
|
|||||||
Reference in new issue
Block a user