This commit is contained in:
Timothy Jaeryang Baek committed 2026-07-27 02:49:08 -04:00
1 parent 7d77efe0f1
commit ba556bd8f0
3 files changed
+34 -21

No files matched your search

+20 -12
View File
@@ -4145,12 +4145,14 @@ async def streaming_chat_response_handler(response, ctx):
if delta_count >= delta_chunk_size: if delta_count >= delta_chunk_size:
await flush_pending_delta_data(delta_chunk_size) await flush_pending_delta_data(delta_chunk_size)
filter_extra_params = {'__body__': form_data, **extra_params} if filter_functions else None
async for line in response.body_iterator: async for line in response.body_iterator:
line = line.decode('utf-8', 'replace') if isinstance(line, bytes) else line line = line.decode('utf-8', 'replace') if isinstance(line, bytes) else line
data = line data = line
# Skip empty lines # Skip empty lines
if not data.strip(): if not data or data.isspace():
continue continue
# "data:" is the prefix for each event # "data:" is the prefix for each event
@@ -4178,20 +4180,21 @@ async def streaming_chat_response_handler(response, ctx):
pass pass
continue continue
# Remove the prefix # Remove the "data:" prefix
data = data[len('data:') :].strip() data = data[5:].strip()
try: try:
data = json.loads(data) data = json.loads(data)
data, _ = await process_filter_functions( if filter_functions:
request=request, data, _ = await process_filter_functions(
filter_context=filter_context, request=request,
filter_functions=filter_functions, filter_context=filter_context,
filter_type='stream', filter_functions=filter_functions,
form_data=data, filter_type='stream',
extra_params={'__body__': form_data, **extra_params}, form_data=data,
) extra_params=filter_extra_params,
)
if data: if data:
if 'event' in data and not getattr(request.state, 'direct', False): if 'event' in data and not getattr(request.state, 'direct', False):
@@ -4433,7 +4436,12 @@ async def streaming_chat_response_handler(response, ctx):
} }
delta_type = 'tool_call' delta_type = 'tool_call'
image_urls = await get_image_urls(delta.get('images', []), request, metadata, user) delta_images = delta.get('images')
image_urls = (
await get_image_urls(delta_images, request, metadata, user)
if delta_images
else []
)
if image_urls: if image_urls:
image_file_list = [{'type': 'image', 'url': url} for url in image_urls] image_file_list = [{'type': 'image', 'url': url} for url in image_urls]
message_files = image_file_list message_files = image_file_list
+4 -3
View File
@@ -663,9 +663,9 @@ def strip_empty_content_blocks(messages: list[dict]) -> list[dict]:
return messages return messages
def openai_chat_message_template(model: str): def openai_chat_message_template(model: str, message_id: str | None = None):
return { return {
'id': f'{model}-{str(uuid.uuid4())}', 'id': message_id if message_id else f'{model}-{str(uuid.uuid4())}',
'created': int(time.time()), 'created': int(time.time()),
'model': model, 'model': model,
'choices': [{'index': 0, 'logprobs': None, 'finish_reason': None}], 'choices': [{'index': 0, 'logprobs': None, 'finish_reason': None}],
@@ -678,8 +678,9 @@ def openai_chat_chunk_message_template(
reasoning_content: str | None = None, reasoning_content: str | None = None,
tool_calls: list[dict | None] = None, tool_calls: list[dict | None] = None,
usage: dict | None = None, usage: dict | None = None,
message_id: str | None = None,
) -> dict: ) -> dict:
template = openai_chat_message_template(model) template = openai_chat_message_template(model, message_id)
template['object'] = 'chat.completion.chunk' template['object'] = 'chat.completion.chunk'
template['choices'][0]['index'] = 0 template['choices'][0]['index'] = 0
+10 -6
View File
@@ -66,6 +66,8 @@ USAGE_COST_KEYS = {
'completion_cost', 'completion_cost',
} }
USAGE_SUMMABLE_KEYS = USAGE_TOKEN_KEYS | USAGE_COST_KEYS
USAGE_DETAIL_KEYS = { USAGE_DETAIL_KEYS = {
'prompt_tokens_details', 'prompt_tokens_details',
'completion_tokens_details', 'completion_tokens_details',
@@ -115,7 +117,7 @@ def merge_usage(current: dict | None, incoming: dict | None) -> dict:
result = {**current_usage, **incoming_usage} result = {**current_usage, **incoming_usage}
for key in USAGE_TOKEN_KEYS | USAGE_COST_KEYS: for key in USAGE_SUMMABLE_KEYS:
if key in current_usage or key in incoming_usage: if key in current_usage or key in incoming_usage:
current_value = current_usage.get(key, 0) current_value = current_usage.get(key, 0)
incoming_value = incoming_usage.get(key, 0) incoming_value = incoming_usage.get(key, 0)
@@ -238,9 +240,10 @@ async def convert_streaming_response_ollama_to_openai(ollama_streaming_response)
data = json.loads(data) data = json.loads(data)
model = data.get('model', 'ollama') model = data.get('model', 'ollama')
message_content = data.get('message', {}).get('content', None) message = data.get('message') or {}
reasoning_content = data.get('message', {}).get('thinking', None) message_content = message.get('content', None)
tool_calls = data.get('message', {}).get('tool_calls', None) reasoning_content = message.get('thinking', None)
tool_calls = message.get('tool_calls', None)
openai_tool_calls = None openai_tool_calls = None
if tool_calls: if tool_calls:
@@ -253,8 +256,9 @@ async def convert_streaming_response_ollama_to_openai(ollama_streaming_response)
if done: if done:
usage = convert_ollama_usage_to_openai(data) usage = convert_ollama_usage_to_openai(data)
data = openai_chat_chunk_message_template(model, message_content, reasoning_content, openai_tool_calls, usage) data = openai_chat_chunk_message_template(
data['id'] = completion_id model, message_content, reasoning_content, openai_tool_calls, usage, message_id=completion_id
)
# First chunk must carry delta.role (OpenAI spec). # First chunk must carry delta.role (OpenAI spec).
if first: if first: