refac
This commit is contained in:
Timothy Jaeryang Baek
@@ -76,6 +76,7 @@ from open_webui.socket.main import (
|
||||
get_event_call,
|
||||
get_event_emitter,
|
||||
)
|
||||
from open_webui.tasks import clear_response_stream, save_response_stream
|
||||
from open_webui.utils.access_control import has_connection_access, has_permission
|
||||
from open_webui.utils.access_control.files import get_owner_accessible_folder_files
|
||||
from open_webui.utils.access_control.folders import has_folder_access
|
||||
@@ -499,7 +500,22 @@ def handle_responses_streaming_event(
|
||||
item = data.get('item', {})
|
||||
if item:
|
||||
new_output = list(current_output)
|
||||
new_output.append(item)
|
||||
output_index = data.get('output_index', len(new_output))
|
||||
existing_index = next(
|
||||
(
|
||||
idx
|
||||
for idx, existing in enumerate(new_output)
|
||||
if (item.get('id') and existing.get('id') == item.get('id'))
|
||||
or (item.get('call_id') and existing.get('call_id') == item.get('call_id'))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing_index is not None:
|
||||
new_output[existing_index] = item
|
||||
elif 0 <= output_index < len(new_output):
|
||||
new_output.insert(output_index, item)
|
||||
else:
|
||||
new_output.append(item)
|
||||
return new_output, None
|
||||
return current_output, None
|
||||
|
||||
@@ -4033,6 +4049,7 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
async def response_handler(response, events):
|
||||
filter_context = FilterContext()
|
||||
tag_scan_positions = {}
|
||||
response_stream_task_id = metadata.get('task_id') or metadata.get('message_id')
|
||||
|
||||
def tag_output_handler(content_type, tags, output):
|
||||
"""
|
||||
@@ -4440,38 +4457,108 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
)
|
||||
last_delta_data = None
|
||||
last_delta_type = None
|
||||
last_delta_key = None
|
||||
|
||||
def response_stream_content(stream_output: list | None = None):
|
||||
return ''.join(content_parts) or get_output_text(
|
||||
stream_output if stream_output is not None else full_output()
|
||||
)
|
||||
|
||||
async def save_current_response_stream(stream_output: list | None = None):
|
||||
if not chat_id or not metadata.get('message_id'):
|
||||
return
|
||||
|
||||
current_stream_output = stream_output if stream_output is not None else full_output()
|
||||
await save_response_stream(
|
||||
request.app.state.redis,
|
||||
response_stream_task_id,
|
||||
chat_id,
|
||||
metadata.get('message_id'),
|
||||
response_stream_content(current_stream_output),
|
||||
current_stream_output,
|
||||
)
|
||||
|
||||
def get_response_delta_key(delta_data: dict):
|
||||
event_type = delta_data.get('type', '')
|
||||
if not event_type.startswith('response.') or not event_type.endswith('.delta'):
|
||||
return None
|
||||
return (
|
||||
event_type,
|
||||
delta_data.get('item_id'),
|
||||
delta_data.get('output_index'),
|
||||
delta_data.get('content_index'),
|
||||
delta_data.get('summary_index'),
|
||||
)
|
||||
|
||||
async def flush_pending_delta_data(threshold: int = 0):
|
||||
nonlocal delta_count
|
||||
nonlocal last_delta_data
|
||||
nonlocal last_delta_type
|
||||
nonlocal last_delta_key
|
||||
|
||||
if delta_count >= threshold and last_delta_data:
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'chat:completion',
|
||||
'type': 'response:completion',
|
||||
'data': last_delta_data,
|
||||
}
|
||||
)
|
||||
await save_current_response_stream()
|
||||
delta_count = 0
|
||||
last_delta_data = None
|
||||
last_delta_type = None
|
||||
last_delta_key = None
|
||||
|
||||
async def queue_pending_delta_data(delta_data: dict, delta_type: str):
|
||||
nonlocal delta_count
|
||||
nonlocal last_delta_data
|
||||
nonlocal last_delta_type
|
||||
nonlocal last_delta_key
|
||||
|
||||
if last_delta_type and last_delta_type != delta_type:
|
||||
await flush_pending_delta_data()
|
||||
delta_key = get_response_delta_key(delta_data)
|
||||
if (
|
||||
last_delta_data
|
||||
and last_delta_key == delta_key
|
||||
and isinstance(last_delta_data.get('delta'), str)
|
||||
and isinstance(delta_data.get('delta'), str)
|
||||
):
|
||||
last_delta_data['delta'] += delta_data['delta']
|
||||
delta_count += 1
|
||||
else:
|
||||
if last_delta_data and (last_delta_type != delta_type or last_delta_key != delta_key):
|
||||
await flush_pending_delta_data()
|
||||
|
||||
delta_count += 1
|
||||
last_delta_data = delta_data
|
||||
last_delta_type = delta_type
|
||||
delta_count += 1
|
||||
last_delta_data = delta_data
|
||||
last_delta_type = delta_type
|
||||
last_delta_key = delta_key
|
||||
|
||||
if delta_count >= delta_chunk_size:
|
||||
await flush_pending_delta_data(delta_chunk_size)
|
||||
|
||||
async def emit_response_completion_event(response_event: dict, stream_output: list | None = None):
|
||||
if prior_output and isinstance(response_event.get('output_index'), int):
|
||||
response_event = {
|
||||
**response_event,
|
||||
'output_index': response_event['output_index'] + len(prior_output),
|
||||
}
|
||||
|
||||
if response_event.get('type', '').endswith('.delta'):
|
||||
await queue_pending_delta_data(
|
||||
response_event,
|
||||
response_event.get('type', 'response.delta'),
|
||||
)
|
||||
return
|
||||
|
||||
await flush_pending_delta_data()
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'response:completion',
|
||||
'data': response_event,
|
||||
}
|
||||
)
|
||||
await save_current_response_stream(stream_output)
|
||||
|
||||
filter_extra_params = {'__body__': form_data, **extra_params} if filter_functions else None
|
||||
|
||||
async for line in response.body_iterator:
|
||||
@@ -4587,13 +4674,6 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
}
|
||||
)
|
||||
|
||||
processed_data = {
|
||||
'output': full_output(),
|
||||
}
|
||||
|
||||
# print(data)
|
||||
# print(processed_data)
|
||||
|
||||
# Merge any metadata (usage, etc.)
|
||||
# Strip 'done' — response.completed emits
|
||||
# it but we may still need to execute tool
|
||||
@@ -4610,22 +4690,21 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
usage = merge_usage(usage, response_metadata['usage'])
|
||||
response_metadata['usage'] = usage
|
||||
|
||||
processed_data.update(response_metadata)
|
||||
processed_data.pop('done', None)
|
||||
if response_metadata.get('error'):
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'chat:completion',
|
||||
'data': {'error': response_metadata['error']},
|
||||
}
|
||||
)
|
||||
|
||||
if response_event_is_delta:
|
||||
response_delta_type = response_event_type.split('.')[1]
|
||||
await queue_pending_delta_data(
|
||||
processed_data,
|
||||
'tool_call'
|
||||
if response_delta_type == 'function_call_arguments'
|
||||
else 'content',
|
||||
)
|
||||
else:
|
||||
await emit_response_completion_event(data)
|
||||
|
||||
if response_metadata and response_metadata.get('usage'):
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'chat:completion',
|
||||
'data': processed_data,
|
||||
'data': {'usage': usage},
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -4723,6 +4802,7 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
# Add the new tool call
|
||||
delta_tool_call.setdefault('function', {})
|
||||
delta_tool_call['function'].setdefault('name', '')
|
||||
delta_tool_call['id'] = delta_tool_call.get('id') or output_id('fc')
|
||||
delta_arguments = delta_tool_call['function'].get('arguments')
|
||||
if not isinstance(delta_arguments, str):
|
||||
delta_tool_call['function']['arguments'] = (
|
||||
@@ -4754,27 +4834,73 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
delta_arguments
|
||||
)
|
||||
|
||||
# Emit pending tool calls in real-time
|
||||
# Emit pending tool calls in real-time as Responses events.
|
||||
if response_tool_calls:
|
||||
# Build pending function_call output items for display
|
||||
pending_fc_items = []
|
||||
output_by_call_id = {
|
||||
item.get('call_id'): (idx, item)
|
||||
for idx, item in enumerate(output)
|
||||
if item.get('type') == 'function_call'
|
||||
}
|
||||
|
||||
for tc in response_tool_calls:
|
||||
call_id = tc.get('id', '')
|
||||
call_id = tc.get('id') or output_id('fc')
|
||||
tc['id'] = call_id
|
||||
func = tc.get('function', {})
|
||||
pending_fc_items.append(
|
||||
{
|
||||
if call_id in output_by_call_id:
|
||||
output_index, item = output_by_call_id[call_id]
|
||||
item['name'] = func.get('name', item.get('name', ''))
|
||||
item['arguments'] = func.get('arguments', item.get('arguments', ''))
|
||||
item['status'] = 'in_progress'
|
||||
else:
|
||||
output_index = len(output)
|
||||
item = {
|
||||
'type': 'function_call',
|
||||
'id': call_id or output_id('fc'),
|
||||
'id': call_id,
|
||||
'call_id': call_id,
|
||||
'name': func.get('name', ''),
|
||||
'arguments': func.get('arguments', '{}'),
|
||||
'arguments': '',
|
||||
'status': 'in_progress',
|
||||
}
|
||||
)
|
||||
output.append(item)
|
||||
output_by_call_id[call_id] = (output_index, item)
|
||||
await emit_response_completion_event(
|
||||
{
|
||||
'type': 'response.output_item.added',
|
||||
'output_index': output_index,
|
||||
'item': item.copy(),
|
||||
}
|
||||
)
|
||||
item['arguments'] = func.get('arguments', '')
|
||||
|
||||
data = {
|
||||
'output': full_output() + pending_fc_items,
|
||||
}
|
||||
for delta_tool_call in delta_tool_calls:
|
||||
tool_call_index = delta_tool_call.get('index')
|
||||
current_response_tool_call = next(
|
||||
(
|
||||
tc
|
||||
for tc in response_tool_calls
|
||||
if tc.get('index') == tool_call_index
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not current_response_tool_call:
|
||||
continue
|
||||
call_id = current_response_tool_call.get('id')
|
||||
output_index, _ = output_by_call_id.get(call_id, (len(output) - 1, {}))
|
||||
delta_arguments = delta_tool_call.get('function', {}).get('arguments')
|
||||
if delta_arguments is not None:
|
||||
if not isinstance(delta_arguments, str):
|
||||
delta_arguments = JSONCodec.dumps(delta_arguments)
|
||||
await emit_response_completion_event(
|
||||
{
|
||||
'type': 'response.function_call_arguments.delta',
|
||||
'item_id': call_id,
|
||||
'output_index': output_index,
|
||||
'delta': delta_arguments,
|
||||
}
|
||||
)
|
||||
|
||||
await save_current_response_stream()
|
||||
data = None
|
||||
delta_type = 'tool_call'
|
||||
|
||||
delta_images = delta.get('images')
|
||||
@@ -4878,20 +5004,26 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
}
|
||||
]
|
||||
|
||||
reasoning_index = output.index(reasoning_item)
|
||||
data = {
|
||||
'output': full_output(),
|
||||
'type': 'response.reasoning_text.delta',
|
||||
'item_id': reasoning_item.get('id'),
|
||||
'output_index': reasoning_index,
|
||||
'content_index': max(
|
||||
len(reasoning_item.get('content', [])) - 1,
|
||||
0,
|
||||
),
|
||||
'delta': reasoning_content,
|
||||
}
|
||||
delta_type = 'content'
|
||||
delta_type = 'response.reasoning_text.delta'
|
||||
|
||||
if reasoning_detail_items:
|
||||
merge_streamed_reasoning_details(
|
||||
reasoning_item.setdefault('reasoning_details', []),
|
||||
reasoning_detail_items,
|
||||
)
|
||||
data = {
|
||||
'output': full_output(),
|
||||
}
|
||||
delta_type = 'content'
|
||||
await save_current_response_stream()
|
||||
data = None
|
||||
|
||||
if value:
|
||||
if (
|
||||
@@ -5037,29 +5169,27 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
if end:
|
||||
break
|
||||
|
||||
if ENABLE_REALTIME_CHAT_SAVE and save_to_chat:
|
||||
current_output = full_output()
|
||||
# Save message in the database
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{
|
||||
'output': current_output,
|
||||
},
|
||||
)
|
||||
data = {
|
||||
'output': current_output,
|
||||
}
|
||||
delta_type = 'content'
|
||||
else:
|
||||
data = {
|
||||
'output': full_output(),
|
||||
}
|
||||
delta_type = 'content'
|
||||
target_index = len(output) - 1
|
||||
target_item = output[target_index] if target_index >= 0 else {}
|
||||
target_content = target_item.get('content', [])
|
||||
content_index = max(len(target_content) - 1, 0)
|
||||
delta_event_type = (
|
||||
'response.reasoning_text.delta'
|
||||
if target_item.get('type') == 'reasoning'
|
||||
else 'response.output_text.delta'
|
||||
)
|
||||
data = {
|
||||
'type': delta_event_type,
|
||||
'item_id': target_item.get('id'),
|
||||
'output_index': target_index,
|
||||
'content_index': content_index,
|
||||
'delta': value,
|
||||
}
|
||||
delta_type = delta_event_type
|
||||
|
||||
if delta:
|
||||
if delta and data:
|
||||
await queue_pending_delta_data(data, delta_type)
|
||||
else:
|
||||
elif data:
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'chat:completion',
|
||||
@@ -5108,6 +5238,29 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
reasoning_item['status'] = 'completed'
|
||||
|
||||
if response_tool_calls:
|
||||
for tc in response_tool_calls:
|
||||
call_id = tc.get('id', '')
|
||||
arguments = tc.get('function', {}).get('arguments', '{}')
|
||||
for output_index, item in enumerate(output):
|
||||
if item.get('type') == 'function_call' and item.get('call_id') == call_id:
|
||||
item['arguments'] = arguments
|
||||
item['status'] = 'completed'
|
||||
await emit_response_completion_event(
|
||||
{
|
||||
'type': 'response.function_call_arguments.done',
|
||||
'item_id': item.get('id'),
|
||||
'output_index': output_index,
|
||||
'arguments': arguments,
|
||||
}
|
||||
)
|
||||
await emit_response_completion_event(
|
||||
{
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': output_index,
|
||||
'item': item.copy(),
|
||||
}
|
||||
)
|
||||
break
|
||||
tool_calls.append(_split_tool_calls(response_tool_calls))
|
||||
|
||||
# Responses API path: extract function_call items from output
|
||||
@@ -5809,31 +5962,22 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
}
|
||||
|
||||
if save_to_chat:
|
||||
if not ENABLE_REALTIME_CHAT_SAVE:
|
||||
# Save message in the database
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{
|
||||
'done': True,
|
||||
'output': current_output,
|
||||
**({'usage': usage} if usage else {}),
|
||||
},
|
||||
)
|
||||
elif usage:
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{'done': True, 'usage': usage},
|
||||
)
|
||||
else:
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{'done': True},
|
||||
)
|
||||
# Save final output once. The delta path keeps in-progress
|
||||
# state in response_streams instead of writing tokens to DB.
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{
|
||||
'done': True,
|
||||
'output': current_output,
|
||||
**({'usage': usage} if usage else {}),
|
||||
},
|
||||
)
|
||||
|
||||
await publish_chat_finished_event(request, user, metadata, title, ''.join(content_parts), current_output)
|
||||
await clear_response_stream(request.app.state.redis, response_stream_task_id)
|
||||
await publish_chat_finished_event(
|
||||
request, user, metadata, title, ''.join(content_parts), current_output
|
||||
)
|
||||
|
||||
await event_emitter(
|
||||
{
|
||||
@@ -5865,22 +6009,15 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
async def save_cancelled_state():
|
||||
await event_emitter({'type': 'chat:tasks:cancel'})
|
||||
if save_to_chat:
|
||||
if not ENABLE_REALTIME_CHAT_SAVE:
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{
|
||||
'done': True,
|
||||
'output': full_output(),
|
||||
},
|
||||
)
|
||||
else:
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{'done': True},
|
||||
touch=False,
|
||||
)
|
||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
||||
metadata['chat_id'],
|
||||
metadata['message_id'],
|
||||
{
|
||||
'done': True,
|
||||
'output': full_output(),
|
||||
},
|
||||
)
|
||||
await clear_response_stream(request.app.state.redis, response_stream_task_id)
|
||||
|
||||
try:
|
||||
await asyncio.shield(save_cancelled_state())
|
||||
|
||||
Reference in New Issue
Block a user