bhetherman/open-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-06-29 13:30:28 -05:00
parent 22a44e67a8
commit 0443ab3a61
8 changed files with 741 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/routers/chats.py
-11
View File
@@ -37,7 +37,6 @@ from open_webui.utils.access_control import filter_allowed_access_grants, has_pe
from open_webui.utils.access_control.folders import has_folder_access
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.context_compaction import compact_chat_branch
from open_webui.utils.middleware import serialize_output
from open_webui.utils.misc import get_message_list
from open_webui.utils.models import get_all_models
from pydantic import BaseModel
@@ -1221,16 +1220,6 @@ async def update_chat_by_id(
form_data.chat.get('history'),
)
# Re-derive content from output for assistant messages so that frontend
# edits to output items are reflected in content. Only when output
# actually changed — otherwise content set independently of output
# (e.g. a `replace` event or an outlet filter footer) would be reverted.
existing_messages = (chat.chat.get('history') or {}).get('messages') or {}
for msg_id, msg in updated_chat.get('history', {}).get('messages', {}).items():
if isinstance(msg, dict) and msg.get('role') == 'assistant' and msg.get('output'):
if msg.get('output') != existing_messages.get(msg_id, {}).get('output'):
msg['content'] = serialize_output(msg['output'])
chat = await Chats.update_chat_by_id(id, updated_chat, db=db)
# Reconcile chat_message rows without inferring deletes from missing IDs.
backend/open_webui/utils/middleware.py
+9 -231
View File
@@ -2,7 +2,6 @@ import ast
import asyncio
import base64
import copy
import html
import inspect
import json
import logging
@@ -378,210 +377,6 @@ def get_citation_source_from_tool_result(
]
def split_content_and_whitespace(content):
content_stripped = content.rstrip()
original_whitespace = content[len(content_stripped) :] if len(content) > len(content_stripped) else ''
return content_stripped, original_whitespace
def is_opening_code_block(content):
backtick_segments = content.split('```')
# Even number of segments means the last backticks are opening a new block
return len(backtick_segments) > 1 and len(backtick_segments) % 2 == 0
_OPENAI_TOOL_DISPLAY_NAMES = {
'web_search_call': 'Web Search',
'file_search_call': 'File Search',
'computer_call': 'Computer Use',
}
def _render_openai_tool_call_handler(item: dict, done: bool) -> str:
"""Render an OpenAI Responses API server-side tool item as a <details> block.
Handles web_search_call, file_search_call, and computer_call items whose
schemas are defined in the openai-python SDK (generated from OpenAPI spec).
"""
item_type = item.get('type', '')
call_id = item.get('id', '')
display_name = _OPENAI_TOOL_DISPLAY_NAMES.get(item_type, item_type)
# Build a short summary of what the tool did
summary = ''
if item_type == 'web_search_call':
action = item.get('action', {})
if isinstance(action, dict):
atype = action.get('type', '')
if atype == 'search':
queries = action.get('queries') or []
query = action.get('query', '')
summary = (
f'Search: {", ".join(str(q) for q in queries)}'
if queries
else (f'Search: {query}' if query else '')
)
elif atype == 'open_page':
summary = f'Open page: {action.get("url", "")}' if action.get('url') else ''
elif atype == 'find_in_page':
summary = f'Find in page: {action.get("pattern", "")}' if action.get('pattern') else ''
elif item_type == 'file_search_call':
queries = item.get('queries', [])
if queries:
summary = f'Queries: {", ".join(str(q) for q in queries)}'
elif item_type == 'computer_call':
action = item.get('action')
actions = item.get('actions')
if isinstance(action, dict):
summary = f'Action: {action.get("type", "unknown")}'
elif isinstance(actions, list) and actions:
summary = f'Actions: {", ".join(a.get("type", "?") for a in actions if isinstance(a, dict))}'
escaped_name = html.escape(display_name)
if done:
return f'<details type="tool_calls" done="true" id="{call_id}" name="{escaped_name}" arguments="">\n<summary>Tool Executed</summary>\n{html.escape(summary)}\n</details>\n'
return f'<details type="tool_calls" done="false" id="{call_id}" name="{escaped_name}" arguments="">\n<summary>Executing...</summary>\n</details>\n'
def serialize_output(output: list) -> str:
"""
Convert OR-aligned output items to HTML for display.
For LLM consumption, use convert_output_to_messages() instead.
"""
parts: list[str] = []
# First pass: collect function_call_output items by call_id for lookup
tool_outputs = {}
for item in output:
if item.get('type') == 'function_call_output':
tool_outputs[item.get('call_id')] = item
# Second pass: render items in order
for idx, item in enumerate(output):
item_type = item.get('type', '')
if item_type == 'message':
for content_part in item.get('content', []):
if 'text' in content_part:
text = content_part.get('text', '').strip()
if text:
parts.append(text)
elif item_type == 'function_call':
call_id = item.get('call_id', '')
name = item.get('name', '')
arguments = item.get('arguments', '')
result_item = tool_outputs.get(call_id)
if result_item:
result_parts: list[str] = []
for result_output in result_item.get('output', []):
if 'text' in result_output:
output_text = result_output.get('text', '')
result_parts.append(str(output_text) if not isinstance(output_text, str) else output_text)
result_text = ''.join(result_parts)
files = result_item.get('files')
embeds = result_item.get('embeds', '')
parts.append(
f'<details type="tool_calls" done="true" id="{call_id}" name="{name}" arguments="{html.escape(json.dumps(arguments))}" files="{html.escape(json.dumps(files)) if files else ""}" embeds="{html.escape(json.dumps(embeds))}">\n<summary>Tool Executed</summary>\n{html.escape(json.dumps(result_text, ensure_ascii=False))}\n</details>'
)
else:
parts.append(
f'<details type="tool_calls" done="false" id="{call_id}" name="{name}" arguments="{html.escape(json.dumps(arguments))}">\n<summary>Executing...</summary>\n</details>'
)
elif item_type == 'function_call_output':
# Already handled inline with function_call above
pass
elif item_type in _OPENAI_TOOL_DISPLAY_NAMES:
status = item.get('status', 'in_progress')
done = status in ('completed', 'failed', 'incomplete') or idx != len(output) - 1
parts.append(_render_openai_tool_call_handler(item, done).rstrip('\n'))
elif item_type == 'reasoning':
reasoning_parts: list[str] = []
# Check for 'summary' (new structure) or 'content' (legacy/fallback)
source_list = item.get('summary', []) or item.get('content', [])
for content_part in source_list:
if 'text' in content_part:
reasoning_parts.append(content_part.get('text', ''))
elif 'summary' in content_part: # Handle potential nested logic if any
pass
reasoning_content = ''.join(reasoning_parts).strip()
duration = item.get('duration')
status = item.get('status', 'in_progress')
# Infer completion: if this reasoning item is NOT the last item,
# render as done (a subsequent item means reasoning is complete)
is_last_item = idx == len(output) - 1
display = html.escape(
'\n'.join(
(f'> {line}' if not line.startswith('>') else line) for line in reasoning_content.splitlines()
)
)
if status == 'completed' or duration is not None or not is_last_item:
parts.append(
f'<details type="reasoning" done="true" duration="{duration or 0}">\n<summary>Thought for {duration or 0} seconds</summary>\n{display}\n</details>'
)
else:
parts.append(
f'<details type="reasoning" done="false">\n<summary>Thinking…</summary>\n{display}\n</details>'
)
elif item_type == 'open_webui:code_interpreter':
# Code interpreter needs to inspect/mutate prior accumulated content
# to strip trailing unclosed code fences — materialize only here.
content = '\n'.join(parts)
content_stripped, original_whitespace = split_content_and_whitespace(content)
if is_opening_code_block(content_stripped):
content = content_stripped.rstrip('`').rstrip() + original_whitespace
else:
content = content_stripped + original_whitespace
# Re-split back into parts list after mutation
parts = [content] if content else []
# Render the code_interpreter item as a <details> block
# so the frontend Collapsible renders "Analyzing..."/"Analyzed".
code = item.get('code', '').strip()
lang = item.get('lang', 'python')
status = item.get('status', 'in_progress')
duration = item.get('duration')
is_last_item = idx == len(output) - 1
# Build inner content: code block
display = ''
if code:
display = f'```{lang}\n{code}\n```'
# Build output attribute as HTML-escaped JSON for CodeBlock.svelte
ci_output = item.get('output')
output_attr = ''
if ci_output:
if isinstance(ci_output, dict):
output_json = json.dumps(ci_output, ensure_ascii=False)
else:
output_json = json.dumps({'result': str(ci_output)}, ensure_ascii=False)
output_attr = f' output="{html.escape(output_json)}"'
if status == 'completed' or duration is not None or not is_last_item:
parts.append(
f'<details type="code_interpreter" done="true" duration="{duration or 0}"{output_attr}>\n<summary>Analyzed</summary>\n{display}\n</details>'
)
else:
parts.append(
f'<details type="code_interpreter" done="false"{output_attr}>\n<summary>Analyzing…</summary>\n{display}\n</details>'
)
return '\n'.join(parts).strip()
def deep_merge(target, source):
"""
Merge source into target recursively (returning new structure).
@@ -3127,7 +2922,6 @@ def update_assistant_message_from_stream(assistant_message, raw):
output, meta = handle_responses_streaming_event(data, assistant_message.get('output', []))
if output:
assistant_message['output'] = output
assistant_message['content'] = serialize_output(output)
if meta and meta.get('usage'):
assistant_message['usage'] = merge_usage(assistant_message.get('usage'), meta['usage'])
continue
@@ -3527,18 +3321,16 @@ async def outlet_filter_handler(ctx):
'output'
)
if content_changed or output_changed:
# If output was modified, re-derive content from it
new_content = message.get('content', original_message.get('content', ''))
if output_changed:
new_content = serialize_output(message['output'])
message_update = {
'originalContent': original_message.get('content'),
**({'output': message['output']} if output_changed else {}),
}
if content_changed:
message_update['content'] = message.get('content', '')
await Chats.upsert_message_to_chat_by_id_and_message_id(
chat_id,
outlet_message_id,
{
'content': new_content,
'originalContent': original_message.get('content'),
**({'output': message['output']} if output_changed else {}),
},
message_update,
)
if event_emitter:
@@ -3722,7 +3514,7 @@ async def non_streaming_chat_response_handler(response, ctx):
if ENABLE_API_OUTLET_FILTERS and (content or output):
usage = normalize_usage(response_data.get('usage', {}) or {})
ctx['assistant_message'] = {
'content': content or serialize_output(output),
**({'content': content} if content else {}),
**({'output': output} if output else {}),
**({'usage': usage} if usage else {}),
}
@@ -4242,7 +4034,6 @@ async def streaming_chat_response_handler(response, ctx):
processed_data = {
'output': full_output(),
'content': serialize_output(full_output()),
}
# print(data)
@@ -4412,7 +4203,6 @@ async def streaming_chat_response_handler(response, ctx):
)
data = {
'content': serialize_output(full_output() + pending_fc_items),
'output': full_output() + pending_fc_items,
}
delta_type = 'tool_call'
@@ -4474,7 +4264,6 @@ async def streaming_chat_response_handler(response, ctx):
]
data = {
'content': serialize_output(full_output()),
'output': full_output(),
}
delta_type = 'content'
@@ -4637,18 +4426,15 @@ async def streaming_chat_response_handler(response, ctx):
metadata['chat_id'],
metadata['message_id'],
{
'content': serialize_output(full_output()),
'output': full_output(),
},
)
data = {
'content': serialize_output(full_output()),
'output': full_output(),
}
delta_type = 'content'
else:
data = {
'content': serialize_output(full_output()),
'output': full_output(),
}
delta_type = 'content'
@@ -4791,7 +4577,6 @@ async def streaming_chat_response_handler(response, ctx):
{
'type': 'chat:completion',
'data': {
'content': serialize_output(full_output()),
'output': full_output(),
},
}
@@ -4948,7 +4733,7 @@ async def streaming_chat_response_handler(response, ctx):
display_files = []
for file_item in result.get('files', []):
if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'):
# LLM-only: add as input_image part (invisible to serialize_output)
# LLM-only: add as input_image part, not frontend display output.
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
else:
# Frontend display (MCP images, audio, etc.)
@@ -5054,7 +4839,6 @@ async def streaming_chat_response_handler(response, ctx):
{
'type': 'chat:completion',
'data': {
'content': serialize_output(output),
'output': frontend_output,
},
}
@@ -5178,7 +4962,6 @@ async def streaming_chat_response_handler(response, ctx):
{
'type': 'chat:completion',
'data': {
'content': serialize_output(output),
'output': output,
},
}
@@ -5305,7 +5088,6 @@ async def streaming_chat_response_handler(response, ctx):
{
'type': 'chat:completion',
'data': {
'content': serialize_output(output),
'output': output,
},
}
@@ -5352,7 +5134,6 @@ async def streaming_chat_response_handler(response, ctx):
)
data = {
'done': True,
'content': serialize_output(output),
'output': output,
'title': title,
**({'usage': usage} if usage else {}),
@@ -5366,7 +5147,6 @@ async def streaming_chat_response_handler(response, ctx):
metadata['message_id'],
{
'done': True,
'content': serialize_output(output),
'output': output,
**({'usage': usage} if usage else {}),
},
@@ -5409,7 +5189,6 @@ async def streaming_chat_response_handler(response, ctx):
)
ctx['assistant_message'] = {
'content': serialize_output(output),
'output': output,
**({'usage': usage} if usage else {}),
}
@@ -5437,7 +5216,6 @@ async def streaming_chat_response_handler(response, ctx):
metadata['message_id'],
{
'done': True,
'content': serialize_output(output),
'output': output,
},
)
src/lib/components/chat/Chat.svelte
+101 -114
View File
@@ -68,6 +68,7 @@
displayFileHandler
} from '$lib/utils';
import { AudioQueue } from '$lib/utils/audio';
import { getOutputText } from './Messages/structuredOutput';
import {
archiveChatById,
@@ -1248,13 +1249,50 @@
$: onHistoryChange(history);
const dispatchCallOverlayAudio = (message, final = false) => {
if (!$showCallOverlay) {
return;
}
const messageContentParts = getMessageContentParts(
getOutputText(message?.output) || removeAllDetails(message?.content ?? ''),
$config?.audio?.tts?.split_on ?? 'punctuation'
);
if (!final) {
messageContentParts.pop();
}
const nextContentPart = messageContentParts.at(-1) ?? '';
if (!nextContentPart || (!final && nextContentPart === message.lastSentence)) {
return;
}
if (!final) {
message.lastSentence = nextContentPart;
}
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: {
id: message.id,
content: nextContentPart
}
})
);
};
const getContents = () => {
const messages = history ? createMessagesList(history, history.currentId) : [];
let contents = [];
messages.forEach((message) => {
if (message?.role !== 'user' && message?.content) {
if (message?.role !== 'user') {
const messageContent = getOutputText(message?.output) || removeAllDetails(message?.content ?? '');
if (!messageContent.trim()) {
return;
}
const { codeBlocks: codeBlocks, htmlGroups: htmlGroups } = getCodeBlockContents(
message.content
messageContent
);
if (htmlGroups && htmlGroups.length > 0) {
@@ -1923,6 +1961,7 @@
// Store raw OR-aligned output items from backend
if (output) {
message.output = output;
dispatchCallOverlayAudio(message);
}
if (error) {
@@ -1933,10 +1972,11 @@
message.sources = sources;
}
if (choices) {
if (choices && !output) {
if (choices[0]?.message?.content) {
// Non-stream response
message.content += choices[0]?.message?.content;
dispatchCallOverlayAudio(message);
} else {
// Stream response
let value = choices[0]?.delta?.content ?? '';
@@ -1948,67 +1988,19 @@
if (navigator.vibrate && ($settings?.hapticFeedback ?? false)) {
navigator.vibrate(5);
}
// Emit chat event for TTS (only when call overlay is active)
if ($showCallOverlay) {
const messageContentParts = getMessageContentParts(
removeAllDetails(message.content),
$config?.audio?.tts?.split_on ?? 'punctuation'
);
messageContentParts.pop();
// dispatch only last sentence and make sure it hasn't been dispatched before
if (
messageContentParts.length > 0 &&
messageContentParts[messageContentParts.length - 1] !== message.lastSentence
) {
message.lastSentence = messageContentParts[messageContentParts.length - 1];
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: {
id: message.id,
content: messageContentParts[messageContentParts.length - 1]
}
})
);
}
}
dispatchCallOverlayAudio(message);
}
}
}
if (content) {
if (content && !output) {
// REALTIME_CHAT_SAVE is disabled
message.content = content;
if (navigator.vibrate && ($settings?.hapticFeedback ?? false)) {
navigator.vibrate(5);
}
// Emit chat event for TTS (only when call overlay is active)
if ($showCallOverlay) {
const messageContentParts = getMessageContentParts(
removeAllDetails(message.content),
$config?.audio?.tts?.split_on ?? 'punctuation'
);
messageContentParts.pop();
// dispatch only last sentence and make sure it hasn't been dispatched before
if (
messageContentParts.length > 0 &&
messageContentParts[messageContentParts.length - 1] !== message.lastSentence
) {
message.lastSentence = messageContentParts[messageContentParts.length - 1];
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: {
id: message.id,
content: messageContentParts[messageContentParts.length - 1]
}
})
);
}
}
dispatchCallOverlayAudio(message);
}
if (selected_model_id) {
@@ -2024,9 +2016,10 @@
if (done) {
message.done = true;
const visibleContent = getOutputText(message?.output) || removeAllDetails(message?.content ?? '');
if ($settings.responseAutoCopy) {
copyToClipboard(message.content);
copyToClipboard(visibleContent);
}
if ($settings.responseAutoPlayback && !$showCallOverlay) {
@@ -2035,25 +2028,12 @@
}
// Emit chat event for TTS (only when call overlay is active)
if ($showCallOverlay) {
let lastMessageContentPart =
getMessageContentParts(
removeAllDetails(message.content),
$config?.audio?.tts?.split_on ?? 'punctuation'
)?.at(-1) ?? '';
if (lastMessageContentPart) {
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: { id: message.id, content: lastMessageContentPart }
})
);
}
}
dispatchCallOverlayAudio(message, true);
eventTarget.dispatchEvent(
new CustomEvent('chat:finish', {
detail: {
id: message.id,
content: message.content
content: visibleContent
}
})
);
@@ -2480,53 +2460,60 @@
true;
// Always include system prompt — backend extracts it and prepends to DB messages.
// Only temp chats need conversation messages (persisted chats load from DB).
let messages = [
params?.system || $settings.system
? { role: 'system', content: `${params?.system ?? $settings?.system ?? ''}` }
: undefined
].filter(Boolean);
let messages: any[] = [
params?.system || $settings.system
? { role: 'system', content: `${params?.system ?? $settings?.system ?? ''}` }
: undefined
].filter(Boolean);
if ($temporaryChatEnabled) {
messages = [
...messages,
..._messages.map((message) => ({
...message,
content: processDetails(message.content),
...(message.output ? { output: message.output } : {})
}))
].filter((message) => message);
if ($temporaryChatEnabled) {
messages = [
...messages,
..._messages.map((message) => ({
...message,
...(message.output && message.role === 'assistant'
? { output: message.output }
: { content: processDetails(message.content) })
}))
].filter((message) => message);
messages = messages
.map((message, idx, arr) => {
const imageFiles = (message?.files ?? []).filter(
(file) => file.type === 'image' || (file?.content_type ?? '').startsWith('image/')
messages = messages
.map((message) => {
const imageFiles = (message?.files ?? []).filter(
(file) => file.type === 'image' || (file?.content_type ?? '').startsWith('image/')
);
if (message.output && message.role === 'assistant') {
return { role: message.role, output: message.output };
}
if (message.role === 'user' && imageFiles.length > 0) {
return {
role: message.role,
content: [
{
type: 'text',
text: message?.merged?.content ?? message.content
},
...imageFiles.map((file) => ({
type: 'image_url',
image_url: {
url: file.url
}
}))
]
};
}
return {
role: message.role,
content: message?.merged?.content ?? message.content
};
})
.filter(
(message) => message?.role === 'user' || message?.content?.trim() || message?.output?.length
);
return {
role: message.role,
...(message.output ? { output: message.output } : {}),
...(message.role === 'user' && imageFiles.length > 0
? {
content: [
{
type: 'text',
text: message?.merged?.content ?? message.content
},
...imageFiles.map((file) => ({
type: 'image_url',
image_url: {
url: file.url
}
}))
]
}
: {
content: message?.merged?.content ?? message.content
})
};
})
.filter((message) => message?.role === 'user' || message?.content?.trim());
}
}
const toolIds = [];
const toolServerIds = [];
src/lib/components/chat/Messages.svelte
+21 -18
View File
@@ -173,7 +173,7 @@
messages: messages
});
// Refresh local message content from backend (e.g. re-derived via serialize_output)
// Keep local plain-content edits aligned with the saved chat response.
if (res?.chat?.history?.messages) {
for (const [id, msg] of Object.entries(res.chat.history.messages)) {
if (history.messages[id] && (msg as any).content) {
@@ -385,16 +385,16 @@
const message = history.messages[messageId];
const parentId = message.parentId;
const responseMessage = {
...message,
id: responseMessageId,
parentId: parentId,
childrenIds: [],
files: undefined,
content: content,
output: output ?? undefined,
timestamp: Math.floor(Date.now() / 1000) // Unix epoch
};
const responseMessage = {
...message,
id: responseMessageId,
parentId: parentId,
childrenIds: [],
files: undefined,
content: output !== undefined ? '' : content,
...(output !== undefined ? { output } : {}),
timestamp: Math.floor(Date.now() / 1000) // Unix epoch
};
history.messages[responseMessageId] = responseMessage;
history.currentId = responseMessageId;
@@ -408,13 +408,16 @@
}
await updateChat();
} else {
// Edit response message
history.messages[messageId].originalContent = history.messages[messageId].content;
history.messages[messageId].content = content;
if (output !== undefined) {
history.messages[messageId].output = output;
}
} else {
// Edit response message
if (content !== undefined) {
history.messages[messageId].originalContent = history.messages[messageId].content;
history.messages[messageId].content = content;
}
if (output !== undefined) {
history.messages[messageId].output = output;
history.messages[messageId].content = '';
}
await updateChat();
}
}
src/lib/components/chat/Messages/ContentRenderer.svelte
+58 -27
View File
@@ -3,6 +3,7 @@
const i18n = getContext('i18n');
import Markdown from './Markdown.svelte';
import StructuredOutputRenderer from './StructuredOutputRenderer.svelte';
import {
artifactCode,
chatId,
@@ -68,6 +69,8 @@
export let id;
export let content;
/** @type {import('./structuredOutput').OutputItem[]} */
export let output = [];
export let history;
export let messageId;
@@ -118,6 +121,39 @@
sourceIds = [...new Set(result)];
};
/** @param {string} messageContent */
const formatMessageContent = (messageContent) =>
model?.info?.meta?.capabilities?.citations == false
? replaceOutsideCode(messageContent, (segment) =>
segment.replace(/\s*(\[(?:\d+(?:#[^,\]\s]+)?(?:,\s*\d+(?:#[^,\]\s]+)?)*)\])+/g, '')
)
: messageContent;
const markdownUpdateHandler = /** @type {any} */ (async (
/** @type {{ lang?: string; text?: string }} */ token
) => {
const { lang = '', text: code = '' } = token;
if (
($settings?.detectArtifacts ?? true) &&
(['html', 'svg'].includes(lang) || (lang === 'xml' && code.includes('svg'))) &&
!$mobile &&
$chatId
) {
await tick();
showArtifacts.set(true);
showControls.set(true);
}
});
const previewHandler = /** @type {any} */ (async (/** @type {string} */ value) => {
console.log('Preview', value);
await artifactCode.set(/** @type {any} */ (value));
await showControls.set(true);
await showArtifacts.set(true);
await showEmbeds.set(false);
});
const updateButtonPosition = (event) => {
const buttonsContainerElement = document.getElementById(`floating-buttons-${id}`);
if (
@@ -225,14 +261,28 @@
</script>
<div bind:this={contentContainerElement}>
{#if $settings?.renderMarkdownInAssistantMessages ?? true}
{#if output?.length}
<StructuredOutputRenderer
{id}
{output}
{model}
{preview}
{done}
{editCodeBlock}
{topPadding}
{sourceIds}
renderMarkdown={$settings?.renderMarkdownInAssistantMessages ?? true}
formatMessageContent={formatMessageContent}
{onSourceClick}
{onTaskClick}
{onSave}
onUpdate={markdownUpdateHandler}
onPreview={previewHandler}
/>
{:else if $settings?.renderMarkdownInAssistantMessages ?? true}
<Markdown
{id}
content={model?.info?.meta?.capabilities?.citations == false
? replaceOutsideCode(content, (segment) =>
segment.replace(/\s*(\[(?:\d+(?:#[^,\]\s]+)?(?:,\s*\d+(?:#[^,\]\s]+)?)*)\])+/g, '')
)
: content}
content={formatMessageContent(content)}
{model}
{save}
{preview}
@@ -243,27 +293,8 @@
{onSourceClick}
{onTaskClick}
{onSave}
onUpdate={async (token) => {
const { lang, text: code } = token;
if (
($settings?.detectArtifacts ?? true) &&
(['html', 'svg'].includes(lang) || (lang === 'xml' && code.includes('svg'))) &&
!$mobile &&
$chatId
) {
await tick();
showArtifacts.set(true);
showControls.set(true);
}
}}
onPreview={async (value) => {
console.log('Preview', value);
await artifactCode.set(value);
await showControls.set(true);
await showArtifacts.set(true);
await showEmbeds.set(false);
}}
onUpdate={markdownUpdateHandler}
onPreview={previewHandler}
/>
{:else}
{@const extracted = extractDetailsBlocks(content)}
src/lib/components/chat/Messages/ResponseMessage.svelte
+42 -24
View File
@@ -64,11 +64,17 @@
import StatusHistory from './ResponseMessage/StatusHistory.svelte';
import FullHeightIframe from '$lib/components/common/FullHeightIframe.svelte';
import OutputEditView from './OutputEditView.svelte';
import {
getOutputText,
replaceOutputMessageText,
type OutputItem
} from './structuredOutput';
interface MessageType {
id: string;
model: string;
content: string;
output?: OutputItem[];
files?: { type: string; url: string }[];
timestamp: number;
role: string;
@@ -127,7 +133,11 @@
if (source) {
// Fast path: O(1) check on the fields that change most often (content during streaming, done at end)
// Avoids 2x O(n) JSON.stringify calls that are always true during streaming anyway
if (message.content !== source.content || message.done !== source.done) {
if (
message.content !== source.content ||
message.done !== source.done ||
message.output?.length !== source.output?.length
) {
message = structuredClone(source);
} else if (!equal(message, source)) {
// Slow path: full comparison for infrequent changes (sources, annotations, status, etc.)
@@ -175,6 +185,8 @@
(model?.info?.meta?.capabilities?.status_updates ?? true) &&
statusEntries.length > 0 &&
!(statusEntries.at(-1)?.hidden ?? false);
$: visibleResponseContent = getOutputText(message.output) || removeAllDetails(message.content ?? '');
$: hasResponseContent = Boolean((message.content ?? '').trim() || message.output?.length);
let edit = false;
let editedContent = '';
@@ -226,7 +238,8 @@
: $config?.audio?.tts?.voice);
const speak = async () => {
if (!(message?.content ?? '').trim().length) {
const content = visibleResponseContent;
if (!content.trim().length) {
toast.info($i18n.t('No content to speak'));
return;
}
@@ -236,7 +249,6 @@
const { signal } = speakAbort;
speaking = true;
const content = removeAllDetails(message.content);
if ($config.audio.tts.engine === '') {
let voices = [];
@@ -370,17 +382,6 @@
return restoredContent;
}
/** Extract plain text from output items for immediate display after edit.
* NOT a serialize_output port — just grabs text parts. Backend re-serializes
* the full rich content (with <details> blocks) on save. */
function extractTextFromOutput(output: any[]): string {
return output
.filter((item) => item.type === 'message')
.flatMap((item) => (item.content ?? []).map((p: any) => p.text ?? ''))
.join('\n')
.trim();
}
const editMessageHandler = async () => {
edit = true;
@@ -407,9 +408,7 @@
const editMessageConfirmHandler = async () => {
if (editedOutput) {
// Structured edit: keep original rich content for immediate display;
// backend will re-derive content from output on save.
editMessage(message.id, { content: message.content, output: editedOutput }, false);
editMessage(message.id, { output: editedOutput }, false);
} else {
// Legacy text edit
const messageContent = postprocessAfterEditing(editedContent ?? '');
@@ -425,7 +424,7 @@
const saveAsCopyHandler = async () => {
if (editedOutput) {
editMessage(message.id, { content: message.content, output: editedOutput });
editMessage(message.id, { output: editedOutput });
} else {
const messageContent = postprocessAfterEditing(editedContent ?? '');
editMessage(message.id, { content: messageContent });
@@ -826,14 +825,15 @@
class="w-full flex flex-col relative {edit ? 'hidden' : ''}"
id="response-content-container"
>
{#if message.content === '' && !message.done && !message.error && !hasVisibleStatus}
{#if !hasResponseContent && !message.done && !message.error && !hasVisibleStatus}
<Skeleton />
{:else if message.content && message.error !== true}
{:else if hasResponseContent && message.error !== true}
<!-- always show message contents even if there's an error -->
<!-- unless message.error === true which is legacy error handling, where the error message is stored in message.content -->
<ContentRenderer
id={`${chatId}-${message.id}`}
content={message.content}
output={message.output}
sources={message.sources}
floatingButtons={message?.done &&
!readOnly &&
@@ -860,9 +860,27 @@
setInputText(text);
}}
onSave={({ raw, oldContent, newContent }) => {
history.messages[message.id].content = history.messages[
message.id
].content.replace(raw, raw.replace(oldContent, newContent));
const sourceMessage = history.messages[message.id];
if (sourceMessage.output?.length) {
const updatedOutput = replaceOutputMessageText(
sourceMessage.output,
oldContent,
newContent
);
if (updatedOutput !== sourceMessage.output) {
sourceMessage.output = updatedOutput;
} else {
sourceMessage.content = sourceMessage.content.replace(
raw,
raw.replace(oldContent, newContent)
);
}
} else {
sourceMessage.content = sourceMessage.content.replace(
raw,
raw.replace(oldContent, newContent)
);
}
updateChat();
}}
@@ -1033,7 +1051,7 @@
? 'visible'
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg dark:hover:text-white hover:text-black transition copy-response-button"
on:click={() => {
copyToClipboard(message.content);
copyToClipboard(visibleResponseContent);
}}
>
<svg
src/lib/components/chat/Messages/StructuredOutputRenderer.svelte
+145
View File
@@ -0,0 +1,145 @@
<script lang="ts">
import Collapsible from '$lib/components/common/Collapsible.svelte';
import ToolCallDisplay from '$lib/components/common/ToolCallDisplay.svelte';
import { settings } from '$lib/stores';
import Markdown from './Markdown.svelte';
import ConsecutiveDetailsGroup from './Markdown/ConsecutiveDetailsGroup.svelte';
import {
buildOutputDisplayItems,
type OutputDetailToken,
type OutputDisplayItem,
type OutputItem
} from './structuredOutput';
export let id = '';
export let output: OutputItem[] = [];
export let done = true;
export let model = null;
export let save = false;
export let preview = false;
export let renderMarkdown = true;
export let editCodeBlock = true;
export let topPadding = false;
export let sourceIds: string[] = [];
export let formatMessageContent: (content: string) => string = (content) => content;
export let onSave: any = () => {};
export let onSourceClick: any = () => {};
export let onTaskClick: any = () => {};
export let onUpdate: any = () => {};
export let onPreview: any = () => {};
const getDetailTitle = (detailToken: OutputDetailToken): any => detailToken.summary;
const getDetailAttributes = (detailToken: OutputDetailToken): any => detailToken.attributes;
$: displayItems = buildOutputDisplayItems(output) as OutputDisplayItem[];
</script>
{#each displayItems as displayItem (displayItem.id)}
{#if displayItem.type === 'message'}
{#if renderMarkdown}
<Markdown
id={`${id}-${displayItem.id}`}
content={formatMessageContent(displayItem.text)}
{model}
{save}
{preview}
{done}
{editCodeBlock}
{topPadding}
{sourceIds}
{onSourceClick}
{onTaskClick}
{onSave}
{onUpdate}
{onPreview}
/>
{:else}
<div class="whitespace-pre-wrap">{displayItem.text}</div>
{/if}
{:else if displayItem.type === 'detail_group'}
<ConsecutiveDetailsGroup
id={`${id}-${displayItem.id}`}
tokens={displayItem.tokens}
messageDone={done}
>
<div slot="content" class="space-y-1">
{#each displayItem.tokens as detailToken, detailIndex}
{#if detailToken.attributes?.type === 'tool_calls'}
<ToolCallDisplay
id={`${id}-${displayItem.id}-${detailIndex}-tool-call`}
attributes={detailToken.attributes}
resultContent={detailToken.text}
grouped={true}
open={$settings?.expandDetails ?? false}
className="w-full space-y-1"
/>
{:else if detailToken.text?.length > 0}
<Collapsible
title={getDetailTitle(detailToken)}
open={$settings?.expandDetails ?? false}
attributes={getDetailAttributes(detailToken)}
messageDone={done}
className="w-full space-y-1"
>
<div class="mb-1.5" slot="content">
<Markdown
id={`${id}-${displayItem.id}-${detailIndex}-detail`}
content={detailToken.text}
{done}
{editCodeBlock}
/>
</div>
</Collapsible>
{:else}
<Collapsible
title={getDetailTitle(detailToken)}
open={false}
disabled={true}
attributes={getDetailAttributes(detailToken)}
messageDone={done}
className="w-full space-y-1"
/>
{/if}
{/each}
</div>
</ConsecutiveDetailsGroup>
{:else}
{@const detailToken = displayItem.token}
{#if detailToken.attributes?.type === 'tool_calls'}
<ToolCallDisplay
id={`${id}-${displayItem.id}-tool-call`}
attributes={detailToken.attributes}
resultContent={detailToken.text}
open={$settings?.expandDetails ?? false}
className="w-full space-y-1"
/>
{:else if detailToken.text?.length > 0}
<Collapsible
title={getDetailTitle(detailToken)}
open={$settings?.expandDetails ?? false}
attributes={getDetailAttributes(detailToken)}
messageDone={done}
className="w-full space-y-1"
>
<div class="mb-1.5" slot="content">
<Markdown
id={`${id}-${displayItem.id}-detail`}
content={detailToken.text}
{done}
{editCodeBlock}
/>
</div>
</Collapsible>
{:else}
<Collapsible
title={getDetailTitle(detailToken)}
open={false}
disabled={true}
attributes={getDetailAttributes(detailToken)}
messageDone={done}
className="w-full space-y-1"
/>
{/if}
{/if}
{/each}
src/lib/components/chat/Messages/structuredOutput.ts
+365
View File
@@ -0,0 +1,365 @@
export type OutputContentPart = {
type?: string;
text?: unknown;
[key: string]: unknown;
};
export type OutputItem = {
type?: string;
id?: string;
call_id?: string;
name?: string;
status?: string;
arguments?: unknown;
content?: OutputContentPart[];
summary?: OutputContentPart[];
output?: OutputContentPart[];
files?: unknown;
embeds?: unknown;
code?: string;
lang?: string;
duration?: number | string | null;
action?: Record<string, unknown>;
actions?: Array<Record<string, unknown>>;
queries?: unknown[];
[key: string]: unknown;
};
export type OutputDetailToken = {
summary: string;
text: string;
attributes: {
type: string;
id?: string;
name?: string;
done?: string;
duration?: string;
arguments?: string;
files?: string;
embeds?: string;
output?: string;
};
};
export type OutputDisplayItem =
| {
type: 'message';
id: string;
text: string;
}
| {
type: 'detail_single';
id: string;
token: OutputDetailToken;
}
| {
type: 'detail_group';
id: string;
tokens: OutputDetailToken[];
};
const GROUPABLE_OUTPUT_TYPES = new Set([
'reasoning',
'function_call',
'open_webui:code_interpreter',
'web_search_call',
'file_search_call',
'computer_call'
]);
const OPENAI_TOOL_NAMES: Record<string, string> = {
web_search_call: 'Web Search',
file_search_call: 'File Search',
computer_call: 'Computer Use'
};
function getTextFromParts(parts: OutputContentPart[] = []): string {
return parts
.map((part) => {
if (part?.text === undefined || part?.text === null) {
return '';
}
return typeof part.text === 'string' ? part.text : String(part.text);
})
.join('');
}
function stringifyAttribute(value: unknown): string {
if (value === undefined || value === null) {
return '';
}
if (typeof value === 'string') {
return value;
}
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}
function isDoneStatus(status?: string): boolean {
return status === 'completed' || status === 'failed' || status === 'incomplete';
}
function getMessageText(item: OutputItem): string {
return getTextFromParts(item.content ?? []);
}
function getReasoningText(item: OutputItem): string {
return getTextFromParts((item.summary ?? item.content) ?? []);
}
function getToolResultText(item?: OutputItem): string {
return (item?.output ?? [])
.filter((part) => part?.type !== 'input_image')
.map((part) => {
if (part?.text === undefined || part?.text === null) {
return '';
}
return typeof part.text === 'string' ? part.text : String(part.text);
})
.join('');
}
function buildToolCallToken(item: OutputItem, toolOutputByCallId: Record<string, OutputItem>) {
const callId = item.call_id ?? '';
const resultItem = toolOutputByCallId[callId];
const isDone = isDoneStatus(item.status) || !!resultItem;
return {
summary: isDone ? 'Tool Executed' : 'Executing...',
text: getToolResultText(resultItem),
attributes: {
type: 'tool_calls',
id: callId,
name: item.name ?? '',
done: isDone ? 'true' : 'false',
arguments: stringifyAttribute(item.arguments ?? ''),
files: stringifyAttribute(resultItem?.files),
embeds: stringifyAttribute(resultItem?.embeds)
}
};
}
function buildReasoningToken(item: OutputItem, isLastItem: boolean) {
const duration = item.duration ?? '';
const isDone = isDoneStatus(item.status) || item.duration !== undefined || !isLastItem;
const text = getReasoningText(item)
.split('\n')
.map((line) => (line.startsWith('>') ? line : `> ${line}`))
.join('\n');
return {
summary: isDone ? `Thought for ${duration || 0} seconds` : 'Thinking...',
text,
attributes: {
type: 'reasoning',
done: isDone ? 'true' : 'false',
duration: String(duration)
}
};
}
function buildCodeInterpreterToken(item: OutputItem, isLastItem: boolean) {
const duration = item.duration ?? '';
const isDone = isDoneStatus(item.status) || item.duration !== undefined || !isLastItem;
const code = item.code ?? '';
const lang = item.lang ?? 'python';
return {
summary: isDone ? 'Analyzed' : 'Analyzing...',
text: code ? `\`\`\`${lang}\n${code}\n\`\`\`` : '',
attributes: {
type: 'code_interpreter',
done: isDone ? 'true' : 'false',
duration: String(duration),
output: stringifyAttribute(item.output)
}
};
}
function getOpenAIToolSummary(item: OutputItem): string {
if (item.type === 'web_search_call') {
const action = item.action ?? {};
const actionType = action.type;
if (actionType === 'search') {
const queries = Array.isArray(action.queries) ? action.queries : [];
const query = typeof action.query === 'string' ? action.query : '';
return queries.length ? `Search: ${queries.join(', ')}` : query ? `Search: ${query}` : '';
}
if (actionType === 'open_page' && typeof action.url === 'string') {
return `Open page: ${action.url}`;
}
if (actionType === 'find_in_page' && typeof action.pattern === 'string') {
return `Find in page: ${action.pattern}`;
}
}
if (item.type === 'file_search_call') {
const queries = item.queries ?? [];
return queries.length ? `Queries: ${queries.join(', ')}` : '';
}
if (item.type === 'computer_call') {
if (item.action?.type) {
return `Action: ${item.action.type}`;
}
if (Array.isArray(item.actions) && item.actions.length) {
return `Actions: ${item.actions.map((action) => action.type ?? '?').join(', ')}`;
}
}
return '';
}
function buildOpenAIToolToken(item: OutputItem, isLastItem: boolean) {
const isDone = isDoneStatus(item.status) || !isLastItem;
return {
summary: isDone ? 'Tool Executed' : 'Executing...',
text: getOpenAIToolSummary(item),
attributes: {
type: 'tool_calls',
id: item.id ?? '',
name: OPENAI_TOOL_NAMES[item.type ?? ''] ?? item.type ?? '',
done: isDone ? 'true' : 'false',
arguments: ''
}
};
}
function buildDetailToken(
item: OutputItem,
isLastItem: boolean,
toolOutputByCallId: Record<string, OutputItem>
): OutputDetailToken | null {
if (item.type === 'function_call') {
return buildToolCallToken(item, toolOutputByCallId);
}
if (item.type === 'reasoning') {
return buildReasoningToken(item, isLastItem);
}
if (item.type === 'open_webui:code_interpreter') {
return buildCodeInterpreterToken(item, isLastItem);
}
if (item.type && OPENAI_TOOL_NAMES[item.type]) {
return buildOpenAIToolToken(item, isLastItem);
}
return null;
}
export function buildOutputDisplayItems(output: OutputItem[] = []): OutputDisplayItem[] {
const displayItems: OutputDisplayItem[] = [];
const currentDetailTokens: OutputDetailToken[] = [];
const toolOutputByCallId: Record<string, OutputItem> = {};
for (const item of output) {
if (item?.type === 'function_call_output' && item.call_id) {
toolOutputByCallId[item.call_id] = item;
}
}
const flushDetails = () => {
if (currentDetailTokens.length > 1) {
displayItems.push({
type: 'detail_group',
id: `detail-group-${displayItems.length}`,
tokens: [...currentDetailTokens]
});
} else if (currentDetailTokens.length === 1) {
displayItems.push({
type: 'detail_single',
id: `detail-${displayItems.length}`,
token: currentDetailTokens[0]
});
}
currentDetailTokens.length = 0;
};
output.forEach((item, index) => {
if (item?.type === 'function_call_output') {
return;
}
if (item?.type && GROUPABLE_OUTPUT_TYPES.has(item.type)) {
const token = buildDetailToken(item, index === output.length - 1, toolOutputByCallId);
if (token) {
currentDetailTokens.push(token);
}
return;
}
if (item?.type === 'message') {
const text = getMessageText(item);
if (text.trim()) {
flushDetails();
displayItems.push({
type: 'message',
id: item.id ?? `message-${index}`,
text
});
}
return;
}
const fallbackText = getMessageText(item);
if (fallbackText.trim()) {
flushDetails();
displayItems.push({
type: 'message',
id: item.id ?? `output-${index}`,
text: fallbackText
});
}
});
flushDetails();
return displayItems;
}
export function getOutputText(output?: OutputItem[] | null): string {
return (output ?? [])
.filter((item) => item?.type === 'message')
.map(getMessageText)
.filter((text) => text.trim())
.join('\n');
}
export function replaceOutputMessageText(
output: OutputItem[] = [],
oldContent: string,
newContent: string
): OutputItem[] {
if (!oldContent) {
return output;
}
let replaced = false;
const nextOutput = output.map((item) => {
if (replaced || item?.type !== 'message' || !Array.isArray(item.content)) {
return item;
}
const partIndex = item.content.findIndex(
(part) => typeof part.text === 'string' && part.text.includes(oldContent)
);
if (partIndex === -1) {
return item;
}
replaced = true;
const nextContent = [...item.content];
const part = nextContent[partIndex];
nextContent[partIndex] = {
...part,
text: (part.text as string).replace(oldContent, newContent)
};
return {
...item,
content: nextContent
};
});
return replaced ? nextOutput : output;
}