refac
This commit is contained in:
1 parent
8dbbc206c5
commit
0800c21c64
3 files changed
+159
-98
No files matched your search
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
@@ -40,6 +41,62 @@ from sqlalchemy.sql.expression import bindparam
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
ACTIVE_CHAT_GAP_SECONDS = 30 * 60
|
ACTIVE_CHAT_GAP_SECONDS = 30 * 60
|
||||||
|
CHAT_SEARCH_FILTER_PREFIXES = ('tag:', 'folder:', 'pinned:', 'archived:', 'shared:')
|
||||||
|
|
||||||
|
|
||||||
|
def chat_search_content_query(text: str) -> str:
|
||||||
|
words = sanitize_text_for_db(text).lower().strip().split()
|
||||||
|
return ' '.join(word for word in words if not word.startswith(CHAT_SEARCH_FILTER_PREFIXES)).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def chat_search_terms(text: str) -> list[str]:
|
||||||
|
return list(dict.fromkeys(re.findall(r'[a-z0-9]+', text.lower())))
|
||||||
|
|
||||||
|
|
||||||
|
def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
|
||||||
|
if dialect_name == 'sqlite':
|
||||||
|
return f"""
|
||||||
|
(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_each(Chat.chat, '$.history.messages') AS history_message
|
||||||
|
WHERE LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||||
|
)
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_each(Chat.chat, '$.messages') AS legacy_message
|
||||||
|
WHERE LOWER(legacy_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
if dialect_name == 'postgresql':
|
||||||
|
return f"""
|
||||||
|
(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM chat_message AS message
|
||||||
|
WHERE message.chat_id = Chat.id
|
||||||
|
AND message.user_id = Chat.user_id
|
||||||
|
AND json_typeof(message.content) = 'string'
|
||||||
|
AND LOWER(message.content #>> '{{}}') LIKE '%' || :{key} || '%'
|
||||||
|
)
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_each(Chat.chat#>'{{history,messages}}') AS history_message
|
||||||
|
WHERE json_typeof(history_message.value->'content') = 'string'
|
||||||
|
AND LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||||
|
)
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_array_elements(Chat.chat->'messages') AS legacy_message
|
||||||
|
WHERE json_typeof(legacy_message->'content') = 'string'
|
||||||
|
AND LOWER(legacy_message->>'content') LIKE '%' || :{key} || '%'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
raise NotImplementedError(f'Unsupported dialect: {dialect_name}')
|
||||||
|
|
||||||
|
|
||||||
def chat_list_order(sort_by: str = 'updated_at', sort_dir: str = 'desc', user_id: str | None = None):
|
def chat_list_order(sort_by: str = 'updated_at', sort_dir: str = 'desc', user_id: str | None = None):
|
||||||
@@ -1731,7 +1788,7 @@ class ChatTable:
|
|||||||
return [ChatModel.model_validate(chat) for chat in result.scalars().all()]
|
return [ChatModel.model_validate(chat) for chat in result.scalars().all()]
|
||||||
|
|
||||||
# search user conversations
|
# search user conversations
|
||||||
async def get_chats_by_user_id_and_search_text(
|
async def get_chats_by_user_id_and_search_text( # noqa: C901
|
||||||
self,
|
self,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
search_text: str,
|
search_text: str,
|
||||||
@@ -1750,7 +1807,7 @@ class ChatTable:
|
|||||||
user_id, include_archived, filter={}, skip=skip, limit=limit, db=db
|
user_id, include_archived, filter={}, skip=skip, limit=limit, db=db
|
||||||
)
|
)
|
||||||
|
|
||||||
search_text_words = search_text.split(' ')
|
search_text_words = search_text.split()
|
||||||
|
|
||||||
# search_text might contain 'tag:tag_name' format so we need to extract the tag_name
|
# search_text might contain 'tag:tag_name' format so we need to extract the tag_name
|
||||||
tag_ids = [
|
tag_ids = [
|
||||||
@@ -1782,19 +1839,10 @@ class ChatTable:
|
|||||||
elif 'shared:false' in search_text_words:
|
elif 'shared:false' in search_text_words:
|
||||||
is_shared = False
|
is_shared = False
|
||||||
|
|
||||||
search_text_words = [
|
search_text_words = [word for word in search_text_words if not word.startswith(CHAT_SEARCH_FILTER_PREFIXES)]
|
||||||
word
|
|
||||||
for word in search_text_words
|
|
||||||
if (
|
|
||||||
not word.startswith('tag:')
|
|
||||||
and not word.startswith('folder:')
|
|
||||||
and not word.startswith('pinned:')
|
|
||||||
and not word.startswith('archived:')
|
|
||||||
and not word.startswith('shared:')
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
search_text = ' '.join(search_text_words)
|
phrase_query = ' '.join(search_text_words).strip()
|
||||||
|
search_terms = chat_search_terms(phrase_query)
|
||||||
|
|
||||||
async with get_async_db_context(db) as session:
|
async with get_async_db_context(db) as session:
|
||||||
stmt = select(Chat).filter(Chat.user_id == user_id)
|
stmt = select(Chat).filter(Chat.user_id == user_id)
|
||||||
@@ -1817,27 +1865,43 @@ class ChatTable:
|
|||||||
if folder_ids:
|
if folder_ids:
|
||||||
stmt = stmt.filter(Chat.folder_id.in_(folder_ids))
|
stmt = stmt.filter(Chat.folder_id.in_(folder_ids))
|
||||||
|
|
||||||
stmt = stmt.order_by(Chat.updated_at.desc(), Chat.id)
|
|
||||||
|
|
||||||
# Check if the database dialect is either 'sqlite' or 'postgresql'
|
# Check if the database dialect is either 'sqlite' or 'postgresql'
|
||||||
bind = await session.connection()
|
bind = await session.connection()
|
||||||
dialect_name = bind.dialect.name
|
dialect_name = bind.dialect.name
|
||||||
if dialect_name == 'sqlite':
|
|
||||||
# SQLite case: using JSON1 extension for JSON searching
|
search_params = {}
|
||||||
sqlite_content_sql = (
|
exact_match_clause = None
|
||||||
'EXISTS ('
|
if phrase_query:
|
||||||
' SELECT 1 '
|
exact_match_clause = or_(
|
||||||
" FROM json_each(Chat.chat, '$.messages') AS message "
|
Chat.title.ilike(bindparam('phrase_title_key')),
|
||||||
" WHERE LOWER(message.value->>'content') LIKE '%' || :content_key || '%'"
|
text(chat_search_message_content_match_sql(dialect_name, 'phrase_content_key')),
|
||||||
')'
|
|
||||||
)
|
)
|
||||||
sqlite_content_clause = text(sqlite_content_sql)
|
search_params.update(
|
||||||
stmt = stmt.filter(
|
{
|
||||||
or_(Chat.title.ilike(bindparam('title_key')), sqlite_content_clause).params(
|
'phrase_title_key': f'%{phrase_query}%',
|
||||||
title_key=f'%{search_text}%', content_key=search_text
|
'phrase_content_key': phrase_query,
|
||||||
)
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
term_clauses = []
|
||||||
|
for term_idx, term in enumerate(search_terms):
|
||||||
|
title_key = f'term_title_key_{term_idx}'
|
||||||
|
content_key = f'term_content_key_{term_idx}'
|
||||||
|
term_clauses.append(
|
||||||
|
or_(
|
||||||
|
Chat.title.ilike(bindparam(title_key)),
|
||||||
|
text(chat_search_message_content_match_sql(dialect_name, content_key)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
search_params[title_key] = f'%{term}%'
|
||||||
|
search_params[content_key] = term
|
||||||
|
|
||||||
|
if term_clauses:
|
||||||
|
stmt = stmt.filter(or_(exact_match_clause, and_(*term_clauses)))
|
||||||
|
else:
|
||||||
|
stmt = stmt.filter(exact_match_clause)
|
||||||
|
|
||||||
|
if dialect_name == 'sqlite':
|
||||||
# Check if there are any tags to filter
|
# Check if there are any tags to filter
|
||||||
if 'none' in tag_ids:
|
if 'none' in tag_ids:
|
||||||
stmt = stmt.filter(
|
stmt = stmt.filter(
|
||||||
@@ -1871,38 +1935,6 @@ class ChatTable:
|
|||||||
# Safety filter: title must not contain actual null bytes
|
# Safety filter: title must not contain actual null bytes
|
||||||
stmt = stmt.filter(text("Chat.title::text NOT LIKE '%\\x00%'"))
|
stmt = stmt.filter(text("Chat.title::text NOT LIKE '%\\x00%'"))
|
||||||
|
|
||||||
postgres_content_sql = """
|
|
||||||
EXISTS (
|
|
||||||
SELECT 1
|
|
||||||
FROM chat_message AS message
|
|
||||||
WHERE message.chat_id = Chat.id
|
|
||||||
AND message.user_id = Chat.user_id
|
|
||||||
AND json_typeof(message.content) = 'string'
|
|
||||||
AND LOWER(message.content #>> '{}') LIKE '%' || :content_key || '%'
|
|
||||||
)
|
|
||||||
OR EXISTS (
|
|
||||||
SELECT 1
|
|
||||||
FROM json_each(Chat.chat#>'{history,messages}') AS history_message
|
|
||||||
WHERE json_typeof(history_message.value->'content') = 'string'
|
|
||||||
AND LOWER(history_message.value->>'content') LIKE '%' || :content_key || '%'
|
|
||||||
)
|
|
||||||
OR EXISTS (
|
|
||||||
SELECT 1
|
|
||||||
FROM json_array_elements(Chat.chat->'messages') AS legacy_message
|
|
||||||
WHERE json_typeof(legacy_message->'content') = 'string'
|
|
||||||
AND LOWER(legacy_message->>'content') LIKE '%' || :content_key || '%'
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
|
|
||||||
postgres_content_clause = text(postgres_content_sql)
|
|
||||||
|
|
||||||
stmt = stmt.filter(
|
|
||||||
or_(
|
|
||||||
Chat.title.ilike(bindparam('title_key')),
|
|
||||||
postgres_content_clause,
|
|
||||||
)
|
|
||||||
).params(title_key=f'%{search_text}%', content_key=search_text.lower())
|
|
||||||
|
|
||||||
if 'none' in tag_ids:
|
if 'none' in tag_ids:
|
||||||
stmt = stmt.filter(
|
stmt = stmt.filter(
|
||||||
text("""
|
text("""
|
||||||
@@ -1930,6 +1962,14 @@ class ChatTable:
|
|||||||
else:
|
else:
|
||||||
raise NotImplementedError(f'Unsupported dialect: {dialect_name}')
|
raise NotImplementedError(f'Unsupported dialect: {dialect_name}')
|
||||||
|
|
||||||
|
if exact_match_clause is not None:
|
||||||
|
stmt = stmt.order_by(case((exact_match_clause, 0), else_=1), Chat.updated_at.desc(), Chat.id)
|
||||||
|
else:
|
||||||
|
stmt = stmt.order_by(Chat.updated_at.desc(), Chat.id)
|
||||||
|
|
||||||
|
if search_params:
|
||||||
|
stmt = stmt.params(**search_params)
|
||||||
|
|
||||||
# Perform pagination at the SQL level
|
# Perform pagination at the SQL level
|
||||||
stmt = stmt.offset(skip).limit(limit)
|
stmt = stmt.offset(skip).limit(limit)
|
||||||
result = await session.execute(stmt)
|
result = await session.execute(stmt)
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ from open_webui.constants import ERROR_MESSAGES
|
|||||||
from open_webui.events import EVENTS, publish_event
|
from open_webui.events import EVENTS, publish_event
|
||||||
from open_webui.internal.db import get_async_session
|
from open_webui.internal.db import get_async_session
|
||||||
from open_webui.models.access_grants import AccessGrants
|
from open_webui.models.access_grants import AccessGrants
|
||||||
from open_webui.models.config import Config
|
|
||||||
from open_webui.models.chat_messages import ChatMessages
|
from open_webui.models.chat_messages import ChatMessages
|
||||||
from open_webui.models.chats import (
|
from open_webui.models.chats import (
|
||||||
AggregateChatStats,
|
AggregateChatStats,
|
||||||
@@ -27,9 +26,12 @@ from open_webui.models.chats import (
|
|||||||
ChatStatsExport,
|
ChatStatsExport,
|
||||||
ChatTitleIdResponse,
|
ChatTitleIdResponse,
|
||||||
ChatUsageStatsListResponse,
|
ChatUsageStatsListResponse,
|
||||||
is_internal_chat,
|
|
||||||
MessageStats,
|
MessageStats,
|
||||||
|
chat_search_content_query,
|
||||||
|
chat_search_terms,
|
||||||
|
is_internal_chat,
|
||||||
)
|
)
|
||||||
|
from open_webui.models.config import Config
|
||||||
from open_webui.models.folders import Folders
|
from open_webui.models.folders import Folders
|
||||||
from open_webui.models.shared_chats import SharedChatResponse, SharedChats
|
from open_webui.models.shared_chats import SharedChatResponse, SharedChats
|
||||||
from open_webui.models.tags import TagModel, Tags
|
from open_webui.models.tags import TagModel, Tags
|
||||||
@@ -49,8 +51,6 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
SEARCH_FILTER_PREFIXES = ('tag:', 'folder:', 'pinned:', 'archived:', 'shared:')
|
|
||||||
|
|
||||||
CHAT_CONFIG_KEYS = {
|
CHAT_CONFIG_KEYS = {
|
||||||
'CONTEXT_COMPACTION_MODEL': 'chat.context_compaction.model',
|
'CONTEXT_COMPACTION_MODEL': 'chat.context_compaction.model',
|
||||||
'ENABLE_CONTEXT_COMPACTION': 'chat.context_compaction.enable',
|
'ENABLE_CONTEXT_COMPACTION': 'chat.context_compaction.enable',
|
||||||
@@ -147,38 +147,42 @@ class CompactChatForm(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
def chat_search_content_text(text: str) -> str:
|
def chat_search_content_text(text: str) -> str:
|
||||||
words = text.lower().strip().split(' ')
|
return chat_search_content_query(text)
|
||||||
return ' '.join(word for word in words if not word.startswith(SEARCH_FILTER_PREFIXES)).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
|
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
|
||||||
if not search_text:
|
if not search_text:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
messages = chat.get('messages', [])
|
history = chat.get('history', {})
|
||||||
|
messages = history.get('messages') if isinstance(history, dict) else None
|
||||||
|
if not messages:
|
||||||
|
messages = chat.get('messages', []) or []
|
||||||
if isinstance(messages, dict):
|
if isinstance(messages, dict):
|
||||||
messages = messages.values()
|
messages = messages.values()
|
||||||
|
|
||||||
for message in messages:
|
needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)]))
|
||||||
if not isinstance(message, dict):
|
for needle in needles:
|
||||||
continue
|
for message in messages:
|
||||||
|
if not isinstance(message, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
content = message.get('content')
|
content = message.get('content')
|
||||||
if not isinstance(content, str):
|
if not isinstance(content, str):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
index = content.lower().find(search_text)
|
index = content.lower().find(needle)
|
||||||
if index == -1:
|
if index == -1:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
start = max(index - max_length // 2, 0)
|
start = max(index - max_length // 2, 0)
|
||||||
end = min(start + max_length, len(content))
|
end = min(start + max_length, len(content))
|
||||||
if index + len(search_text) > end:
|
if index + len(needle) > end:
|
||||||
end = min(index + len(search_text), len(content))
|
end = min(index + len(needle), len(content))
|
||||||
start = max(end - max_length, 0)
|
start = max(end - max_length, 0)
|
||||||
|
|
||||||
snippet = ' '.join(content[start:end].split())
|
snippet = ' '.join(content[start:end].split())
|
||||||
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
|
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from open_webui.env import (
|
|||||||
)
|
)
|
||||||
from open_webui.events import EVENTS, publish_event
|
from open_webui.events import EVENTS, publish_event
|
||||||
from open_webui.models.channels import Channel, ChannelMember, Channels
|
from open_webui.models.channels import Channel, ChannelMember, Channels
|
||||||
from open_webui.models.chats import Chats
|
from open_webui.models.chats import Chats, chat_search_content_query, chat_search_terms
|
||||||
from open_webui.models.config import Config
|
from open_webui.models.config import Config
|
||||||
from open_webui.models.groups import Groups
|
from open_webui.models.groups import Groups
|
||||||
from open_webui.models.memories import Memories
|
from open_webui.models.memories import Memories
|
||||||
@@ -1384,10 +1384,13 @@ async def search_chats(
|
|||||||
__chat_id__: str = None,
|
__chat_id__: str = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Search the user's previous chat conversations by title and message content.
|
Search the user's previous chat conversations by title and message content,
|
||||||
Helpful for finding details from earlier conversations.
|
excluding the current chat. Helpful for finding details from earlier
|
||||||
|
conversations when they are not already visible in the current context.
|
||||||
|
Exact phrase matches are preferred, and descriptive keyword queries are
|
||||||
|
supported.
|
||||||
|
|
||||||
:param query: The search query to find matching chats
|
:param query: Exact phrase or descriptive keyword query to find matching previous chats
|
||||||
:param count: Maximum number of results to return (default: 5)
|
:param count: Maximum number of results to return (default: 5)
|
||||||
:param start_timestamp: Only include chats updated after this Unix timestamp (seconds)
|
:param start_timestamp: Only include chats updated after this Unix timestamp (seconds)
|
||||||
:param end_timestamp: Only include chats updated before this Unix timestamp (seconds)
|
:param end_timestamp: Only include chats updated before this Unix timestamp (seconds)
|
||||||
@@ -1425,19 +1428,33 @@ async def search_chats(
|
|||||||
# Find a matching message snippet
|
# Find a matching message snippet
|
||||||
snippet = ''
|
snippet = ''
|
||||||
messages = (getattr(chat, 'chat', None) or {}).get('history', {}).get('messages', {})
|
messages = (getattr(chat, 'chat', None) or {}).get('history', {}).get('messages', {})
|
||||||
lower_query = query.lower()
|
if not messages:
|
||||||
|
messages = (getattr(chat, 'chat', None) or {}).get('messages', {}) or {}
|
||||||
|
if isinstance(messages, list):
|
||||||
|
messages = {str(idx): message for idx, message in enumerate(messages)}
|
||||||
|
|
||||||
for msg_id, msg in messages.items():
|
lower_query = chat_search_content_query(query)
|
||||||
content = msg.get('content', '')
|
needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else []
|
||||||
if isinstance(content, str) and lower_query in content.lower():
|
|
||||||
idx = content.lower().find(lower_query)
|
for needle in needles:
|
||||||
start = max(0, idx - 50)
|
for msg_id, msg in messages.items():
|
||||||
end = min(len(content), idx + len(query) + 100)
|
content = msg.get('content', '') if isinstance(msg, dict) else ''
|
||||||
snippet = ('...' if start > 0 else '') + content[start:end] + ('...' if end < len(content) else '')
|
if isinstance(content, str) and needle in content.lower():
|
||||||
|
idx = content.lower().find(needle)
|
||||||
|
start = max(0, idx - 50)
|
||||||
|
end = min(len(content), idx + len(needle) + 100)
|
||||||
|
snippet = (
|
||||||
|
('...' if start > 0 else '')
|
||||||
|
+ content[start:end]
|
||||||
|
+ ('...' if end < len(content) else '')
|
||||||
|
)
|
||||||
|
break
|
||||||
|
if snippet:
|
||||||
break
|
break
|
||||||
|
|
||||||
if not snippet and lower_query in chat.title.lower():
|
title = chat.title or ''
|
||||||
snippet = f'Title match: {chat.title}'
|
if not snippet and any(needle in title.lower() for needle in needles):
|
||||||
|
snippet = f'Title match: {title}'
|
||||||
|
|
||||||
results.append(
|
results.append(
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in new issue
Block a user