bhetherman/open-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-06-29 04:43:08 -05:00
parent 67c9de8efd
commit e5b5e5917b
3 changed files with 87 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/retrieval/utils.py
+3 -3
View File
@@ -68,9 +68,9 @@ def is_youtube_url(url: str) -> bool:
LOADER_CONFIG_KEYS = {
'youtube_language': 'rag.youtube_loader_language',
'youtube_proxy_url': 'rag.youtube_loader_proxy_url',
'web_loader_ssl_verification': 'rag.web.loader.ssl_verification',
'web_loader_concurrent_requests': 'rag.web.loader.concurrent_requests',
'web_search_trust_env': 'rag.web.search.trust_env',
'web_loader_ssl_verification': 'web.loader.ssl_verification',
'web_loader_concurrent_requests': 'web.loader.concurrent_requests',
'web_search_trust_env': 'web.search.trust_env',
'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key',
'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
backend/open_webui/routers/retrieval.py
+81 -71
View File
@@ -248,17 +248,17 @@ router = APIRouter()
RETRIEVAL_CONFIG_KEYS = {
'ALLOWED_FILE_EXTENSIONS': 'rag.file.allowed_extensions',
'AZURE_AI_SEARCH_API_KEY': 'rag.web.search.azure_ai_search_api_key',
'AZURE_AI_SEARCH_ENDPOINT': 'rag.web.search.azure_ai_search_endpoint',
'AZURE_AI_SEARCH_INDEX_NAME': 'rag.web.search.azure_ai_search_index_name',
'BING_SEARCH_V7_ENDPOINT': 'rag.web.search.bing_search_v7_endpoint',
'BING_SEARCH_V7_SUBSCRIPTION_KEY': 'rag.web.search.bing_search_v7_subscription_key',
'BOCHA_SEARCH_API_KEY': 'rag.web.search.bocha_search_api_key',
'BRAVE_SEARCH_API_KEY': 'rag.web.search.brave_search_api_key',
'BRAVE_SEARCH_CONTEXT_TOKENS': 'rag.web.search.brave_search_context_tokens',
'AZURE_AI_SEARCH_API_KEY': 'web.search.azure_ai_search_api_key',
'AZURE_AI_SEARCH_ENDPOINT': 'web.search.azure_ai_search_endpoint',
'AZURE_AI_SEARCH_INDEX_NAME': 'web.search.azure_ai_search_index_name',
'BING_SEARCH_V7_ENDPOINT': 'web.search.bing_search_v7_endpoint',
'BING_SEARCH_V7_SUBSCRIPTION_KEY': 'web.search.bing_search_v7_subscription_key',
'BOCHA_SEARCH_API_KEY': 'web.search.bocha_search_api_key',
'BRAVE_SEARCH_API_KEY': 'web.search.brave_search_api_key',
'BRAVE_SEARCH_CONTEXT_TOKENS': 'web.search.brave_search_context_tokens',
'BYPASS_EMBEDDING_AND_RETRIEVAL': 'rag.bypass_embedding_and_retrieval',
'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': 'rag.web.search.bypass_embedding_and_retrieval',
'BYPASS_WEB_SEARCH_WEB_LOADER': 'rag.web.search.bypass_web_loader',
'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': 'web.search.bypass_embedding_and_retrieval',
'BYPASS_WEB_SEARCH_WEB_LOADER': 'web.search.bypass_web_loader',
'CHUNK_MIN_SIZE_TARGET': 'rag.chunk_min_size_target',
'CHUNK_OVERLAP': 'rag.chunk_overlap',
'CHUNK_SIZE': 'rag.chunk_size',
@@ -274,7 +274,7 @@ RETRIEVAL_CONFIG_KEYS = {
'DATALAB_MARKER_SKIP_CACHE': 'rag.datalab_marker_skip_cache',
'DATALAB_MARKER_STRIP_EXISTING_OCR': 'rag.datalab_marker_strip_existing_ocr',
'DATALAB_MARKER_USE_LLM': 'rag.datalab_marker_use_llm',
'DDGS_BACKEND': 'rag.web.search.ddgs_backend',
'DDGS_BACKEND': 'web.search.ddgs_backend',
'DOCLING_API_KEY': 'rag.docling_api_key',
'DOCLING_PARAMS': 'rag.docling_params',
'DOCLING_SERVER_URL': 'rag.docling_server_url',
@@ -287,54 +287,56 @@ RETRIEVAL_CONFIG_KEYS = {
'ENABLE_ONEDRIVE_INTEGRATION': 'onedrive.enable',
'ENABLE_RAG_HYBRID_SEARCH': 'rag.enable_hybrid_search',
'ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS': 'rag.enable_hybrid_search_enriched_texts',
'ENABLE_WEB_LOADER_SSL_VERIFICATION': 'rag.web.loader.ssl_verification',
'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
'EXA_API_KEY': 'rag.web.search.exa_api_key',
'ENABLE_WEB_LOADER_SSL_VERIFICATION': 'web.loader.ssl_verification',
'ENABLE_WEB_SEARCH': 'web.search.enable',
'ENABLE_WEB_SEARCH_CONFIRMATION': 'web.search.confirmation.enable',
'WEB_SEARCH_CONFIRMATION_CONTENT': 'web.search.confirmation.content',
'EXA_API_KEY': 'web.search.exa_api_key',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
'EXTERNAL_WEB_SEARCH_API_KEY': 'rag.web.search.external_web_search_api_key',
'EXTERNAL_WEB_SEARCH_URL': 'rag.web.search.external_web_search_url',
'EXTERNAL_WEB_LOADER_API_KEY': 'web.loader.external_web_loader_api_key',
'EXTERNAL_WEB_LOADER_URL': 'web.loader.external_web_loader_url',
'EXTERNAL_WEB_SEARCH_API_KEY': 'web.search.external_web_search_api_key',
'EXTERNAL_WEB_SEARCH_URL': 'web.search.external_web_search_url',
'FILE_IMAGE_COMPRESSION_HEIGHT': 'file.image_compression_height',
'FILE_IMAGE_COMPRESSION_WIDTH': 'file.image_compression_width',
'FILE_MAX_COUNT': 'rag.file.max_count',
'FILE_MAX_SIZE': 'rag.file.max_size',
'FIRECRAWL_API_BASE_URL': 'rag.web.loader.firecrawl_api_url',
'FIRECRAWL_API_KEY': 'rag.web.loader.firecrawl_api_key',
'FIRECRAWL_TIMEOUT': 'rag.web.loader.firecrawl_timeout',
'GOOGLE_PSE_API_KEY': 'rag.web.search.google_pse_api_key',
'GOOGLE_PSE_ENGINE_ID': 'rag.web.search.google_pse_engine_id',
'FIRECRAWL_API_BASE_URL': 'web.loader.firecrawl_api_url',
'FIRECRAWL_API_KEY': 'web.loader.firecrawl_api_key',
'FIRECRAWL_TIMEOUT': 'web.loader.firecrawl_timeout',
'GOOGLE_PSE_API_KEY': 'web.search.google_pse_api_key',
'GOOGLE_PSE_ENGINE_ID': 'web.search.google_pse_engine_id',
'HYBRID_BM25_WEIGHT': 'rag.hybrid_bm25_weight',
'JINA_API_BASE_URL': 'rag.web.search.jina_api_base_url',
'JINA_API_KEY': 'rag.web.search.jina_api_key',
'KAGI_SEARCH_API_KEY': 'rag.web.search.kagi_search_api_key',
'LINKUP_API_KEY': 'rag.web.search.linkup_api_key',
'LINKUP_SEARCH_PARAMS': 'rag.web.search.linkup_search_params',
'JINA_API_BASE_URL': 'web.search.jina_api_base_url',
'JINA_API_KEY': 'web.search.jina_api_key',
'KAGI_SEARCH_API_KEY': 'web.search.kagi_search_api_key',
'LINKUP_API_KEY': 'web.search.linkup_api_key',
'LINKUP_SEARCH_PARAMS': 'web.search.linkup_search_params',
'MINERU_API_KEY': 'rag.mineru_api_key',
'MINERU_API_MODE': 'rag.mineru_api_mode',
'MINERU_API_TIMEOUT': 'rag.mineru_api_timeout',
'MINERU_API_URL': 'rag.mineru_api_url',
'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions',
'MINERU_PARAMS': 'rag.mineru_params',
'MICROSOFT_WEB_IQ_API_BASE_URL': 'rag.web.search.microsoft_web_iq_api_base_url',
'MICROSOFT_WEB_IQ_API_KEY': 'rag.web.search.microsoft_web_iq_api_key',
'MICROSOFT_WEB_IQ_LANGUAGE': 'rag.web.search.microsoft_web_iq_language',
'MICROSOFT_WEB_IQ_API_BASE_URL': 'web.search.microsoft_web_iq_api_base_url',
'MICROSOFT_WEB_IQ_API_KEY': 'web.search.microsoft_web_iq_api_key',
'MICROSOFT_WEB_IQ_LANGUAGE': 'web.search.microsoft_web_iq_language',
'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url',
'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key',
'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key',
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': 'rag.web.search.ollama_cloud_api_key',
'MOJEEK_SEARCH_API_KEY': 'web.search.mojeek_search_api_key',
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': 'web.search.ollama_cloud_api_key',
'PADDLEOCR_VL_BASE_URL': 'rag.paddleocr_vl_base_url',
'PADDLEOCR_VL_TOKEN': 'rag.paddleocr_vl_token',
'PDF_EXTRACT_IMAGES': 'rag.pdf_extract_images',
'PDF_LOADER_MODE': 'rag.pdf_loader_mode',
'PERPLEXITY_API_KEY': 'rag.web.search.perplexity_api_key',
'PERPLEXITY_MODEL': 'rag.web.search.perplexity_model',
'PERPLEXITY_SEARCH_API_URL': 'rag.web.search.perplexity_search_api_url',
'PERPLEXITY_SEARCH_CONTEXT_USAGE': 'rag.web.search.perplexity_search_context_usage',
'PLAYWRIGHT_TIMEOUT': 'rag.web.loader.playwright_timeout',
'PLAYWRIGHT_WS_URL': 'rag.web.loader.playwright_ws_url',
'PERPLEXITY_API_KEY': 'web.search.perplexity_api_key',
'PERPLEXITY_MODEL': 'web.search.perplexity_model',
'PERPLEXITY_SEARCH_API_URL': 'web.search.perplexity_search_api_url',
'PERPLEXITY_SEARCH_CONTEXT_USAGE': 'web.search.perplexity_search_context_usage',
'PLAYWRIGHT_TIMEOUT': 'web.loader.playwright_timeout',
'PLAYWRIGHT_WS_URL': 'web.loader.playwright_ws_url',
'RAG_AZURE_OPENAI_API_KEY': 'rag.azure_openai.api_key',
'RAG_AZURE_OPENAI_API_VERSION': 'rag.azure_openai.api_version',
'RAG_AZURE_OPENAI_BASE_URL': 'rag.azure_openai.base_url',
@@ -355,22 +357,22 @@ RETRIEVAL_CONFIG_KEYS = {
'RAG_RERANKING_MODEL': 'rag.reranking_model',
'RAG_TEMPLATE': 'rag.template',
'RELEVANCE_THRESHOLD': 'rag.relevance_threshold',
'SEARCHAPI_API_KEY': 'rag.web.search.searchapi_api_key',
'SEARCHAPI_ENGINE': 'rag.web.search.searchapi_engine',
'SEARXNG_LANGUAGE': 'rag.web.search.searxng_language',
'SEARXNG_QUERY_URL': 'rag.web.search.searxng_query_url',
'SERPAPI_API_KEY': 'rag.web.search.serpapi_api_key',
'SERPAPI_ENGINE': 'rag.web.search.serpapi_engine',
'SERPER_API_KEY': 'rag.web.search.serper_api_key',
'SERPHOUSE_API_KEY': 'rag.web.search.serphouse_api_key',
'SERPHOUSE_DOMAIN': 'rag.web.search.serphouse_domain',
'SERPLY_API_KEY': 'rag.web.search.serply_api_key',
'SERPSTACK_API_KEY': 'rag.web.search.serpstack_api_key',
'SERPSTACK_HTTPS': 'rag.web.search.serpstack_https',
'SOUGOU_API_SID': 'rag.web.search.sougou_api_sid',
'SOUGOU_API_SK': 'rag.web.search.sougou_api_sk',
'TAVILY_API_KEY': 'rag.web.search.tavily_api_key',
'TAVILY_EXTRACT_DEPTH': 'rag.web.search.tavily_extract_depth',
'SEARCHAPI_API_KEY': 'web.search.searchapi_api_key',
'SEARCHAPI_ENGINE': 'web.search.searchapi_engine',
'SEARXNG_LANGUAGE': 'web.search.searxng_language',
'SEARXNG_QUERY_URL': 'web.search.searxng_query_url',
'SERPAPI_API_KEY': 'web.search.serpapi_api_key',
'SERPAPI_ENGINE': 'web.search.serpapi_engine',
'SERPER_API_KEY': 'web.search.serper_api_key',
'SERPHOUSE_API_KEY': 'web.search.serphouse_api_key',
'SERPHOUSE_DOMAIN': 'web.search.serphouse_domain',
'SERPLY_API_KEY': 'web.search.serply_api_key',
'SERPSTACK_API_KEY': 'web.search.serpstack_api_key',
'SERPSTACK_HTTPS': 'web.search.serpstack_https',
'SOUGOU_API_SID': 'web.search.sougou_api_sid',
'SOUGOU_API_SK': 'web.search.sougou_api_sk',
'TAVILY_API_KEY': 'web.search.tavily_api_key',
'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth',
'TEXT_SPLITTER': 'rag.text_splitter',
'TIKA_SERVER_URL': 'rag.tika_server_url',
'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name',
@@ -378,22 +380,22 @@ RETRIEVAL_CONFIG_KEYS = {
'TOP_K_RERANKER': 'rag.top_k_reranker',
'USER_PERMISSIONS': 'user.permissions',
'WEBUI_URL': 'webui.url',
'WEB_FETCH_MAX_CONTENT_LENGTH': 'rag.web.fetch.max_content_length',
'WEB_LOADER_CONCURRENT_REQUESTS': 'rag.web.loader.concurrent_requests',
'WEB_LOADER_ENGINE': 'rag.web.loader.engine',
'WEB_LOADER_TIMEOUT': 'rag.web.loader.timeout',
'WEB_SEARCH_CONCURRENT_REQUESTS': 'rag.web.search.concurrent_requests',
'WEB_SEARCH_DOMAIN_FILTER_LIST': 'rag.web.search.domain.filter_list',
'WEB_SEARCH_ENGINE': 'rag.web.search.engine',
'WEB_SEARCH_RESULT_COUNT': 'rag.web.search.result_count',
'WEB_SEARCH_TRUST_ENV': 'rag.web.search.trust_env',
'YACY_PASSWORD': 'rag.web.search.yacy_password',
'YACY_QUERY_URL': 'rag.web.search.yacy_query_url',
'YACY_USERNAME': 'rag.web.search.yacy_username',
'YANDEX_WEB_SEARCH_API_KEY': 'rag.web.search.yandex_web_search_api_key',
'YANDEX_WEB_SEARCH_CONFIG': 'rag.web.search.yandex_web_search_config',
'YANDEX_WEB_SEARCH_URL': 'rag.web.search.yandex_web_search_url',
'YOUCOM_API_KEY': 'rag.web.search.youcom_api_key',
'WEB_FETCH_MAX_CONTENT_LENGTH': 'web.fetch.max_content_length',
'WEB_LOADER_CONCURRENT_REQUESTS': 'web.loader.concurrent_requests',
'WEB_LOADER_ENGINE': 'web.loader.engine',
'WEB_LOADER_TIMEOUT': 'web.loader.timeout',
'WEB_SEARCH_CONCURRENT_REQUESTS': 'web.search.concurrent_requests',
'WEB_SEARCH_DOMAIN_FILTER_LIST': 'web.search.domain.filter_list',
'WEB_SEARCH_ENGINE': 'web.search.engine',
'WEB_SEARCH_RESULT_COUNT': 'web.search.result_count',
'WEB_SEARCH_TRUST_ENV': 'web.search.trust_env',
'YACY_PASSWORD': 'web.search.yacy_password',
'YACY_QUERY_URL': 'web.search.yacy_query_url',
'YACY_USERNAME': 'web.search.yacy_username',
'YANDEX_WEB_SEARCH_API_KEY': 'web.search.yandex_web_search_api_key',
'YANDEX_WEB_SEARCH_CONFIG': 'web.search.yandex_web_search_config',
'YANDEX_WEB_SEARCH_URL': 'web.search.yandex_web_search_url',
'YOUCOM_API_KEY': 'web.search.youcom_api_key',
'YOUTUBE_LOADER_LANGUAGE': 'rag.youtube_loader_language',
'YOUTUBE_LOADER_PROXY_URL': 'rag.youtube_loader_proxy_url',
}
@@ -685,6 +687,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
# Web search settings
'web': {
'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH,
'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION,
'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT,
'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE,
'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV,
'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT,
@@ -761,6 +765,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
class WebConfig(BaseModel):
ENABLE_WEB_SEARCH: bool | None = None
ENABLE_WEB_SEARCH_CONFIRMATION: bool | None = None
WEB_SEARCH_CONFIRMATION_CONTENT: str | None = None
WEB_SEARCH_ENGINE: str | None = None
WEB_SEARCH_TRUST_ENV: bool | None = None
WEB_SEARCH_RESULT_COUNT: int | None = None
@@ -1262,6 +1268,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
if form_data.web is not None:
# Web search settings
config.ENABLE_WEB_SEARCH = form_data.web.ENABLE_WEB_SEARCH
config.ENABLE_WEB_SEARCH_CONFIRMATION = form_data.web.ENABLE_WEB_SEARCH_CONFIRMATION
config.WEB_SEARCH_CONFIRMATION_CONTENT = form_data.web.WEB_SEARCH_CONFIRMATION_CONTENT
config.WEB_SEARCH_ENGINE = form_data.web.WEB_SEARCH_ENGINE
config.WEB_SEARCH_TRUST_ENV = form_data.web.WEB_SEARCH_TRUST_ENV
config.WEB_SEARCH_RESULT_COUNT = form_data.web.WEB_SEARCH_RESULT_COUNT
@@ -1411,6 +1419,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
# Web search settings
'web': {
'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH,
'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION,
'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT,
'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE,
'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV,
'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT,
backend/open_webui/tools/builtin.py
+3 -3
View File
@@ -228,10 +228,10 @@ async def search_web(
return json.dumps({'error': 'Request context not available'})
try:
engine = await Config.get('rag.web.search.engine')
engine = await Config.get('web.search.engine')
user = UserModel(**__user__) if __user__ else None
configured = await Config.get('rag.web.search.result_count')
configured = await Config.get('web.search.result_count')
max_count = 5 if configured is None else configured
count = max(1, min(count, max_count)) if count is not None else max_count
@@ -269,7 +269,7 @@ async def fetch_url(
# Truncate if configured (WEB_FETCH_MAX_CONTENT_LENGTH)
# Guard: content may be None if the web loader silently failed
if content is not None:
max_length = await Config.get('rag.web.fetch.max_content_length')
max_length = await Config.get('web.fetch.max_content_length')
if max_length and max_length > 0 and len(content) > max_length:
content = content[:max_length] + '\n\n[Content truncated...]'
else: