diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index f72625f721..16a3c2c249 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -68,9 +68,9 @@ def is_youtube_url(url: str) -> bool: LOADER_CONFIG_KEYS = { 'youtube_language': 'rag.youtube_loader_language', 'youtube_proxy_url': 'rag.youtube_loader_proxy_url', - 'web_loader_ssl_verification': 'rag.web.loader.ssl_verification', - 'web_loader_concurrent_requests': 'rag.web.loader.concurrent_requests', - 'web_search_trust_env': 'rag.web.search.trust_env', + 'web_loader_ssl_verification': 'web.loader.ssl_verification', + 'web_loader_concurrent_requests': 'web.loader.concurrent_requests', + 'web_search_trust_env': 'web.search.trust_env', 'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine', 'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key', 'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url', diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index ec19922fa9..080a344a36 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -248,17 +248,17 @@ router = APIRouter() RETRIEVAL_CONFIG_KEYS = { 'ALLOWED_FILE_EXTENSIONS': 'rag.file.allowed_extensions', - 'AZURE_AI_SEARCH_API_KEY': 'rag.web.search.azure_ai_search_api_key', - 'AZURE_AI_SEARCH_ENDPOINT': 'rag.web.search.azure_ai_search_endpoint', - 'AZURE_AI_SEARCH_INDEX_NAME': 'rag.web.search.azure_ai_search_index_name', - 'BING_SEARCH_V7_ENDPOINT': 'rag.web.search.bing_search_v7_endpoint', - 'BING_SEARCH_V7_SUBSCRIPTION_KEY': 'rag.web.search.bing_search_v7_subscription_key', - 'BOCHA_SEARCH_API_KEY': 'rag.web.search.bocha_search_api_key', - 'BRAVE_SEARCH_API_KEY': 'rag.web.search.brave_search_api_key', - 'BRAVE_SEARCH_CONTEXT_TOKENS': 'rag.web.search.brave_search_context_tokens', + 'AZURE_AI_SEARCH_API_KEY': 'web.search.azure_ai_search_api_key', + 'AZURE_AI_SEARCH_ENDPOINT': 'web.search.azure_ai_search_endpoint', + 'AZURE_AI_SEARCH_INDEX_NAME': 'web.search.azure_ai_search_index_name', + 'BING_SEARCH_V7_ENDPOINT': 'web.search.bing_search_v7_endpoint', + 'BING_SEARCH_V7_SUBSCRIPTION_KEY': 'web.search.bing_search_v7_subscription_key', + 'BOCHA_SEARCH_API_KEY': 'web.search.bocha_search_api_key', + 'BRAVE_SEARCH_API_KEY': 'web.search.brave_search_api_key', + 'BRAVE_SEARCH_CONTEXT_TOKENS': 'web.search.brave_search_context_tokens', 'BYPASS_EMBEDDING_AND_RETRIEVAL': 'rag.bypass_embedding_and_retrieval', - 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': 'rag.web.search.bypass_embedding_and_retrieval', - 'BYPASS_WEB_SEARCH_WEB_LOADER': 'rag.web.search.bypass_web_loader', + 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': 'web.search.bypass_embedding_and_retrieval', + 'BYPASS_WEB_SEARCH_WEB_LOADER': 'web.search.bypass_web_loader', 'CHUNK_MIN_SIZE_TARGET': 'rag.chunk_min_size_target', 'CHUNK_OVERLAP': 'rag.chunk_overlap', 'CHUNK_SIZE': 'rag.chunk_size', @@ -274,7 +274,7 @@ RETRIEVAL_CONFIG_KEYS = { 'DATALAB_MARKER_SKIP_CACHE': 'rag.datalab_marker_skip_cache', 'DATALAB_MARKER_STRIP_EXISTING_OCR': 'rag.datalab_marker_strip_existing_ocr', 'DATALAB_MARKER_USE_LLM': 'rag.datalab_marker_use_llm', - 'DDGS_BACKEND': 'rag.web.search.ddgs_backend', + 'DDGS_BACKEND': 'web.search.ddgs_backend', 'DOCLING_API_KEY': 'rag.docling_api_key', 'DOCLING_PARAMS': 'rag.docling_params', 'DOCLING_SERVER_URL': 'rag.docling_server_url', @@ -287,54 +287,56 @@ RETRIEVAL_CONFIG_KEYS = { 'ENABLE_ONEDRIVE_INTEGRATION': 'onedrive.enable', 'ENABLE_RAG_HYBRID_SEARCH': 'rag.enable_hybrid_search', 'ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS': 'rag.enable_hybrid_search_enriched_texts', - 'ENABLE_WEB_LOADER_SSL_VERIFICATION': 'rag.web.loader.ssl_verification', - 'ENABLE_WEB_SEARCH': 'rag.web.search.enable', - 'EXA_API_KEY': 'rag.web.search.exa_api_key', + 'ENABLE_WEB_LOADER_SSL_VERIFICATION': 'web.loader.ssl_verification', + 'ENABLE_WEB_SEARCH': 'web.search.enable', + 'ENABLE_WEB_SEARCH_CONFIRMATION': 'web.search.confirmation.enable', + 'WEB_SEARCH_CONFIRMATION_CONTENT': 'web.search.confirmation.content', + 'EXA_API_KEY': 'web.search.exa_api_key', 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', 'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers', 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', - 'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key', - 'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url', - 'EXTERNAL_WEB_SEARCH_API_KEY': 'rag.web.search.external_web_search_api_key', - 'EXTERNAL_WEB_SEARCH_URL': 'rag.web.search.external_web_search_url', + 'EXTERNAL_WEB_LOADER_API_KEY': 'web.loader.external_web_loader_api_key', + 'EXTERNAL_WEB_LOADER_URL': 'web.loader.external_web_loader_url', + 'EXTERNAL_WEB_SEARCH_API_KEY': 'web.search.external_web_search_api_key', + 'EXTERNAL_WEB_SEARCH_URL': 'web.search.external_web_search_url', 'FILE_IMAGE_COMPRESSION_HEIGHT': 'file.image_compression_height', 'FILE_IMAGE_COMPRESSION_WIDTH': 'file.image_compression_width', 'FILE_MAX_COUNT': 'rag.file.max_count', 'FILE_MAX_SIZE': 'rag.file.max_size', - 'FIRECRAWL_API_BASE_URL': 'rag.web.loader.firecrawl_api_url', - 'FIRECRAWL_API_KEY': 'rag.web.loader.firecrawl_api_key', - 'FIRECRAWL_TIMEOUT': 'rag.web.loader.firecrawl_timeout', - 'GOOGLE_PSE_API_KEY': 'rag.web.search.google_pse_api_key', - 'GOOGLE_PSE_ENGINE_ID': 'rag.web.search.google_pse_engine_id', + 'FIRECRAWL_API_BASE_URL': 'web.loader.firecrawl_api_url', + 'FIRECRAWL_API_KEY': 'web.loader.firecrawl_api_key', + 'FIRECRAWL_TIMEOUT': 'web.loader.firecrawl_timeout', + 'GOOGLE_PSE_API_KEY': 'web.search.google_pse_api_key', + 'GOOGLE_PSE_ENGINE_ID': 'web.search.google_pse_engine_id', 'HYBRID_BM25_WEIGHT': 'rag.hybrid_bm25_weight', - 'JINA_API_BASE_URL': 'rag.web.search.jina_api_base_url', - 'JINA_API_KEY': 'rag.web.search.jina_api_key', - 'KAGI_SEARCH_API_KEY': 'rag.web.search.kagi_search_api_key', - 'LINKUP_API_KEY': 'rag.web.search.linkup_api_key', - 'LINKUP_SEARCH_PARAMS': 'rag.web.search.linkup_search_params', + 'JINA_API_BASE_URL': 'web.search.jina_api_base_url', + 'JINA_API_KEY': 'web.search.jina_api_key', + 'KAGI_SEARCH_API_KEY': 'web.search.kagi_search_api_key', + 'LINKUP_API_KEY': 'web.search.linkup_api_key', + 'LINKUP_SEARCH_PARAMS': 'web.search.linkup_search_params', 'MINERU_API_KEY': 'rag.mineru_api_key', 'MINERU_API_MODE': 'rag.mineru_api_mode', 'MINERU_API_TIMEOUT': 'rag.mineru_api_timeout', 'MINERU_API_URL': 'rag.mineru_api_url', 'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions', 'MINERU_PARAMS': 'rag.mineru_params', - 'MICROSOFT_WEB_IQ_API_BASE_URL': 'rag.web.search.microsoft_web_iq_api_base_url', - 'MICROSOFT_WEB_IQ_API_KEY': 'rag.web.search.microsoft_web_iq_api_key', - 'MICROSOFT_WEB_IQ_LANGUAGE': 'rag.web.search.microsoft_web_iq_language', + 'MICROSOFT_WEB_IQ_API_BASE_URL': 'web.search.microsoft_web_iq_api_base_url', + 'MICROSOFT_WEB_IQ_API_KEY': 'web.search.microsoft_web_iq_api_key', + 'MICROSOFT_WEB_IQ_LANGUAGE': 'web.search.microsoft_web_iq_language', 'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url', 'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key', - 'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key', - 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': 'rag.web.search.ollama_cloud_api_key', + 'MOJEEK_SEARCH_API_KEY': 'web.search.mojeek_search_api_key', + 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': 'web.search.ollama_cloud_api_key', 'PADDLEOCR_VL_BASE_URL': 'rag.paddleocr_vl_base_url', 'PADDLEOCR_VL_TOKEN': 'rag.paddleocr_vl_token', 'PDF_EXTRACT_IMAGES': 'rag.pdf_extract_images', 'PDF_LOADER_MODE': 'rag.pdf_loader_mode', - 'PERPLEXITY_API_KEY': 'rag.web.search.perplexity_api_key', - 'PERPLEXITY_MODEL': 'rag.web.search.perplexity_model', - 'PERPLEXITY_SEARCH_API_URL': 'rag.web.search.perplexity_search_api_url', - 'PERPLEXITY_SEARCH_CONTEXT_USAGE': 'rag.web.search.perplexity_search_context_usage', - 'PLAYWRIGHT_TIMEOUT': 'rag.web.loader.playwright_timeout', - 'PLAYWRIGHT_WS_URL': 'rag.web.loader.playwright_ws_url', + 'PERPLEXITY_API_KEY': 'web.search.perplexity_api_key', + 'PERPLEXITY_MODEL': 'web.search.perplexity_model', + 'PERPLEXITY_SEARCH_API_URL': 'web.search.perplexity_search_api_url', + 'PERPLEXITY_SEARCH_CONTEXT_USAGE': 'web.search.perplexity_search_context_usage', + 'PLAYWRIGHT_TIMEOUT': 'web.loader.playwright_timeout', + 'PLAYWRIGHT_WS_URL': 'web.loader.playwright_ws_url', 'RAG_AZURE_OPENAI_API_KEY': 'rag.azure_openai.api_key', 'RAG_AZURE_OPENAI_API_VERSION': 'rag.azure_openai.api_version', 'RAG_AZURE_OPENAI_BASE_URL': 'rag.azure_openai.base_url', @@ -355,22 +357,22 @@ RETRIEVAL_CONFIG_KEYS = { 'RAG_RERANKING_MODEL': 'rag.reranking_model', 'RAG_TEMPLATE': 'rag.template', 'RELEVANCE_THRESHOLD': 'rag.relevance_threshold', - 'SEARCHAPI_API_KEY': 'rag.web.search.searchapi_api_key', - 'SEARCHAPI_ENGINE': 'rag.web.search.searchapi_engine', - 'SEARXNG_LANGUAGE': 'rag.web.search.searxng_language', - 'SEARXNG_QUERY_URL': 'rag.web.search.searxng_query_url', - 'SERPAPI_API_KEY': 'rag.web.search.serpapi_api_key', - 'SERPAPI_ENGINE': 'rag.web.search.serpapi_engine', - 'SERPER_API_KEY': 'rag.web.search.serper_api_key', - 'SERPHOUSE_API_KEY': 'rag.web.search.serphouse_api_key', - 'SERPHOUSE_DOMAIN': 'rag.web.search.serphouse_domain', - 'SERPLY_API_KEY': 'rag.web.search.serply_api_key', - 'SERPSTACK_API_KEY': 'rag.web.search.serpstack_api_key', - 'SERPSTACK_HTTPS': 'rag.web.search.serpstack_https', - 'SOUGOU_API_SID': 'rag.web.search.sougou_api_sid', - 'SOUGOU_API_SK': 'rag.web.search.sougou_api_sk', - 'TAVILY_API_KEY': 'rag.web.search.tavily_api_key', - 'TAVILY_EXTRACT_DEPTH': 'rag.web.search.tavily_extract_depth', + 'SEARCHAPI_API_KEY': 'web.search.searchapi_api_key', + 'SEARCHAPI_ENGINE': 'web.search.searchapi_engine', + 'SEARXNG_LANGUAGE': 'web.search.searxng_language', + 'SEARXNG_QUERY_URL': 'web.search.searxng_query_url', + 'SERPAPI_API_KEY': 'web.search.serpapi_api_key', + 'SERPAPI_ENGINE': 'web.search.serpapi_engine', + 'SERPER_API_KEY': 'web.search.serper_api_key', + 'SERPHOUSE_API_KEY': 'web.search.serphouse_api_key', + 'SERPHOUSE_DOMAIN': 'web.search.serphouse_domain', + 'SERPLY_API_KEY': 'web.search.serply_api_key', + 'SERPSTACK_API_KEY': 'web.search.serpstack_api_key', + 'SERPSTACK_HTTPS': 'web.search.serpstack_https', + 'SOUGOU_API_SID': 'web.search.sougou_api_sid', + 'SOUGOU_API_SK': 'web.search.sougou_api_sk', + 'TAVILY_API_KEY': 'web.search.tavily_api_key', + 'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth', 'TEXT_SPLITTER': 'rag.text_splitter', 'TIKA_SERVER_URL': 'rag.tika_server_url', 'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name', @@ -378,22 +380,22 @@ RETRIEVAL_CONFIG_KEYS = { 'TOP_K_RERANKER': 'rag.top_k_reranker', 'USER_PERMISSIONS': 'user.permissions', 'WEBUI_URL': 'webui.url', - 'WEB_FETCH_MAX_CONTENT_LENGTH': 'rag.web.fetch.max_content_length', - 'WEB_LOADER_CONCURRENT_REQUESTS': 'rag.web.loader.concurrent_requests', - 'WEB_LOADER_ENGINE': 'rag.web.loader.engine', - 'WEB_LOADER_TIMEOUT': 'rag.web.loader.timeout', - 'WEB_SEARCH_CONCURRENT_REQUESTS': 'rag.web.search.concurrent_requests', - 'WEB_SEARCH_DOMAIN_FILTER_LIST': 'rag.web.search.domain.filter_list', - 'WEB_SEARCH_ENGINE': 'rag.web.search.engine', - 'WEB_SEARCH_RESULT_COUNT': 'rag.web.search.result_count', - 'WEB_SEARCH_TRUST_ENV': 'rag.web.search.trust_env', - 'YACY_PASSWORD': 'rag.web.search.yacy_password', - 'YACY_QUERY_URL': 'rag.web.search.yacy_query_url', - 'YACY_USERNAME': 'rag.web.search.yacy_username', - 'YANDEX_WEB_SEARCH_API_KEY': 'rag.web.search.yandex_web_search_api_key', - 'YANDEX_WEB_SEARCH_CONFIG': 'rag.web.search.yandex_web_search_config', - 'YANDEX_WEB_SEARCH_URL': 'rag.web.search.yandex_web_search_url', - 'YOUCOM_API_KEY': 'rag.web.search.youcom_api_key', + 'WEB_FETCH_MAX_CONTENT_LENGTH': 'web.fetch.max_content_length', + 'WEB_LOADER_CONCURRENT_REQUESTS': 'web.loader.concurrent_requests', + 'WEB_LOADER_ENGINE': 'web.loader.engine', + 'WEB_LOADER_TIMEOUT': 'web.loader.timeout', + 'WEB_SEARCH_CONCURRENT_REQUESTS': 'web.search.concurrent_requests', + 'WEB_SEARCH_DOMAIN_FILTER_LIST': 'web.search.domain.filter_list', + 'WEB_SEARCH_ENGINE': 'web.search.engine', + 'WEB_SEARCH_RESULT_COUNT': 'web.search.result_count', + 'WEB_SEARCH_TRUST_ENV': 'web.search.trust_env', + 'YACY_PASSWORD': 'web.search.yacy_password', + 'YACY_QUERY_URL': 'web.search.yacy_query_url', + 'YACY_USERNAME': 'web.search.yacy_username', + 'YANDEX_WEB_SEARCH_API_KEY': 'web.search.yandex_web_search_api_key', + 'YANDEX_WEB_SEARCH_CONFIG': 'web.search.yandex_web_search_config', + 'YANDEX_WEB_SEARCH_URL': 'web.search.yandex_web_search_url', + 'YOUCOM_API_KEY': 'web.search.youcom_api_key', 'YOUTUBE_LOADER_LANGUAGE': 'rag.youtube_loader_language', 'YOUTUBE_LOADER_PROXY_URL': 'rag.youtube_loader_proxy_url', } @@ -685,6 +687,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): # Web search settings 'web': { 'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH, + 'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION, + 'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT, 'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE, 'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV, 'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT, @@ -761,6 +765,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): class WebConfig(BaseModel): ENABLE_WEB_SEARCH: bool | None = None + ENABLE_WEB_SEARCH_CONFIRMATION: bool | None = None + WEB_SEARCH_CONFIRMATION_CONTENT: str | None = None WEB_SEARCH_ENGINE: str | None = None WEB_SEARCH_TRUST_ENV: bool | None = None WEB_SEARCH_RESULT_COUNT: int | None = None @@ -1262,6 +1268,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend if form_data.web is not None: # Web search settings config.ENABLE_WEB_SEARCH = form_data.web.ENABLE_WEB_SEARCH + config.ENABLE_WEB_SEARCH_CONFIRMATION = form_data.web.ENABLE_WEB_SEARCH_CONFIRMATION + config.WEB_SEARCH_CONFIRMATION_CONTENT = form_data.web.WEB_SEARCH_CONFIRMATION_CONTENT config.WEB_SEARCH_ENGINE = form_data.web.WEB_SEARCH_ENGINE config.WEB_SEARCH_TRUST_ENV = form_data.web.WEB_SEARCH_TRUST_ENV config.WEB_SEARCH_RESULT_COUNT = form_data.web.WEB_SEARCH_RESULT_COUNT @@ -1411,6 +1419,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend # Web search settings 'web': { 'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH, + 'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION, + 'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT, 'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE, 'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV, 'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT, diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 06f34dd890..72bf809661 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -228,10 +228,10 @@ async def search_web( return json.dumps({'error': 'Request context not available'}) try: - engine = await Config.get('rag.web.search.engine') + engine = await Config.get('web.search.engine') user = UserModel(**__user__) if __user__ else None - configured = await Config.get('rag.web.search.result_count') + configured = await Config.get('web.search.result_count') max_count = 5 if configured is None else configured count = max(1, min(count, max_count)) if count is not None else max_count @@ -269,7 +269,7 @@ async def fetch_url( # Truncate if configured (WEB_FETCH_MAX_CONTENT_LENGTH) # Guard: content may be None if the web loader silently failed if content is not None: - max_length = await Config.get('rag.web.fetch.max_content_length') + max_length = await Config.get('web.fetch.max_content_length') if max_length and max_length > 0 and len(content) > max_length: content = content[:max_length] + '\n\n[Content truncated...]' else: