diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index b402c12e03..f816efb8fc 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -73,6 +73,20 @@ LOADER_CONFIG_KEYS = { 'web_loader_ssl_verification': 'web.loader.ssl_verification', 'web_loader_concurrent_requests': 'web.loader.concurrent_requests', 'web_search_trust_env': 'web.search.trust_env', + 'web_loader_engine': 'web.loader.engine', + 'web_loader_timeout': 'web.loader.timeout', + 'playwright_ws_url': 'web.loader.playwright_ws_url', + 'playwright_timeout': 'web.loader.playwright_timeout', + 'firecrawl_api_key': 'web.loader.firecrawl_api_key', + 'firecrawl_api_url': 'web.loader.firecrawl_api_url', + 'firecrawl_timeout': 'web.loader.firecrawl_timeout', + 'tavily_api_key': 'web.search.tavily_api_key', + 'tavily_extract_depth': 'web.search.tavily_extract_depth', + 'microsoft_web_iq_api_base_url': 'web.search.microsoft_web_iq_api_base_url', + 'microsoft_web_iq_api_key': 'web.search.microsoft_web_iq_api_key', + 'microsoft_web_iq_language': 'web.search.microsoft_web_iq_language', + 'external_web_loader_url': 'web.loader.external_web_loader_url', + 'external_web_loader_api_key': 'web.loader.external_web_loader_api_key', 'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine', 'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key', 'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url', @@ -128,6 +142,7 @@ def get_loader(request, url: str, config: dict): verify_ssl=config.get('web_loader_ssl_verification'), requests_per_second=config.get('web_loader_concurrent_requests'), trust_env=config.get('web_search_trust_env'), + loader_config=config, ) diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 34be1db167..22fe7d98ae 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -841,6 +841,7 @@ def get_web_loader( verify_ssl: bool = True, requests_per_second: int = 2, trust_env: bool = False, + loader_config: Optional[dict] = None, ): # Check if the URLs are valid safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls) @@ -849,6 +850,16 @@ def get_web_loader( log.warning(f'All provided URLs were blocked or invalid: {urls}') raise ValueError(ERROR_MESSAGES.INVALID_URL) + loader_config = loader_config or {} + + def cfg(key, env_value): + # Admin-saved DB value wins; env constant covers keys never saved. + value = loader_config.get(key) + return env_value if value is None else value + + engine = cfg('web_loader_engine', WEB_LOADER_ENGINE) + web_loader_timeout = cfg('web_loader_timeout', WEB_LOADER_TIMEOUT) + web_loader_args = { 'web_paths': safe_urls, 'verify_ssl': verify_ssl, @@ -857,13 +868,15 @@ def get_web_loader( 'trust_env': trust_env, } - if WEB_LOADER_ENGINE == '' or WEB_LOADER_ENGINE == 'safe_web': + WebLoaderClass = None + + if engine == '' or engine == 'safe_web': WebLoaderClass = SafeWebBaseLoader request_kwargs = {} - if WEB_LOADER_TIMEOUT: + if web_loader_timeout: try: - timeout_value = float(WEB_LOADER_TIMEOUT) + timeout_value = float(web_loader_timeout) except ValueError: timeout_value = None @@ -873,42 +886,44 @@ def get_web_loader( if request_kwargs: web_loader_args['requests_kwargs'] = request_kwargs - if WEB_LOADER_ENGINE == 'playwright': + if engine == 'playwright': WebLoaderClass = SafePlaywrightURLLoader - web_loader_args['playwright_timeout'] = PLAYWRIGHT_TIMEOUT - if PLAYWRIGHT_WS_URL: - web_loader_args['playwright_ws_url'] = PLAYWRIGHT_WS_URL + web_loader_args['playwright_timeout'] = cfg('playwright_timeout', PLAYWRIGHT_TIMEOUT) + playwright_ws_url = cfg('playwright_ws_url', PLAYWRIGHT_WS_URL) + if playwright_ws_url: + web_loader_args['playwright_ws_url'] = playwright_ws_url - if WEB_LOADER_ENGINE == 'firecrawl': + if engine == 'firecrawl': WebLoaderClass = SafeFireCrawlLoader - web_loader_args['api_key'] = FIRECRAWL_API_KEY - web_loader_args['api_url'] = FIRECRAWL_API_BASE_URL - if FIRECRAWL_TIMEOUT: + web_loader_args['api_key'] = cfg('firecrawl_api_key', FIRECRAWL_API_KEY) + web_loader_args['api_url'] = cfg('firecrawl_api_url', FIRECRAWL_API_BASE_URL) + firecrawl_timeout = cfg('firecrawl_timeout', FIRECRAWL_TIMEOUT) + if firecrawl_timeout: try: - web_loader_args['timeout'] = int(FIRECRAWL_TIMEOUT) + web_loader_args['timeout'] = int(firecrawl_timeout) except ValueError: pass - if WEB_LOADER_ENGINE == 'tavily': + if engine == 'tavily': WebLoaderClass = SafeTavilyLoader - web_loader_args['api_key'] = TAVILY_API_KEY - web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH + web_loader_args['api_key'] = cfg('tavily_api_key', TAVILY_API_KEY) + web_loader_args['extract_depth'] = cfg('tavily_extract_depth', TAVILY_EXTRACT_DEPTH) - if WEB_LOADER_ENGINE == 'microsoft_web_iq': + if engine == 'microsoft_web_iq': WebLoaderClass = SafeMicrosoftWebIQLoader - web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL - web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY - web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE - if WEB_LOADER_TIMEOUT: + web_loader_args['api_base_url'] = cfg('microsoft_web_iq_api_base_url', MICROSOFT_WEB_IQ_API_BASE_URL) + web_loader_args['api_key'] = cfg('microsoft_web_iq_api_key', MICROSOFT_WEB_IQ_API_KEY) + web_loader_args['language'] = cfg('microsoft_web_iq_language', MICROSOFT_WEB_IQ_LANGUAGE) + if web_loader_timeout: try: - web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT) + web_loader_args['timeout'] = int(web_loader_timeout) except ValueError: pass - if WEB_LOADER_ENGINE == 'external': + if engine == 'external': WebLoaderClass = ExternalWebLoader - web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL - web_loader_args['external_api_key'] = EXTERNAL_WEB_LOADER_API_KEY + web_loader_args['external_url'] = cfg('external_web_loader_url', EXTERNAL_WEB_LOADER_URL) + web_loader_args['external_api_key'] = cfg('external_web_loader_api_key', EXTERNAL_WEB_LOADER_API_KEY) if WebLoaderClass: web_loader = WebLoaderClass(**web_loader_args) @@ -922,6 +937,6 @@ def get_web_loader( return web_loader else: raise ValueError( - f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. ' + f'Invalid WEB_LOADER_ENGINE: {engine}. ' "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'." ) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 5ca0e464dd..9116dc80de 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -2612,6 +2612,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen verify_ssl=config.ENABLE_WEB_LOADER_SSL_VERIFICATION, requests_per_second=config.WEB_LOADER_CONCURRENT_REQUESTS, trust_env=config.WEB_SEARCH_TRUST_ENV, + loader_config=await get_loader_config(), ) docs = await loader.aload()