From f7e7f32102635cd1fc77c703eb6c46ebc2848e4c Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 24 Jul 2026 08:30:47 +0200 Subject: [PATCH] fix: honor Admin UI web loader settings in get_web_loader (#26749) Since the config refactor, get_web_loader dispatched on the WEB_LOADER_ENGINE module constant, which is read from the environment once at import time. The engine selected in the Admin UI is stored under web.loader.engine in the config table but was never consulted, so UI-configured loader engines (external, playwright, firecrawl, tavily, microsoft_web_iq) were silently ignored and the built-in SafeWebBaseLoader always fetched pages directly. The same applied to the per-engine settings such as the external web loader URL and API key. This breaks egress-restricted deployments that rely on an external web loader: pages are fetched directly from the container and fail with errors like "Network is unreachable" even though an external loader is configured. Pass the DB-backed loader settings into get_web_loader from both call sites, web search in process_web_search and web fetch via get_loader, and resolve every engine setting from them, keeping the module-level env constants as the fallback for keys that were never saved. Also initialise WebLoaderClass so an unknown engine raises the intended ValueError instead of an UnboundLocalError. Fixes #26747 --- backend/open_webui/retrieval/utils.py | 15 ++++++ backend/open_webui/retrieval/web/utils.py | 65 ++++++++++++++--------- backend/open_webui/routers/retrieval.py | 1 + 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index b402c12e03..f816efb8fc 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -73,6 +73,20 @@ LOADER_CONFIG_KEYS = { 'web_loader_ssl_verification': 'web.loader.ssl_verification', 'web_loader_concurrent_requests': 'web.loader.concurrent_requests', 'web_search_trust_env': 'web.search.trust_env', + 'web_loader_engine': 'web.loader.engine', + 'web_loader_timeout': 'web.loader.timeout', + 'playwright_ws_url': 'web.loader.playwright_ws_url', + 'playwright_timeout': 'web.loader.playwright_timeout', + 'firecrawl_api_key': 'web.loader.firecrawl_api_key', + 'firecrawl_api_url': 'web.loader.firecrawl_api_url', + 'firecrawl_timeout': 'web.loader.firecrawl_timeout', + 'tavily_api_key': 'web.search.tavily_api_key', + 'tavily_extract_depth': 'web.search.tavily_extract_depth', + 'microsoft_web_iq_api_base_url': 'web.search.microsoft_web_iq_api_base_url', + 'microsoft_web_iq_api_key': 'web.search.microsoft_web_iq_api_key', + 'microsoft_web_iq_language': 'web.search.microsoft_web_iq_language', + 'external_web_loader_url': 'web.loader.external_web_loader_url', + 'external_web_loader_api_key': 'web.loader.external_web_loader_api_key', 'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine', 'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key', 'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url', @@ -128,6 +142,7 @@ def get_loader(request, url: str, config: dict): verify_ssl=config.get('web_loader_ssl_verification'), requests_per_second=config.get('web_loader_concurrent_requests'), trust_env=config.get('web_search_trust_env'), + loader_config=config, ) diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 34be1db167..22fe7d98ae 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -841,6 +841,7 @@ def get_web_loader( verify_ssl: bool = True, requests_per_second: int = 2, trust_env: bool = False, + loader_config: Optional[dict] = None, ): # Check if the URLs are valid safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls) @@ -849,6 +850,16 @@ def get_web_loader( log.warning(f'All provided URLs were blocked or invalid: {urls}') raise ValueError(ERROR_MESSAGES.INVALID_URL) + loader_config = loader_config or {} + + def cfg(key, env_value): + # Admin-saved DB value wins; env constant covers keys never saved. + value = loader_config.get(key) + return env_value if value is None else value + + engine = cfg('web_loader_engine', WEB_LOADER_ENGINE) + web_loader_timeout = cfg('web_loader_timeout', WEB_LOADER_TIMEOUT) + web_loader_args = { 'web_paths': safe_urls, 'verify_ssl': verify_ssl, @@ -857,13 +868,15 @@ def get_web_loader( 'trust_env': trust_env, } - if WEB_LOADER_ENGINE == '' or WEB_LOADER_ENGINE == 'safe_web': + WebLoaderClass = None + + if engine == '' or engine == 'safe_web': WebLoaderClass = SafeWebBaseLoader request_kwargs = {} - if WEB_LOADER_TIMEOUT: + if web_loader_timeout: try: - timeout_value = float(WEB_LOADER_TIMEOUT) + timeout_value = float(web_loader_timeout) except ValueError: timeout_value = None @@ -873,42 +886,44 @@ def get_web_loader( if request_kwargs: web_loader_args['requests_kwargs'] = request_kwargs - if WEB_LOADER_ENGINE == 'playwright': + if engine == 'playwright': WebLoaderClass = SafePlaywrightURLLoader - web_loader_args['playwright_timeout'] = PLAYWRIGHT_TIMEOUT - if PLAYWRIGHT_WS_URL: - web_loader_args['playwright_ws_url'] = PLAYWRIGHT_WS_URL + web_loader_args['playwright_timeout'] = cfg('playwright_timeout', PLAYWRIGHT_TIMEOUT) + playwright_ws_url = cfg('playwright_ws_url', PLAYWRIGHT_WS_URL) + if playwright_ws_url: + web_loader_args['playwright_ws_url'] = playwright_ws_url - if WEB_LOADER_ENGINE == 'firecrawl': + if engine == 'firecrawl': WebLoaderClass = SafeFireCrawlLoader - web_loader_args['api_key'] = FIRECRAWL_API_KEY - web_loader_args['api_url'] = FIRECRAWL_API_BASE_URL - if FIRECRAWL_TIMEOUT: + web_loader_args['api_key'] = cfg('firecrawl_api_key', FIRECRAWL_API_KEY) + web_loader_args['api_url'] = cfg('firecrawl_api_url', FIRECRAWL_API_BASE_URL) + firecrawl_timeout = cfg('firecrawl_timeout', FIRECRAWL_TIMEOUT) + if firecrawl_timeout: try: - web_loader_args['timeout'] = int(FIRECRAWL_TIMEOUT) + web_loader_args['timeout'] = int(firecrawl_timeout) except ValueError: pass - if WEB_LOADER_ENGINE == 'tavily': + if engine == 'tavily': WebLoaderClass = SafeTavilyLoader - web_loader_args['api_key'] = TAVILY_API_KEY - web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH + web_loader_args['api_key'] = cfg('tavily_api_key', TAVILY_API_KEY) + web_loader_args['extract_depth'] = cfg('tavily_extract_depth', TAVILY_EXTRACT_DEPTH) - if WEB_LOADER_ENGINE == 'microsoft_web_iq': + if engine == 'microsoft_web_iq': WebLoaderClass = SafeMicrosoftWebIQLoader - web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL - web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY - web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE - if WEB_LOADER_TIMEOUT: + web_loader_args['api_base_url'] = cfg('microsoft_web_iq_api_base_url', MICROSOFT_WEB_IQ_API_BASE_URL) + web_loader_args['api_key'] = cfg('microsoft_web_iq_api_key', MICROSOFT_WEB_IQ_API_KEY) + web_loader_args['language'] = cfg('microsoft_web_iq_language', MICROSOFT_WEB_IQ_LANGUAGE) + if web_loader_timeout: try: - web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT) + web_loader_args['timeout'] = int(web_loader_timeout) except ValueError: pass - if WEB_LOADER_ENGINE == 'external': + if engine == 'external': WebLoaderClass = ExternalWebLoader - web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL - web_loader_args['external_api_key'] = EXTERNAL_WEB_LOADER_API_KEY + web_loader_args['external_url'] = cfg('external_web_loader_url', EXTERNAL_WEB_LOADER_URL) + web_loader_args['external_api_key'] = cfg('external_web_loader_api_key', EXTERNAL_WEB_LOADER_API_KEY) if WebLoaderClass: web_loader = WebLoaderClass(**web_loader_args) @@ -922,6 +937,6 @@ def get_web_loader( return web_loader else: raise ValueError( - f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. ' + f'Invalid WEB_LOADER_ENGINE: {engine}. ' "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'." ) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 5ca0e464dd..9116dc80de 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -2612,6 +2612,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen verify_ssl=config.ENABLE_WEB_LOADER_SSL_VERIFICATION, requests_per_second=config.WEB_LOADER_CONCURRENT_REQUESTS, trust_env=config.WEB_SEARCH_TRUST_ENV, + loader_config=await get_loader_config(), ) docs = await loader.aload()