fix: honor Admin UI web loader settings in get_web_loader (#26749)

Since the config refactor, get_web_loader dispatched on the WEB_LOADER_ENGINE module constant, which is read from the environment once at import time. The engine selected in the Admin UI is stored under web.loader.engine in the config table but was never consulted, so UI-configured loader engines (external, playwright, firecrawl, tavily, microsoft_web_iq) were silently ignored and the built-in SafeWebBaseLoader always fetched pages directly. The same applied to the per-engine settings such as the external web loader URL and API key. This breaks egress-restricted deployments that rely on an external web loader: pages are fetched directly from the container and fail with errors like "Network is unreachable" even though an external loader is configured.

Pass the DB-backed loader settings into get_web_loader from both call sites, web search in process_web_search and web fetch via get_loader, and resolve every engine setting from them, keeping the module-level env constants as the fallback for keys that were never saved. Also initialise WebLoaderClass so an unknown engine raises the intended ValueError instead of an UnboundLocalError.

Fixes #26747

backend/open_webui/retrieval/utils.py

@@ -73,6 +73,20 @@ LOADER_CONFIG_KEYS = {
     'web_loader_ssl_verification': 'web.loader.ssl_verification',
     'web_loader_concurrent_requests': 'web.loader.concurrent_requests',
     'web_search_trust_env': 'web.search.trust_env',
+    'web_loader_engine': 'web.loader.engine',
+    'web_loader_timeout': 'web.loader.timeout',
+    'playwright_ws_url': 'web.loader.playwright_ws_url',
+    'playwright_timeout': 'web.loader.playwright_timeout',
+    'firecrawl_api_key': 'web.loader.firecrawl_api_key',
+    'firecrawl_api_url': 'web.loader.firecrawl_api_url',
+    'firecrawl_timeout': 'web.loader.firecrawl_timeout',
+    'tavily_api_key': 'web.search.tavily_api_key',
+    'tavily_extract_depth': 'web.search.tavily_extract_depth',
+    'microsoft_web_iq_api_base_url': 'web.search.microsoft_web_iq_api_base_url',
+    'microsoft_web_iq_api_key': 'web.search.microsoft_web_iq_api_key',
+    'microsoft_web_iq_language': 'web.search.microsoft_web_iq_language',
+    'external_web_loader_url': 'web.loader.external_web_loader_url',
+    'external_web_loader_api_key': 'web.loader.external_web_loader_api_key',
     'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
     'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key',
     'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
@@ -128,6 +142,7 @@ def get_loader(request, url: str, config: dict):
         verify_ssl=config.get('web_loader_ssl_verification'),
         requests_per_second=config.get('web_loader_concurrent_requests'),
         trust_env=config.get('web_search_trust_env'),
+        loader_config=config,
     )
 
 

backend/open_webui/retrieval/web/utils.py

@@ -841,6 +841,7 @@ def get_web_loader(
     verify_ssl: bool = True,
     requests_per_second: int = 2,
     trust_env: bool = False,
+    loader_config: Optional[dict] = None,
 ):
     # Check if the URLs are valid
     safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
@@ -849,6 +850,16 @@ def get_web_loader(
         log.warning(f'All provided URLs were blocked or invalid: {urls}')
         raise ValueError(ERROR_MESSAGES.INVALID_URL)
 
+    loader_config = loader_config or {}
+
+    def cfg(key, env_value):
+        # Admin-saved DB value wins; env constant covers keys never saved.
+        value = loader_config.get(key)
+        return env_value if value is None else value
+
+    engine = cfg('web_loader_engine', WEB_LOADER_ENGINE)
+    web_loader_timeout = cfg('web_loader_timeout', WEB_LOADER_TIMEOUT)
+
     web_loader_args = {
         'web_paths': safe_urls,
         'verify_ssl': verify_ssl,
@@ -857,13 +868,15 @@ def get_web_loader(
         'trust_env': trust_env,
     }
 
-    if WEB_LOADER_ENGINE == '' or WEB_LOADER_ENGINE == 'safe_web':
+    WebLoaderClass = None
+
+    if engine == '' or engine == 'safe_web':
         WebLoaderClass = SafeWebBaseLoader
 
         request_kwargs = {}
-        if WEB_LOADER_TIMEOUT:
+        if web_loader_timeout:
             try:
-                timeout_value = float(WEB_LOADER_TIMEOUT)
+                timeout_value = float(web_loader_timeout)
             except ValueError:
                 timeout_value = None
 
@@ -873,42 +886,44 @@ def get_web_loader(
         if request_kwargs:
             web_loader_args['requests_kwargs'] = request_kwargs
 
-    if WEB_LOADER_ENGINE == 'playwright':
+    if engine == 'playwright':
         WebLoaderClass = SafePlaywrightURLLoader
-        web_loader_args['playwright_timeout'] = PLAYWRIGHT_TIMEOUT
+        web_loader_args['playwright_timeout'] = cfg('playwright_timeout', PLAYWRIGHT_TIMEOUT)
-        if PLAYWRIGHT_WS_URL:
+        playwright_ws_url = cfg('playwright_ws_url', PLAYWRIGHT_WS_URL)
-            web_loader_args['playwright_ws_url'] = PLAYWRIGHT_WS_URL
+        if playwright_ws_url:
+            web_loader_args['playwright_ws_url'] = playwright_ws_url
 
-    if WEB_LOADER_ENGINE == 'firecrawl':
+    if engine == 'firecrawl':
         WebLoaderClass = SafeFireCrawlLoader
-        web_loader_args['api_key'] = FIRECRAWL_API_KEY
+        web_loader_args['api_key'] = cfg('firecrawl_api_key', FIRECRAWL_API_KEY)
-        web_loader_args['api_url'] = FIRECRAWL_API_BASE_URL
+        web_loader_args['api_url'] = cfg('firecrawl_api_url', FIRECRAWL_API_BASE_URL)
-        if FIRECRAWL_TIMEOUT:
+        firecrawl_timeout = cfg('firecrawl_timeout', FIRECRAWL_TIMEOUT)
+        if firecrawl_timeout:
             try:
-                web_loader_args['timeout'] = int(FIRECRAWL_TIMEOUT)
+                web_loader_args['timeout'] = int(firecrawl_timeout)
             except ValueError:
                 pass
 
-    if WEB_LOADER_ENGINE == 'tavily':
+    if engine == 'tavily':
         WebLoaderClass = SafeTavilyLoader
-        web_loader_args['api_key'] = TAVILY_API_KEY
+        web_loader_args['api_key'] = cfg('tavily_api_key', TAVILY_API_KEY)
-        web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH
+        web_loader_args['extract_depth'] = cfg('tavily_extract_depth', TAVILY_EXTRACT_DEPTH)
 
-    if WEB_LOADER_ENGINE == 'microsoft_web_iq':
+    if engine == 'microsoft_web_iq':
         WebLoaderClass = SafeMicrosoftWebIQLoader
-        web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL
+        web_loader_args['api_base_url'] = cfg('microsoft_web_iq_api_base_url', MICROSOFT_WEB_IQ_API_BASE_URL)
-        web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY
+        web_loader_args['api_key'] = cfg('microsoft_web_iq_api_key', MICROSOFT_WEB_IQ_API_KEY)
-        web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE
+        web_loader_args['language'] = cfg('microsoft_web_iq_language', MICROSOFT_WEB_IQ_LANGUAGE)
-        if WEB_LOADER_TIMEOUT:
+        if web_loader_timeout:
             try:
-                web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT)
+                web_loader_args['timeout'] = int(web_loader_timeout)
             except ValueError:
                 pass
 
-    if WEB_LOADER_ENGINE == 'external':
+    if engine == 'external':
         WebLoaderClass = ExternalWebLoader
-        web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL
+        web_loader_args['external_url'] = cfg('external_web_loader_url', EXTERNAL_WEB_LOADER_URL)
-        web_loader_args['external_api_key'] = EXTERNAL_WEB_LOADER_API_KEY
+        web_loader_args['external_api_key'] = cfg('external_web_loader_api_key', EXTERNAL_WEB_LOADER_API_KEY)
 
     if WebLoaderClass:
         web_loader = WebLoaderClass(**web_loader_args)
@@ -922,6 +937,6 @@ def get_web_loader(
         return web_loader
     else:
         raise ValueError(
-            f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. '
+            f'Invalid WEB_LOADER_ENGINE: {engine}. '
             "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'."
         )

backend/open_webui/routers/retrieval.py

@@ -2612,6 +2612,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen
                 verify_ssl=config.ENABLE_WEB_LOADER_SSL_VERIFICATION,
                 requests_per_second=config.WEB_LOADER_CONCURRENT_REQUESTS,
                 trust_env=config.WEB_SEARCH_TRUST_ENV,
+                loader_config=await get_loader_config(),
             )
             docs = await loader.aload()