diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 4f2743d169..2c9cdb228d 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -421,6 +421,62 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing self.trust_env = trust_env self.playwright_timeout = playwright_timeout + def _intercept_navigation_sync(self, route, request=None): + req = request or route.request + + if req.resource_type != 'document': + route.continue_() + return + + try: + validate_url(req.url) + except Exception: + route.abort() + return + + if AIOHTTP_CLIENT_ALLOW_REDIRECTS: + resp = route.fetch() + else: + try: + resp = route.fetch(max_redirects=0) + except TypeError: + route.abort() + return + + if 300 <= resp.status < 400: + route.abort() + return + + route.fulfill(response=resp) + + async def _intercept_navigation(self, route, request=None): + req = request or route.request + + if req.resource_type != 'document': + await route.continue_() + return + + try: + await run_in_threadpool(validate_url, req.url) + except Exception: + await route.abort() + return + + if AIOHTTP_CLIENT_ALLOW_REDIRECTS: + resp = await route.fetch() + else: + try: + resp = await route.fetch(max_redirects=0) + except TypeError: + await route.abort() + return + + if 300 <= resp.status < 400: + await route.abort() + return + + await route.fulfill(response=resp) + def lazy_load(self) -> Iterator[Document]: """Safely load URLs synchronously with support for remote browser.""" from playwright.sync_api import sync_playwright @@ -436,6 +492,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing try: self._safe_process_url_sync(url) page = browser.new_page() + page.route('**/*', self._intercept_navigation_sync) response = page.goto(url, timeout=self.playwright_timeout) if response is None: raise ValueError(f'page.goto() returned None for url {url}') @@ -465,6 +522,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing try: await self._safe_process_url(url) page = await browser.new_page() + await page.route('**/*', self._intercept_navigation) response = await page.goto(url, timeout=self.playwright_timeout) if response is None: raise ValueError(f'page.goto() returned None for url {url}')