087878ce84
is_string_allowed does endswith() matching and was called with the full URL (retrieval/web/utils.py) against WEB_FETCH_FILTER_LIST, so a blocklisted host with any path (https://blocked.example/x) ended with /x, not the host, and slipped through; the allowlist direction false-rejected legitimate URLs and admitted attacker URLs ending in an allowed string. The same endswith caused label confusion at the hostname call site (retrieval/web/main.py): corp.com matched evilcorp.com, 10.0.0.1 matched 110.0.0.1. Add is_host_allowed(host, ...) matching on DNS label boundaries (host == pattern or host.endswith('.' + pattern)), called with the parsed hostname at both web-fetch call sites. is_string_allowed is left unchanged for the unrelated function-name filters (utils/middleware.py, utils/tools.py). The separate is_global guard (validate_url / _ssrf_safe_new_conn, active when ENABLE_RAG_LOCAL_WEB_FETCH is off) already blocks RFC1918/loopback/link-local, so this restores the admin's intended blocking of specific public hosts. Co-authored-by: addcontent <59762500+addcontent@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
46 lines
1.1 KiB
Python
46 lines
1.1 KiB
Python
from __future__ import annotations
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
import validators
|
|
from open_webui.retrieval.web.utils import resolve_hostname
|
|
from open_webui.utils.misc import is_host_allowed
|
|
from pydantic import BaseModel
|
|
|
|
|
|
def get_filtered_results(results, filter_list):
|
|
if not filter_list:
|
|
return results
|
|
|
|
filtered_results = []
|
|
|
|
for result in results:
|
|
url = result.get('url') or result.get('link', '') or result.get('href', '')
|
|
if not validators.url(url):
|
|
continue
|
|
|
|
domain = urlparse(url).netloc
|
|
if not domain:
|
|
continue
|
|
|
|
hostnames = [domain]
|
|
|
|
try:
|
|
ipv4_addresses, ipv6_addresses = resolve_hostname(domain)
|
|
hostnames.extend(ipv4_addresses)
|
|
hostnames.extend(ipv6_addresses)
|
|
except Exception:
|
|
pass
|
|
|
|
if is_host_allowed(hostnames, filter_list):
|
|
filtered_results.append(result)
|
|
continue
|
|
|
|
return filtered_results
|
|
|
|
|
|
class SearchResult(BaseModel):
|
|
link: str
|
|
title: str | None
|
|
snippet: str | None
|