feat: add OpenSERP self-hosted web search backend (#27437)
Add self-hosted OpenSERP as a web search engine option. OpenSERP provides browser-rendered search across Google, Bing, Yandex, Baidu, DuckDuckGo, and Ecosia with no API keys required. - New module: retrieval/web/openserp.py (async, uses aiohttp session pool) - Config: OPENSERP_BASE_URL env var (defaults to http://localhost:7070) - Routing: search_web() dispatch for 'openserp' engine - Follows existing patterns (searxng, brave) Co-authored-by: crustopher-lgtm <crustopher-lgtm@users.noreply.github.com>
This commit is contained in:
1 parent
42ea8a5a2f
commit
5efe0951d5
3 files changed
+63
No files matched your search
@@ -1176,6 +1176,7 @@ WEB_SEARCH_TRUST_ENV = os.getenv('WEB_SEARCH_TRUST_ENV', 'True').lower() == 'tru
|
|||||||
OLLAMA_CLOUD_WEB_SEARCH_API_KEY = os.getenv('OLLAMA_CLOUD_API_KEY', '')
|
OLLAMA_CLOUD_WEB_SEARCH_API_KEY = os.getenv('OLLAMA_CLOUD_API_KEY', '')
|
||||||
|
|
||||||
SEARXNG_QUERY_URL = os.getenv('SEARXNG_QUERY_URL', '')
|
SEARXNG_QUERY_URL = os.getenv('SEARXNG_QUERY_URL', '')
|
||||||
|
OPENSERP_BASE_URL = os.getenv('OPENSERP_BASE_URL', 'http://localhost:7000')
|
||||||
|
|
||||||
SEARXNG_LANGUAGE = os.getenv('SEARXNG_LANGUAGE', 'all')
|
SEARXNG_LANGUAGE = os.getenv('SEARXNG_LANGUAGE', 'all')
|
||||||
|
|
||||||
@@ -2907,6 +2908,7 @@ DEFAULT_CONFIG = {
|
|||||||
'web.search.trust_env': WEB_SEARCH_TRUST_ENV,
|
'web.search.trust_env': WEB_SEARCH_TRUST_ENV,
|
||||||
'web.search.ollama_cloud_api_key': OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
'web.search.ollama_cloud_api_key': OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
'web.search.searxng_query_url': SEARXNG_QUERY_URL,
|
'web.search.searxng_query_url': SEARXNG_QUERY_URL,
|
||||||
|
'web.search.openserp_base_url': OPENSERP_BASE_URL,
|
||||||
'web.search.searxng_language': SEARXNG_LANGUAGE,
|
'web.search.searxng_language': SEARXNG_LANGUAGE,
|
||||||
'web.search.yacy_query_url': YACY_QUERY_URL,
|
'web.search.yacy_query_url': YACY_QUERY_URL,
|
||||||
'web.search.yacy_username': YACY_USERNAME,
|
'web.search.yacy_username': YACY_USERNAME,
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
|
||||||
|
from open_webui.utils.session_pool import get_session
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def search_openserp(
|
||||||
|
base_url: str,
|
||||||
|
query: str,
|
||||||
|
count: int,
|
||||||
|
filter_list: list[str | None] | None = None,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Query an OpenSERP instance and return normalised results.
|
||||||
|
|
||||||
|
OpenSERP aggregates results from 6 engines (google, bing, yandex,
|
||||||
|
baidu, duckduckgo, ecosia) at once via ``/mega/search``.
|
||||||
|
|
||||||
|
No API key is required -- only a reachable OpenSERP base URL.
|
||||||
|
"""
|
||||||
|
url = f"{base_url.rstrip('/')}/mega/search"
|
||||||
|
params = {"text": query, "limit": count}
|
||||||
|
|
||||||
|
log.debug("searching OpenSERP at %s", url)
|
||||||
|
|
||||||
|
session = await get_session()
|
||||||
|
async with session.get(url, params=params) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = await response.json()
|
||||||
|
|
||||||
|
results = payload.get("results", [])
|
||||||
|
if filter_list:
|
||||||
|
results = get_filtered_results(results, filter_list)
|
||||||
|
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
link=item.get("url", ""),
|
||||||
|
title=item.get("title"),
|
||||||
|
snippet=item.get("snippet"),
|
||||||
|
)
|
||||||
|
for item in results[:count]
|
||||||
|
]
|
||||||
@@ -101,6 +101,7 @@ from open_webui.retrieval.web.ollama import search_ollama_cloud
|
|||||||
from open_webui.retrieval.web.perplexity import search_perplexity
|
from open_webui.retrieval.web.perplexity import search_perplexity
|
||||||
from open_webui.retrieval.web.perplexity_search import search_perplexity_search
|
from open_webui.retrieval.web.perplexity_search import search_perplexity_search
|
||||||
from open_webui.retrieval.web.searchapi import search_searchapi
|
from open_webui.retrieval.web.searchapi import search_searchapi
|
||||||
|
from open_webui.retrieval.web.openserp import search_openserp
|
||||||
from open_webui.retrieval.web.searxng import search_searxng
|
from open_webui.retrieval.web.searxng import search_searxng
|
||||||
from open_webui.retrieval.web.serpapi import search_serpapi
|
from open_webui.retrieval.web.serpapi import search_serpapi
|
||||||
from open_webui.retrieval.web.serper import search_serper
|
from open_webui.retrieval.web.serper import search_serper
|
||||||
@@ -365,6 +366,7 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||||||
'SEARCHAPI_ENGINE': 'web.search.searchapi_engine',
|
'SEARCHAPI_ENGINE': 'web.search.searchapi_engine',
|
||||||
'SEARXNG_LANGUAGE': 'web.search.searxng_language',
|
'SEARXNG_LANGUAGE': 'web.search.searxng_language',
|
||||||
'SEARXNG_QUERY_URL': 'web.search.searxng_query_url',
|
'SEARXNG_QUERY_URL': 'web.search.searxng_query_url',
|
||||||
|
'OPENSERP_BASE_URL': 'web.search.openserp_base_url',
|
||||||
'SERPAPI_API_KEY': 'web.search.serpapi_api_key',
|
'SERPAPI_API_KEY': 'web.search.serpapi_api_key',
|
||||||
'SERPAPI_ENGINE': 'web.search.serpapi_engine',
|
'SERPAPI_ENGINE': 'web.search.serpapi_engine',
|
||||||
'SERPER_API_KEY': 'web.search.serper_api_key',
|
'SERPER_API_KEY': 'web.search.serper_api_key',
|
||||||
@@ -703,6 +705,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL,
|
'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL,
|
||||||
'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE,
|
'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE,
|
||||||
|
'OPENSERP_BASE_URL': config.OPENSERP_BASE_URL,
|
||||||
'YACY_QUERY_URL': config.YACY_QUERY_URL,
|
'YACY_QUERY_URL': config.YACY_QUERY_URL,
|
||||||
'YACY_USERNAME': config.YACY_USERNAME,
|
'YACY_USERNAME': config.YACY_USERNAME,
|
||||||
'YACY_PASSWORD': config.YACY_PASSWORD,
|
'YACY_PASSWORD': config.YACY_PASSWORD,
|
||||||
@@ -781,6 +784,7 @@ class WebConfig(BaseModel):
|
|||||||
OLLAMA_CLOUD_WEB_SEARCH_API_KEY: str | None = None
|
OLLAMA_CLOUD_WEB_SEARCH_API_KEY: str | None = None
|
||||||
SEARXNG_QUERY_URL: str | None = None
|
SEARXNG_QUERY_URL: str | None = None
|
||||||
SEARXNG_LANGUAGE: str | None = None
|
SEARXNG_LANGUAGE: str | None = None
|
||||||
|
OPENSERP_BASE_URL: str | None = None
|
||||||
YACY_QUERY_URL: str | None = None
|
YACY_QUERY_URL: str | None = None
|
||||||
YACY_USERNAME: str | None = None
|
YACY_USERNAME: str | None = None
|
||||||
YACY_PASSWORD: str | None = None
|
YACY_PASSWORD: str | None = None
|
||||||
@@ -1249,6 +1253,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = form_data.web.OLLAMA_CLOUD_WEB_SEARCH_API_KEY
|
config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = form_data.web.OLLAMA_CLOUD_WEB_SEARCH_API_KEY
|
||||||
config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
||||||
config.SEARXNG_LANGUAGE = form_data.web.SEARXNG_LANGUAGE
|
config.SEARXNG_LANGUAGE = form_data.web.SEARXNG_LANGUAGE
|
||||||
|
config.OPENSERP_BASE_URL = form_data.web.OPENSERP_BASE_URL
|
||||||
config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
||||||
config.YACY_USERNAME = form_data.web.YACY_USERNAME
|
config.YACY_USERNAME = form_data.web.YACY_USERNAME
|
||||||
config.YACY_PASSWORD = form_data.web.YACY_PASSWORD
|
config.YACY_PASSWORD = form_data.web.YACY_PASSWORD
|
||||||
@@ -1400,6 +1405,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL,
|
'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL,
|
||||||
'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE,
|
'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE,
|
||||||
|
'OPENSERP_BASE_URL': config.OPENSERP_BASE_URL,
|
||||||
'YACY_QUERY_URL': config.YACY_QUERY_URL,
|
'YACY_QUERY_URL': config.YACY_QUERY_URL,
|
||||||
'YACY_USERNAME': config.YACY_USERNAME,
|
'YACY_USERNAME': config.YACY_USERNAME,
|
||||||
'YACY_PASSWORD': config.YACY_PASSWORD,
|
'YACY_PASSWORD': config.YACY_PASSWORD,
|
||||||
@@ -2212,6 +2218,16 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
raise Exception('No SEARXNG_QUERY_URL found in environment variables')
|
raise Exception('No SEARXNG_QUERY_URL found in environment variables')
|
||||||
|
elif engine == 'openserp':
|
||||||
|
if config.OPENSERP_BASE_URL:
|
||||||
|
return await search_openserp(
|
||||||
|
config.OPENSERP_BASE_URL,
|
||||||
|
query,
|
||||||
|
config.WEB_SEARCH_RESULT_COUNT,
|
||||||
|
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise Exception('No OPENSERP_BASE_URL found in environment variables')
|
||||||
elif engine == 'yacy':
|
elif engine == 'yacy':
|
||||||
if config.YACY_QUERY_URL:
|
if config.YACY_QUERY_URL:
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
|
|||||||
Reference in new issue
Block a user