enh: search_ollama_cloud
This commit is contained in:
1 parent
f27ab14bd8
commit
e06489d92b
5 files changed
+79
-1
No files matched your search
@@ -2767,6 +2767,12 @@ WEB_SEARCH_TRUST_ENV = PersistentConfig(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
OLLAMA_CLOUD_WEB_SEARCH_API_KEY = PersistentConfig(
|
||||||
|
"OLLAMA_CLOUD_WEB_SEARCH_API_KEY",
|
||||||
|
"rag.web.search.ollama_cloud_api_key",
|
||||||
|
os.getenv("OLLAMA_CLOUD_API_KEY", ""),
|
||||||
|
)
|
||||||
|
|
||||||
SEARXNG_QUERY_URL = PersistentConfig(
|
SEARXNG_QUERY_URL = PersistentConfig(
|
||||||
"SEARXNG_QUERY_URL",
|
"SEARXNG_QUERY_URL",
|
||||||
"rag.web.search.searxng_query_url",
|
"rag.web.search.searxng_query_url",
|
||||||
|
|||||||
@@ -269,6 +269,7 @@ from open_webui.config import (
|
|||||||
WEB_SEARCH_CONCURRENT_REQUESTS,
|
WEB_SEARCH_CONCURRENT_REQUESTS,
|
||||||
WEB_SEARCH_TRUST_ENV,
|
WEB_SEARCH_TRUST_ENV,
|
||||||
WEB_SEARCH_DOMAIN_FILTER_LIST,
|
WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
JINA_API_KEY,
|
JINA_API_KEY,
|
||||||
SEARCHAPI_API_KEY,
|
SEARCHAPI_API_KEY,
|
||||||
SEARCHAPI_ENGINE,
|
SEARCHAPI_ENGINE,
|
||||||
@@ -883,6 +884,8 @@ app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER = BYPASS_WEB_SEARCH_WEB_LOADER
|
|||||||
|
|
||||||
app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION
|
app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION
|
||||||
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
|
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
|
||||||
|
|
||||||
|
app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = OLLAMA_CLOUD_WEB_SEARCH_API_KEY
|
||||||
app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
|
app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
|
||||||
app.state.config.YACY_QUERY_URL = YACY_QUERY_URL
|
app.state.config.YACY_QUERY_URL = YACY_QUERY_URL
|
||||||
app.state.config.YACY_USERNAME = YACY_USERNAME
|
app.state.config.YACY_USERNAME = YACY_USERNAME
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from open_webui.env import SRC_LOG_LEVELS
|
||||||
|
from open_webui.retrieval.web.main import SearchResult
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
log.setLevel(SRC_LOG_LEVELS["RAG"])
|
||||||
|
|
||||||
|
|
||||||
|
def search_ollama_cloud(
|
||||||
|
url: str,
|
||||||
|
api_key: str,
|
||||||
|
query: str,
|
||||||
|
count: int,
|
||||||
|
filter_list: Optional[list[str]] = None,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search using Ollama Search API and return the results as a list of SearchResult objects.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
api_key (str): A Ollama Search API key
|
||||||
|
query (str): The query to search for
|
||||||
|
count (int): Number of results to return
|
||||||
|
filter_list (Optional[list[str]]): List of domains to filter results by
|
||||||
|
"""
|
||||||
|
log.info(f"Searching with Ollama for query: {query}")
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"query": query,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(f"{url}/api/web_search", headers=headers, json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
results = data.get("results", [])
|
||||||
|
log.info(f"Found {len(results)} results")
|
||||||
|
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
link=result.get("url", ""),
|
||||||
|
title=result.get("title", ""),
|
||||||
|
snippet=result.get("content", ""),
|
||||||
|
)
|
||||||
|
for result in results
|
||||||
|
]
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"Error searching Ollama: {e}")
|
||||||
|
return []
|
||||||
@@ -45,6 +45,7 @@ from open_webui.retrieval.loaders.youtube import YoutubeLoader
|
|||||||
# Web search engines
|
# Web search engines
|
||||||
from open_webui.retrieval.web.main import SearchResult
|
from open_webui.retrieval.web.main import SearchResult
|
||||||
from open_webui.retrieval.web.utils import get_web_loader
|
from open_webui.retrieval.web.utils import get_web_loader
|
||||||
|
from open_webui.retrieval.web.ollama import search_ollama
|
||||||
from open_webui.retrieval.web.brave import search_brave
|
from open_webui.retrieval.web.brave import search_brave
|
||||||
from open_webui.retrieval.web.kagi import search_kagi
|
from open_webui.retrieval.web.kagi import search_kagi
|
||||||
from open_webui.retrieval.web.mojeek import search_mojeek
|
from open_webui.retrieval.web.mojeek import search_mojeek
|
||||||
@@ -469,6 +470,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
"WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
"WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
||||||
"BYPASS_WEB_SEARCH_WEB_LOADER": request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER,
|
"BYPASS_WEB_SEARCH_WEB_LOADER": request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER,
|
||||||
|
"OLLAMA_CLOUD_WEB_SEARCH_API_KEY": request.app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
||||||
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
||||||
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
||||||
@@ -525,6 +527,7 @@ class WebConfig(BaseModel):
|
|||||||
WEB_SEARCH_DOMAIN_FILTER_LIST: Optional[List[str]] = []
|
WEB_SEARCH_DOMAIN_FILTER_LIST: Optional[List[str]] = []
|
||||||
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: Optional[bool] = None
|
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: Optional[bool] = None
|
||||||
BYPASS_WEB_SEARCH_WEB_LOADER: Optional[bool] = None
|
BYPASS_WEB_SEARCH_WEB_LOADER: Optional[bool] = None
|
||||||
|
OLLAMA_CLOUD_WEB_SEARCH_API_KEY: Optional[str] = None
|
||||||
SEARXNG_QUERY_URL: Optional[str] = None
|
SEARXNG_QUERY_URL: Optional[str] = None
|
||||||
YACY_QUERY_URL: Optional[str] = None
|
YACY_QUERY_URL: Optional[str] = None
|
||||||
YACY_USERNAME: Optional[str] = None
|
YACY_USERNAME: Optional[str] = None
|
||||||
@@ -988,6 +991,9 @@ async def update_rag_config(
|
|||||||
request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER = (
|
request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER = (
|
||||||
form_data.web.BYPASS_WEB_SEARCH_WEB_LOADER
|
form_data.web.BYPASS_WEB_SEARCH_WEB_LOADER
|
||||||
)
|
)
|
||||||
|
request.app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = (
|
||||||
|
form_data.web.OLLAMA_CLOUD_WEB_SEARCH_API_KEY
|
||||||
|
)
|
||||||
request.app.state.config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
request.app.state.config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
||||||
request.app.state.config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
request.app.state.config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
||||||
request.app.state.config.YACY_USERNAME = form_data.web.YACY_USERNAME
|
request.app.state.config.YACY_USERNAME = form_data.web.YACY_USERNAME
|
||||||
@@ -1139,6 +1145,7 @@ async def update_rag_config(
|
|||||||
"WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
"WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
||||||
"BYPASS_WEB_SEARCH_WEB_LOADER": request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER,
|
"BYPASS_WEB_SEARCH_WEB_LOADER": request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER,
|
||||||
|
"OLLAMA_CLOUD_WEB_SEARCH_API_KEY": request.app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
||||||
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
||||||
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
||||||
@@ -1786,7 +1793,15 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# TODO: add playwright to search the web
|
# TODO: add playwright to search the web
|
||||||
if engine == "searxng":
|
if engine == "ollama_cloud":
|
||||||
|
return search_ollama(
|
||||||
|
"https://ollama.com",
|
||||||
|
request.app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY,
|
||||||
|
query,
|
||||||
|
request.app.state.config.WEB_SEARCH_RESULT_COUNT,
|
||||||
|
request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
)
|
||||||
|
elif engine == "searxng":
|
||||||
if request.app.state.config.SEARXNG_QUERY_URL:
|
if request.app.state.config.SEARXNG_QUERY_URL:
|
||||||
return search_searxng(
|
return search_searxng(
|
||||||
request.app.state.config.SEARXNG_QUERY_URL,
|
request.app.state.config.SEARXNG_QUERY_URL,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
export let saveHandler: Function;
|
export let saveHandler: Function;
|
||||||
|
|
||||||
let webSearchEngines = [
|
let webSearchEngines = [
|
||||||
|
'ollama_cloud',
|
||||||
'searxng',
|
'searxng',
|
||||||
'yacy',
|
'yacy',
|
||||||
'google_pse',
|
'google_pse',
|
||||||
|
|||||||
Reference in new issue
Block a user