refac
This commit is contained in:
1 parent
b7934e9182
commit
ce4a323f43
6 files changed
+384
-2
No files matched your search
@@ -1179,6 +1179,12 @@ PERPLEXITY_SEARCH_CONTEXT_USAGE = os.getenv('PERPLEXITY_SEARCH_CONTEXT_USAGE', '
|
|||||||
|
|
||||||
PERPLEXITY_SEARCH_API_URL = os.getenv('PERPLEXITY_SEARCH_API_URL', 'https://api.perplexity.ai/search')
|
PERPLEXITY_SEARCH_API_URL = os.getenv('PERPLEXITY_SEARCH_API_URL', 'https://api.perplexity.ai/search')
|
||||||
|
|
||||||
|
MICROSOFT_WEB_IQ_API_BASE_URL = os.getenv('MICROSOFT_WEB_IQ_API_BASE_URL', 'https://api.microsoft.ai/v3')
|
||||||
|
|
||||||
|
MICROSOFT_WEB_IQ_API_KEY = os.getenv('MICROSOFT_WEB_IQ_API_KEY', '')
|
||||||
|
|
||||||
|
MICROSOFT_WEB_IQ_LANGUAGE = os.getenv('MICROSOFT_WEB_IQ_LANGUAGE', 'en')
|
||||||
|
|
||||||
SOUGOU_API_SID = os.getenv('SOUGOU_API_SID', '')
|
SOUGOU_API_SID = os.getenv('SOUGOU_API_SID', '')
|
||||||
|
|
||||||
SOUGOU_API_SK = os.getenv('SOUGOU_API_SK', '')
|
SOUGOU_API_SK = os.getenv('SOUGOU_API_SK', '')
|
||||||
@@ -2805,6 +2811,9 @@ DEFAULT_CONFIG = {
|
|||||||
'rag.web.search.perplexity_model': PERPLEXITY_MODEL,
|
'rag.web.search.perplexity_model': PERPLEXITY_MODEL,
|
||||||
'rag.web.search.perplexity_search_context_usage': PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
'rag.web.search.perplexity_search_context_usage': PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
||||||
'rag.web.search.perplexity_search_api_url': PERPLEXITY_SEARCH_API_URL,
|
'rag.web.search.perplexity_search_api_url': PERPLEXITY_SEARCH_API_URL,
|
||||||
|
'rag.web.search.microsoft_web_iq_api_base_url': MICROSOFT_WEB_IQ_API_BASE_URL,
|
||||||
|
'rag.web.search.microsoft_web_iq_api_key': MICROSOFT_WEB_IQ_API_KEY,
|
||||||
|
'rag.web.search.microsoft_web_iq_language': MICROSOFT_WEB_IQ_LANGUAGE,
|
||||||
'rag.web.search.sougou_api_sid': SOUGOU_API_SID,
|
'rag.web.search.sougou_api_sid': SOUGOU_API_SID,
|
||||||
'rag.web.search.sougou_api_sk': SOUGOU_API_SK,
|
'rag.web.search.sougou_api_sk': SOUGOU_API_SK,
|
||||||
'rag.web.search.tavily_api_key': TAVILY_API_KEY,
|
'rag.web.search.tavily_api_key': TAVILY_API_KEY,
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from langchain_core.document_loaders import BaseLoader
|
||||||
|
from langchain_core.documents import Document
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3'
|
||||||
|
MICROSOFT_BROWSE_RETRY_STATUS_CODES = {202, 429, 500, 502, 503, 504}
|
||||||
|
MICROSOFT_BROWSE_MAX_RETRIES = 2
|
||||||
|
|
||||||
|
|
||||||
|
class MicrosoftWebIQLoader(BaseLoader):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
urls: str | list[str],
|
||||||
|
api_base_url: str,
|
||||||
|
api_key: str,
|
||||||
|
language: str = 'en',
|
||||||
|
verify_ssl: bool = True,
|
||||||
|
timeout: Any = None,
|
||||||
|
continue_on_failure: bool = True,
|
||||||
|
) -> None:
|
||||||
|
self.urls = urls if isinstance(urls, list) else [urls]
|
||||||
|
self.api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/')
|
||||||
|
self.api_key = api_key
|
||||||
|
self.language = language
|
||||||
|
self.verify_ssl = verify_ssl
|
||||||
|
self.timeout = timeout
|
||||||
|
self.continue_on_failure = continue_on_failure
|
||||||
|
|
||||||
|
def lazy_load(self) -> Iterator[Document]:
|
||||||
|
for url in self.urls:
|
||||||
|
try:
|
||||||
|
doc = self._browse_url(url)
|
||||||
|
if doc is not None:
|
||||||
|
yield doc
|
||||||
|
except Exception as e:
|
||||||
|
if self.continue_on_failure:
|
||||||
|
log.warning(f'Error browsing {url} with Microsoft Web IQ: {e}')
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
def _browse_url(self, url: str) -> Document | None:
|
||||||
|
headers = {
|
||||||
|
'host': urlparse(self.api_base_url).netloc or 'api.microsoft.ai',
|
||||||
|
'x-apikey': self.api_key,
|
||||||
|
'content-type': 'application/json',
|
||||||
|
}
|
||||||
|
payload = {
|
||||||
|
'url': url,
|
||||||
|
'contentFormat': 'markdown',
|
||||||
|
'liveCrawl': 'fallback',
|
||||||
|
'renderDynamicPages': True,
|
||||||
|
'language': self.language,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
request_timeout = float(self.timeout)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
request_timeout = 60
|
||||||
|
request_timeout = request_timeout if request_timeout > 0 else 60
|
||||||
|
|
||||||
|
data: dict[str, Any] = {}
|
||||||
|
for attempt in range(MICROSOFT_BROWSE_MAX_RETRIES + 1):
|
||||||
|
response = requests.post(
|
||||||
|
f'{self.api_base_url}/browse',
|
||||||
|
json=payload,
|
||||||
|
headers=headers,
|
||||||
|
timeout=request_timeout,
|
||||||
|
verify=self.verify_ssl,
|
||||||
|
)
|
||||||
|
|
||||||
|
if response.status_code in MICROSOFT_BROWSE_RETRY_STATUS_CODES and attempt < MICROSOFT_BROWSE_MAX_RETRIES:
|
||||||
|
try:
|
||||||
|
body = response.json()
|
||||||
|
except Exception:
|
||||||
|
body = {}
|
||||||
|
retry_after = body.get('retryAfter') if isinstance(body, dict) else None
|
||||||
|
retry_after = retry_after or response.headers.get('Retry-After')
|
||||||
|
try:
|
||||||
|
delay = min(10.0, max(0.0, float(str(retry_after).rstrip('s'))))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
delay = min(8.0, float(2**attempt))
|
||||||
|
log.warning(
|
||||||
|
'Microsoft Browse %s returned HTTP %s; retrying in %.1fs',
|
||||||
|
url,
|
||||||
|
response.status_code,
|
||||||
|
delay,
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
continue
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
break
|
||||||
|
|
||||||
|
content = data.get('content') or ''
|
||||||
|
if not isinstance(content, str) or not content.strip():
|
||||||
|
return None
|
||||||
|
|
||||||
|
metadata = {'source': data.get('url') or url}
|
||||||
|
if data.get('title'):
|
||||||
|
metadata['title'] = data['title']
|
||||||
|
|
||||||
|
return Document(page_content=content, metadata=metadata)
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
|
||||||
|
from open_webui.utils.headers import include_user_info_headers
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3'
|
||||||
|
|
||||||
|
|
||||||
|
def search_microsoft_web_iq(
|
||||||
|
api_base_url: str,
|
||||||
|
api_key: str,
|
||||||
|
query: str,
|
||||||
|
count: int,
|
||||||
|
filter_list: list[str | None] | None = None,
|
||||||
|
language: str = 'en',
|
||||||
|
user=None,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
try:
|
||||||
|
api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/')
|
||||||
|
headers = {
|
||||||
|
'host': urlparse(api_base_url).netloc or 'api.microsoft.ai',
|
||||||
|
'x-apikey': api_key,
|
||||||
|
'content-type': 'application/json',
|
||||||
|
}
|
||||||
|
if user is not None:
|
||||||
|
headers = include_user_info_headers(headers, user)
|
||||||
|
|
||||||
|
response = requests.post(
|
||||||
|
f'{api_base_url}/search/web',
|
||||||
|
json={
|
||||||
|
'query': query,
|
||||||
|
'maxResults': count,
|
||||||
|
'language': language,
|
||||||
|
'contentFormat': 'passage',
|
||||||
|
},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
results = response.json().get('webResults', [])
|
||||||
|
if filter_list:
|
||||||
|
results = get_filtered_results(results, filter_list)
|
||||||
|
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
link=result['url'],
|
||||||
|
title=result.get('title'),
|
||||||
|
snippet=result.get('content'),
|
||||||
|
)
|
||||||
|
for result in results
|
||||||
|
]
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f'Error searching with Microsoft Web IQ API: {e}')
|
||||||
|
return []
|
||||||
@@ -36,6 +36,9 @@ from open_webui.config import (
|
|||||||
FIRECRAWL_API_BASE_URL,
|
FIRECRAWL_API_BASE_URL,
|
||||||
FIRECRAWL_API_KEY,
|
FIRECRAWL_API_KEY,
|
||||||
FIRECRAWL_TIMEOUT,
|
FIRECRAWL_TIMEOUT,
|
||||||
|
MICROSOFT_WEB_IQ_API_BASE_URL,
|
||||||
|
MICROSOFT_WEB_IQ_API_KEY,
|
||||||
|
MICROSOFT_WEB_IQ_LANGUAGE,
|
||||||
PLAYWRIGHT_TIMEOUT,
|
PLAYWRIGHT_TIMEOUT,
|
||||||
PLAYWRIGHT_WS_URL,
|
PLAYWRIGHT_WS_URL,
|
||||||
TAVILY_API_KEY,
|
TAVILY_API_KEY,
|
||||||
@@ -52,6 +55,7 @@ from open_webui.env import (
|
|||||||
USER_AGENT,
|
USER_AGENT,
|
||||||
)
|
)
|
||||||
from open_webui.retrieval.loaders.external_web import ExternalWebLoader
|
from open_webui.retrieval.loaders.external_web import ExternalWebLoader
|
||||||
|
from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader
|
||||||
from open_webui.retrieval.loaders.tavily import TavilyLoader
|
from open_webui.retrieval.loaders.tavily import TavilyLoader
|
||||||
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
|
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
|
||||||
from open_webui.utils.misc import is_host_allowed
|
from open_webui.utils.misc import is_host_allowed
|
||||||
@@ -356,6 +360,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
web_paths: Union[str, List[str]],
|
web_paths: Union[str, List[str]],
|
||||||
|
api_base_url: str,
|
||||||
api_key: str,
|
api_key: str,
|
||||||
extract_depth: Literal['basic', 'advanced'] = 'basic',
|
extract_depth: Literal['basic', 'advanced'] = 'basic',
|
||||||
continue_on_failure: bool = True,
|
continue_on_failure: bool = True,
|
||||||
@@ -389,6 +394,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
|
|
||||||
# Store parameters for creating TavilyLoader instances
|
# Store parameters for creating TavilyLoader instances
|
||||||
self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths]
|
self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths]
|
||||||
|
self.api_base_url = api_base_url
|
||||||
self.api_key = api_key
|
self.api_key = api_key
|
||||||
self.extract_depth = extract_depth
|
self.extract_depth = extract_depth
|
||||||
self.continue_on_failure = continue_on_failure
|
self.continue_on_failure = continue_on_failure
|
||||||
@@ -464,6 +470,67 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
class SafeMicrosoftWebIQLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
web_paths: Union[str, List[str]],
|
||||||
|
api_key: str,
|
||||||
|
language: str = 'en',
|
||||||
|
verify_ssl: bool = True,
|
||||||
|
trust_env: bool = False,
|
||||||
|
requests_per_second: Optional[float] = None,
|
||||||
|
continue_on_failure: bool = True,
|
||||||
|
timeout: Optional[int] = None,
|
||||||
|
):
|
||||||
|
self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths]
|
||||||
|
self.api_key = api_key
|
||||||
|
self.language = language
|
||||||
|
self.verify_ssl = verify_ssl
|
||||||
|
self.trust_env = trust_env
|
||||||
|
self.requests_per_second = requests_per_second
|
||||||
|
self.last_request_time = None
|
||||||
|
self.continue_on_failure = continue_on_failure
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def lazy_load(self) -> Iterator[Document]:
|
||||||
|
valid_urls = []
|
||||||
|
for url in self.web_paths:
|
||||||
|
try:
|
||||||
|
self._safe_process_url_sync(url)
|
||||||
|
valid_urls.append(url)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f'SSL verification failed for {url}: {str(e)}')
|
||||||
|
if not self.continue_on_failure:
|
||||||
|
raise e
|
||||||
|
if not valid_urls:
|
||||||
|
if self.continue_on_failure:
|
||||||
|
log.warning('No valid URLs to process after SSL verification')
|
||||||
|
return
|
||||||
|
raise ValueError('No valid URLs to process after SSL verification')
|
||||||
|
|
||||||
|
loader = MicrosoftWebIQLoader(
|
||||||
|
urls=valid_urls,
|
||||||
|
api_base_url=self.api_base_url,
|
||||||
|
api_key=self.api_key,
|
||||||
|
language=self.language,
|
||||||
|
verify_ssl=self.verify_ssl,
|
||||||
|
timeout=self.timeout,
|
||||||
|
continue_on_failure=self.continue_on_failure,
|
||||||
|
)
|
||||||
|
yield from loader.lazy_load()
|
||||||
|
|
||||||
|
async def alazy_load(self) -> AsyncIterator[Document]:
|
||||||
|
try:
|
||||||
|
docs = await run_in_threadpool(lambda: list(self.lazy_load()))
|
||||||
|
for doc in docs:
|
||||||
|
yield doc
|
||||||
|
except Exception as e:
|
||||||
|
if self.continue_on_failure:
|
||||||
|
log.warning(f'Error browsing URLs with Microsoft Web IQ: {e}')
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessingMixin):
|
class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessingMixin):
|
||||||
"""Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection.
|
"""Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection.
|
||||||
|
|
||||||
@@ -813,6 +880,17 @@ def get_web_loader(
|
|||||||
web_loader_args['api_key'] = TAVILY_API_KEY
|
web_loader_args['api_key'] = TAVILY_API_KEY
|
||||||
web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH
|
web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH
|
||||||
|
|
||||||
|
if WEB_LOADER_ENGINE == 'microsoft_web_iq':
|
||||||
|
WebLoaderClass = SafeMicrosoftWebIQLoader
|
||||||
|
web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL
|
||||||
|
web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY
|
||||||
|
web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE
|
||||||
|
if WEB_LOADER_TIMEOUT:
|
||||||
|
try:
|
||||||
|
web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
if WEB_LOADER_ENGINE == 'external':
|
if WEB_LOADER_ENGINE == 'external':
|
||||||
WebLoaderClass = ExternalWebLoader
|
WebLoaderClass = ExternalWebLoader
|
||||||
web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL
|
web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL
|
||||||
@@ -831,5 +909,5 @@ def get_web_loader(
|
|||||||
else:
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. '
|
f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. '
|
||||||
"Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'."
|
"Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'."
|
||||||
)
|
)
|
||||||
@@ -95,6 +95,7 @@ from open_webui.retrieval.web.kagi import search_kagi
|
|||||||
|
|
||||||
# Web search engines
|
# Web search engines
|
||||||
from open_webui.retrieval.web.main import SearchResult
|
from open_webui.retrieval.web.main import SearchResult
|
||||||
|
from open_webui.retrieval.web.microsoft_web_iq import search_microsoft_web_iq
|
||||||
from open_webui.retrieval.web.mojeek import search_mojeek
|
from open_webui.retrieval.web.mojeek import search_mojeek
|
||||||
from open_webui.retrieval.web.ollama import search_ollama_cloud
|
from open_webui.retrieval.web.ollama import search_ollama_cloud
|
||||||
from open_webui.retrieval.web.perplexity import search_perplexity
|
from open_webui.retrieval.web.perplexity import search_perplexity
|
||||||
@@ -317,6 +318,9 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||||||
'MINERU_API_URL': 'rag.mineru_api_url',
|
'MINERU_API_URL': 'rag.mineru_api_url',
|
||||||
'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions',
|
'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions',
|
||||||
'MINERU_PARAMS': 'rag.mineru_params',
|
'MINERU_PARAMS': 'rag.mineru_params',
|
||||||
|
'MICROSOFT_WEB_IQ_API_BASE_URL': 'rag.web.search.microsoft_web_iq_api_base_url',
|
||||||
|
'MICROSOFT_WEB_IQ_API_KEY': 'rag.web.search.microsoft_web_iq_api_key',
|
||||||
|
'MICROSOFT_WEB_IQ_LANGUAGE': 'rag.web.search.microsoft_web_iq_language',
|
||||||
'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url',
|
'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url',
|
||||||
'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key',
|
'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key',
|
||||||
'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key',
|
'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key',
|
||||||
@@ -724,6 +728,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
|
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
|
||||||
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
||||||
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
|
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
|
||||||
|
'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL,
|
||||||
|
'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY,
|
||||||
|
'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE,
|
||||||
'SOUGOU_API_SID': config.SOUGOU_API_SID,
|
'SOUGOU_API_SID': config.SOUGOU_API_SID,
|
||||||
'SOUGOU_API_SK': config.SOUGOU_API_SK,
|
'SOUGOU_API_SK': config.SOUGOU_API_SK,
|
||||||
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
|
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
|
||||||
@@ -797,6 +804,9 @@ class WebConfig(BaseModel):
|
|||||||
PERPLEXITY_MODEL: str | None = None
|
PERPLEXITY_MODEL: str | None = None
|
||||||
PERPLEXITY_SEARCH_CONTEXT_USAGE: str | None = None
|
PERPLEXITY_SEARCH_CONTEXT_USAGE: str | None = None
|
||||||
PERPLEXITY_SEARCH_API_URL: str | None = None
|
PERPLEXITY_SEARCH_API_URL: str | None = None
|
||||||
|
MICROSOFT_WEB_IQ_API_BASE_URL: str | None = None
|
||||||
|
MICROSOFT_WEB_IQ_API_KEY: str | None = None
|
||||||
|
MICROSOFT_WEB_IQ_LANGUAGE: str | None = None
|
||||||
SOUGOU_API_SID: str | None = None
|
SOUGOU_API_SID: str | None = None
|
||||||
SOUGOU_API_SK: str | None = None
|
SOUGOU_API_SK: str | None = None
|
||||||
WEB_LOADER_ENGINE: str | None = None
|
WEB_LOADER_ENGINE: str | None = None
|
||||||
@@ -1298,6 +1308,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
config.PERPLEXITY_MODEL = form_data.web.PERPLEXITY_MODEL
|
config.PERPLEXITY_MODEL = form_data.web.PERPLEXITY_MODEL
|
||||||
config.PERPLEXITY_SEARCH_CONTEXT_USAGE = form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE
|
config.PERPLEXITY_SEARCH_CONTEXT_USAGE = form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE
|
||||||
config.PERPLEXITY_SEARCH_API_URL = form_data.web.PERPLEXITY_SEARCH_API_URL
|
config.PERPLEXITY_SEARCH_API_URL = form_data.web.PERPLEXITY_SEARCH_API_URL
|
||||||
|
config.MICROSOFT_WEB_IQ_API_BASE_URL = form_data.web.MICROSOFT_WEB_IQ_API_BASE_URL
|
||||||
|
config.MICROSOFT_WEB_IQ_API_KEY = form_data.web.MICROSOFT_WEB_IQ_API_KEY
|
||||||
|
config.MICROSOFT_WEB_IQ_LANGUAGE = form_data.web.MICROSOFT_WEB_IQ_LANGUAGE
|
||||||
config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID
|
config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID
|
||||||
config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK
|
config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK
|
||||||
|
|
||||||
@@ -1326,6 +1339,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
config.LINKUP_API_KEY = form_data.web.LINKUP_API_KEY
|
config.LINKUP_API_KEY = form_data.web.LINKUP_API_KEY
|
||||||
config.LINKUP_SEARCH_PARAMS = form_data.web.LINKUP_SEARCH_PARAMS
|
config.LINKUP_SEARCH_PARAMS = form_data.web.LINKUP_SEARCH_PARAMS
|
||||||
|
|
||||||
|
await config.save()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'status': True,
|
'status': True,
|
||||||
# RAG settings
|
# RAG settings
|
||||||
@@ -1438,6 +1453,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
|
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
|
||||||
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
||||||
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
|
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
|
||||||
|
'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL,
|
||||||
|
'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY,
|
||||||
|
'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE,
|
||||||
'SOUGOU_API_SID': config.SOUGOU_API_SID,
|
'SOUGOU_API_SID': config.SOUGOU_API_SID,
|
||||||
'SOUGOU_API_SK': config.SOUGOU_API_SK,
|
'SOUGOU_API_SK': config.SOUGOU_API_SK,
|
||||||
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
|
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
|
||||||
@@ -2394,6 +2412,20 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li
|
|||||||
model=config.PERPLEXITY_MODEL,
|
model=config.PERPLEXITY_MODEL,
|
||||||
search_context_usage=config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
search_context_usage=config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
|
||||||
)
|
)
|
||||||
|
elif engine == 'microsoft_web_iq':
|
||||||
|
if config.MICROSOFT_WEB_IQ_API_KEY:
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
search_microsoft_web_iq,
|
||||||
|
config.MICROSOFT_WEB_IQ_API_BASE_URL,
|
||||||
|
config.MICROSOFT_WEB_IQ_API_KEY,
|
||||||
|
query,
|
||||||
|
config.WEB_SEARCH_RESULT_COUNT,
|
||||||
|
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
config.MICROSOFT_WEB_IQ_LANGUAGE,
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise Exception('No MICROSOFT_WEB_IQ_API_KEY found in environment variables')
|
||||||
elif engine == 'sougou':
|
elif engine == 'sougou':
|
||||||
if config.SOUGOU_API_SID and config.SOUGOU_API_SK:
|
if config.SOUGOU_API_SID and config.SOUGOU_API_SK:
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
|
|||||||
@@ -36,6 +36,7 @@
|
|||||||
'bing',
|
'bing',
|
||||||
'exa',
|
'exa',
|
||||||
'perplexity',
|
'perplexity',
|
||||||
|
'microsoft_web_iq',
|
||||||
'sougou',
|
'sougou',
|
||||||
'firecrawl',
|
'firecrawl',
|
||||||
'external',
|
'external',
|
||||||
@@ -43,7 +44,7 @@
|
|||||||
'youcom',
|
'youcom',
|
||||||
'linkup'
|
'linkup'
|
||||||
];
|
];
|
||||||
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external'];
|
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'microsoft_web_iq', 'external'];
|
||||||
|
|
||||||
let webConfig = null;
|
let webConfig = null;
|
||||||
|
|
||||||
@@ -714,6 +715,51 @@
|
|||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{:else if webConfig.WEB_SEARCH_ENGINE === 'microsoft_web_iq'}
|
||||||
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
|
<div>
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Microsoft Web IQ API Base URL')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex w-full">
|
||||||
|
<div class="flex-1">
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
type="text"
|
||||||
|
placeholder={$i18n.t('Enter Microsoft Web IQ API Base URL')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_API_BASE_URL}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-2">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Microsoft Web IQ API Key')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<SensitiveInput
|
||||||
|
placeholder={$i18n.t('Enter Microsoft Web IQ API Key')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_API_KEY}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-2">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Language')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
type="text"
|
||||||
|
placeholder={$i18n.t('Enter language')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_LANGUAGE}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
|
{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
|
||||||
<div class="mb-2.5 flex w-full flex-col">
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
<div>
|
<div>
|
||||||
@@ -1211,6 +1257,53 @@
|
|||||||
</div>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
{:else if webConfig.WEB_LOADER_ENGINE === 'microsoft_web_iq'}
|
||||||
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
|
{#if webConfig.WEB_SEARCH_ENGINE !== 'microsoft_web_iq'}
|
||||||
|
<div>
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Microsoft Web IQ API Base URL')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex w-full">
|
||||||
|
<div class="flex-1">
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
type="text"
|
||||||
|
placeholder={$i18n.t('Enter Microsoft Web IQ API Base URL')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_API_BASE_URL}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-2">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Microsoft Web IQ API Key')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<SensitiveInput
|
||||||
|
placeholder={$i18n.t('Enter Microsoft Web IQ API Key')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_API_KEY}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-2">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Language')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
type="text"
|
||||||
|
placeholder={$i18n.t('Enter language')}
|
||||||
|
bind:value={webConfig.MICROSOFT_WEB_IQ_LANGUAGE}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
{:else if webConfig.WEB_LOADER_ENGINE === 'external'}
|
{:else if webConfig.WEB_LOADER_ENGINE === 'external'}
|
||||||
<div class="mb-2.5 flex w-full flex-col">
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
<div>
|
<div>
|
||||||
|
|||||||
Reference in new issue
Block a user