bhetherman/open-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-08-24 19:48:49 -04:00
parent a9a802d95f
commit 170ad0595d
6 changed files with 40 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/config.py
+3
View File
@@ -925,6 +925,8 @@ EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
TIKA_SERVER_VERSION = os.getenv('TIKA_SERVER_VERSION', '3')
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
DOCLING_API_KEY = os.getenv('DOCLING_API_KEY', '')
@@ -2878,6 +2880,7 @@ DEFAULT_CONFIG = {
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
'rag.tika_server_url': TIKA_SERVER_URL,
'rag.tika_server_version': TIKA_SERVER_VERSION,
'rag.docling_server_url': DOCLING_SERVER_URL,
'rag.docling_api_key': DOCLING_API_KEY,
'rag.docling_params': DOCLING_PARAMS,
backend/open_webui/retrieval/loaders/main.py
+7 -6
View File
@@ -186,10 +186,11 @@ class PptxLoader:
class TikaLoader:
def __init__(self, url, file_path, mime_type=None, extract_images=None):
def __init__(self, url, file_path, mime_type=None, extract_images=None, server_version='3'):
self.url = url
self.file_path = file_path
self.mime_type = mime_type
self.server_version = str(server_version or '3')
self.extract_images = extract_images
@@ -205,16 +206,15 @@ class TikaLoader:
if self.extract_images == True:
headers['X-Tika-PDFextractInlineImages'] = 'true'
endpoint = self.url
if not endpoint.endswith('/'):
endpoint += '/'
endpoint += 'tika/text'
endpoint_path = 'tika/json/text' if self.server_version == '4' else 'tika/text'
content_key = 'tk:content' if self.server_version == '4' else 'X-TIKA:content'
endpoint = f'{self.url.rstrip("/")}/{endpoint_path}'
r = requests.put(endpoint, data=data, headers=headers, verify=REQUESTS_VERIFY)
if r.ok:
raw_metadata = r.json()
text = raw_metadata.get('X-TIKA:content', '<No text content found>').strip()
text = raw_metadata.get(content_key, '<No text content found>').strip()
if 'Content-Type' in raw_metadata:
headers['Content-Type'] = raw_metadata['Content-Type']
@@ -503,6 +503,7 @@ class Loader:
loader = TikaLoader(
url=self.kwargs.get('TIKA_SERVER_URL'),
file_path=file_path,
server_version=self.kwargs.get('TIKA_SERVER_VERSION'),
extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'),
)
elif (
backend/open_webui/retrieval/utils.py
+1
View File
@@ -104,6 +104,7 @@ LOADER_CONFIG_KEYS = {
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'TIKA_SERVER_URL': 'rag.tika_server_url',
'TIKA_SERVER_VERSION': 'rag.tika_server_version',
'DOCLING_SERVER_URL': 'rag.docling_server_url',
'DOCLING_API_KEY': 'rag.docling_api_key',
'DOCLING_PARAMS': 'rag.docling_params',
backend/open_webui/routers/retrieval.py
+7
View File
@@ -387,6 +387,7 @@ RETRIEVAL_CONFIG_KEYS = {
'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth',
'TEXT_SPLITTER': 'rag.text_splitter',
'TIKA_SERVER_URL': 'rag.tika_server_url',
'TIKA_SERVER_VERSION': 'rag.tika_server_version',
'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name',
'TOP_K': 'rag.top_k',
'TOP_K_RERANKER': 'rag.top_k_reranker',
@@ -659,6 +660,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY,
'DOCLING_PARAMS': config.DOCLING_PARAMS,
@@ -894,6 +896,7 @@ class ConfigForm(BaseModel):
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
TIKA_SERVER_URL: str | None = None
TIKA_SERVER_VERSION: str | None = None
DOCLING_SERVER_URL: str | None = None
DOCLING_API_KEY: str | None = None
DOCLING_PARAMS: dict | None = None
@@ -1070,6 +1073,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
config.TIKA_SERVER_URL = (
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
)
config.TIKA_SERVER_VERSION = (
form_data.TIKA_SERVER_VERSION if form_data.TIKA_SERVER_VERSION is not None else config.TIKA_SERVER_VERSION
)
config.DOCLING_SERVER_URL = (
form_data.DOCLING_SERVER_URL if form_data.DOCLING_SERVER_URL is not None else config.DOCLING_SERVER_URL
)
@@ -1368,6 +1374,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY,
'DOCLING_PARAMS': config.DOCLING_PARAMS,
src/lib/apis/retrieval/index.ts
+1
View File
@@ -56,6 +56,7 @@ type RAGConfigForm = {
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
TIKA_SERVER_VERSION?: string | null;
chunk?: ChunkConfigForm;
content_extraction?: ContentExtractConfigForm;
web_loader_ssl_verification?: boolean;
src/lib/components/admin/Settings/Documents.svelte
+21 -10
View File
@@ -653,16 +653,27 @@
{/if}
</AdminSettingField>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
<AdminSettingField
label={$i18n.t('Tika Server URL')}
description={$i18n.t('Tika server endpoint used for content extraction.')}
>
<input
class={inputClass}
placeholder={$i18n.t('Enter Tika Server URL')}
bind:value={RAGConfig.TIKA_SERVER_URL}
/>
</AdminSettingField>
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
<AdminSettingField
label={$i18n.t('Tika Server URL')}
description={$i18n.t('Tika server endpoint used for content extraction.')}
>
<input
class={inputClass}
placeholder={$i18n.t('Enter Tika Server URL')}
bind:value={RAGConfig.TIKA_SERVER_URL}
/>
</AdminSettingField>
<AdminSettingField
label={$i18n.t('Tika Server Version')}
description={$i18n.t('Select the Tika server API version.')}
>
<SettingsSelect bind:value={RAGConfig.TIKA_SERVER_VERSION}>
<option value="3">{$i18n.t('Tika 3.x')}</option>
<option value="4">{$i18n.t('Tika 4.x')}</option>
</SettingsSelect>
</AdminSettingField>
</div>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling'}
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
<AdminSettingField