diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index bc00611565..f8868940d8 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -925,6 +925,8 @@ EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998') +TIKA_SERVER_VERSION = os.getenv('TIKA_SERVER_VERSION', '3') + DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001') DOCLING_API_KEY = os.getenv('DOCLING_API_KEY', '') @@ -2878,6 +2880,7 @@ DEFAULT_CONFIG = { 'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY, 'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS, 'rag.tika_server_url': TIKA_SERVER_URL, + 'rag.tika_server_version': TIKA_SERVER_VERSION, 'rag.docling_server_url': DOCLING_SERVER_URL, 'rag.docling_api_key': DOCLING_API_KEY, 'rag.docling_params': DOCLING_PARAMS, diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index c86590583b..e1864eac49 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -186,10 +186,11 @@ class PptxLoader: class TikaLoader: - def __init__(self, url, file_path, mime_type=None, extract_images=None): + def __init__(self, url, file_path, mime_type=None, extract_images=None, server_version='3'): self.url = url self.file_path = file_path self.mime_type = mime_type + self.server_version = str(server_version or '3') self.extract_images = extract_images @@ -205,16 +206,15 @@ class TikaLoader: if self.extract_images == True: headers['X-Tika-PDFextractInlineImages'] = 'true' - endpoint = self.url - if not endpoint.endswith('/'): - endpoint += '/' - endpoint += 'tika/text' + endpoint_path = 'tika/json/text' if self.server_version == '4' else 'tika/text' + content_key = 'tk:content' if self.server_version == '4' else 'X-TIKA:content' + endpoint = f'{self.url.rstrip("/")}/{endpoint_path}' r = requests.put(endpoint, data=data, headers=headers, verify=REQUESTS_VERIFY) if r.ok: raw_metadata = r.json() - text = raw_metadata.get('X-TIKA:content', '').strip() + text = raw_metadata.get(content_key, '').strip() if 'Content-Type' in raw_metadata: headers['Content-Type'] = raw_metadata['Content-Type'] @@ -503,6 +503,7 @@ class Loader: loader = TikaLoader( url=self.kwargs.get('TIKA_SERVER_URL'), file_path=file_path, + server_version=self.kwargs.get('TIKA_SERVER_VERSION'), extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'), ) elif ( diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index a6e1bd6857..2c1ff1e7cd 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -104,6 +104,7 @@ LOADER_CONFIG_KEYS = { 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', 'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers', 'TIKA_SERVER_URL': 'rag.tika_server_url', + 'TIKA_SERVER_VERSION': 'rag.tika_server_version', 'DOCLING_SERVER_URL': 'rag.docling_server_url', 'DOCLING_API_KEY': 'rag.docling_api_key', 'DOCLING_PARAMS': 'rag.docling_params', diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 12d9c23db5..ee9eeb9339 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -387,6 +387,7 @@ RETRIEVAL_CONFIG_KEYS = { 'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth', 'TEXT_SPLITTER': 'rag.text_splitter', 'TIKA_SERVER_URL': 'rag.tika_server_url', + 'TIKA_SERVER_VERSION': 'rag.tika_server_version', 'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name', 'TOP_K': 'rag.top_k', 'TOP_K_RERANKER': 'rag.top_k_reranker', @@ -659,6 +660,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, + 'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_API_KEY': config.DOCLING_API_KEY, 'DOCLING_PARAMS': config.DOCLING_PARAMS, @@ -894,6 +896,7 @@ class ConfigForm(BaseModel): EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None TIKA_SERVER_URL: str | None = None + TIKA_SERVER_VERSION: str | None = None DOCLING_SERVER_URL: str | None = None DOCLING_API_KEY: str | None = None DOCLING_PARAMS: dict | None = None @@ -1070,6 +1073,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend config.TIKA_SERVER_URL = ( form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL ) + config.TIKA_SERVER_VERSION = ( + form_data.TIKA_SERVER_VERSION if form_data.TIKA_SERVER_VERSION is not None else config.TIKA_SERVER_VERSION + ) config.DOCLING_SERVER_URL = ( form_data.DOCLING_SERVER_URL if form_data.DOCLING_SERVER_URL is not None else config.DOCLING_SERVER_URL ) @@ -1368,6 +1374,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, + 'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_API_KEY': config.DOCLING_API_KEY, 'DOCLING_PARAMS': config.DOCLING_PARAMS, diff --git a/src/lib/apis/retrieval/index.ts b/src/lib/apis/retrieval/index.ts index f913e3e868..3e3a2d2a00 100644 --- a/src/lib/apis/retrieval/index.ts +++ b/src/lib/apis/retrieval/index.ts @@ -56,6 +56,7 @@ type RAGConfigForm = { ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean; ENABLE_ONEDRIVE_INTEGRATION?: boolean; EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record; + TIKA_SERVER_VERSION?: string | null; chunk?: ChunkConfigForm; content_extraction?: ContentExtractConfigForm; web_loader_ssl_verification?: boolean; diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 2ae89640eb..2c49b05520 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -653,16 +653,27 @@ {/if} {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'} - - - +
+ + + + + + + + + +
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling'}