refac
This commit is contained in:
1 parent
a9a802d95f
commit
170ad0595d
6 files changed
+40
-16
No files matched your search
@@ -925,6 +925,8 @@ EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
|
|||||||
|
|
||||||
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
|
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
|
||||||
|
|
||||||
|
TIKA_SERVER_VERSION = os.getenv('TIKA_SERVER_VERSION', '3')
|
||||||
|
|
||||||
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
|
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
|
||||||
|
|
||||||
DOCLING_API_KEY = os.getenv('DOCLING_API_KEY', '')
|
DOCLING_API_KEY = os.getenv('DOCLING_API_KEY', '')
|
||||||
@@ -2878,6 +2880,7 @@ DEFAULT_CONFIG = {
|
|||||||
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'rag.tika_server_url': TIKA_SERVER_URL,
|
'rag.tika_server_url': TIKA_SERVER_URL,
|
||||||
|
'rag.tika_server_version': TIKA_SERVER_VERSION,
|
||||||
'rag.docling_server_url': DOCLING_SERVER_URL,
|
'rag.docling_server_url': DOCLING_SERVER_URL,
|
||||||
'rag.docling_api_key': DOCLING_API_KEY,
|
'rag.docling_api_key': DOCLING_API_KEY,
|
||||||
'rag.docling_params': DOCLING_PARAMS,
|
'rag.docling_params': DOCLING_PARAMS,
|
||||||
|
|||||||
@@ -186,10 +186,11 @@ class PptxLoader:
|
|||||||
|
|
||||||
|
|
||||||
class TikaLoader:
|
class TikaLoader:
|
||||||
def __init__(self, url, file_path, mime_type=None, extract_images=None):
|
def __init__(self, url, file_path, mime_type=None, extract_images=None, server_version='3'):
|
||||||
self.url = url
|
self.url = url
|
||||||
self.file_path = file_path
|
self.file_path = file_path
|
||||||
self.mime_type = mime_type
|
self.mime_type = mime_type
|
||||||
|
self.server_version = str(server_version or '3')
|
||||||
|
|
||||||
self.extract_images = extract_images
|
self.extract_images = extract_images
|
||||||
|
|
||||||
@@ -205,16 +206,15 @@ class TikaLoader:
|
|||||||
if self.extract_images == True:
|
if self.extract_images == True:
|
||||||
headers['X-Tika-PDFextractInlineImages'] = 'true'
|
headers['X-Tika-PDFextractInlineImages'] = 'true'
|
||||||
|
|
||||||
endpoint = self.url
|
endpoint_path = 'tika/json/text' if self.server_version == '4' else 'tika/text'
|
||||||
if not endpoint.endswith('/'):
|
content_key = 'tk:content' if self.server_version == '4' else 'X-TIKA:content'
|
||||||
endpoint += '/'
|
endpoint = f'{self.url.rstrip("/")}/{endpoint_path}'
|
||||||
endpoint += 'tika/text'
|
|
||||||
|
|
||||||
r = requests.put(endpoint, data=data, headers=headers, verify=REQUESTS_VERIFY)
|
r = requests.put(endpoint, data=data, headers=headers, verify=REQUESTS_VERIFY)
|
||||||
|
|
||||||
if r.ok:
|
if r.ok:
|
||||||
raw_metadata = r.json()
|
raw_metadata = r.json()
|
||||||
text = raw_metadata.get('X-TIKA:content', '<No text content found>').strip()
|
text = raw_metadata.get(content_key, '<No text content found>').strip()
|
||||||
|
|
||||||
if 'Content-Type' in raw_metadata:
|
if 'Content-Type' in raw_metadata:
|
||||||
headers['Content-Type'] = raw_metadata['Content-Type']
|
headers['Content-Type'] = raw_metadata['Content-Type']
|
||||||
@@ -503,6 +503,7 @@ class Loader:
|
|||||||
loader = TikaLoader(
|
loader = TikaLoader(
|
||||||
url=self.kwargs.get('TIKA_SERVER_URL'),
|
url=self.kwargs.get('TIKA_SERVER_URL'),
|
||||||
file_path=file_path,
|
file_path=file_path,
|
||||||
|
server_version=self.kwargs.get('TIKA_SERVER_VERSION'),
|
||||||
extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'),
|
extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'),
|
||||||
)
|
)
|
||||||
elif (
|
elif (
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ LOADER_CONFIG_KEYS = {
|
|||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
||||||
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
|
||||||
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
||||||
|
'TIKA_SERVER_VERSION': 'rag.tika_server_version',
|
||||||
'DOCLING_SERVER_URL': 'rag.docling_server_url',
|
'DOCLING_SERVER_URL': 'rag.docling_server_url',
|
||||||
'DOCLING_API_KEY': 'rag.docling_api_key',
|
'DOCLING_API_KEY': 'rag.docling_api_key',
|
||||||
'DOCLING_PARAMS': 'rag.docling_params',
|
'DOCLING_PARAMS': 'rag.docling_params',
|
||||||
|
|||||||
@@ -387,6 +387,7 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||||||
'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth',
|
'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth',
|
||||||
'TEXT_SPLITTER': 'rag.text_splitter',
|
'TEXT_SPLITTER': 'rag.text_splitter',
|
||||||
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
||||||
|
'TIKA_SERVER_VERSION': 'rag.tika_server_version',
|
||||||
'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name',
|
'TIKTOKEN_ENCODING_NAME': 'rag.tiktoken_encoding_name',
|
||||||
'TOP_K': 'rag.top_k',
|
'TOP_K': 'rag.top_k',
|
||||||
'TOP_K_RERANKER': 'rag.top_k_reranker',
|
'TOP_K_RERANKER': 'rag.top_k_reranker',
|
||||||
@@ -659,6 +660,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
||||||
|
'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION,
|
||||||
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
||||||
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
||||||
'DOCLING_PARAMS': config.DOCLING_PARAMS,
|
'DOCLING_PARAMS': config.DOCLING_PARAMS,
|
||||||
@@ -894,6 +896,7 @@ class ConfigForm(BaseModel):
|
|||||||
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
|
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
|
||||||
|
|
||||||
TIKA_SERVER_URL: str | None = None
|
TIKA_SERVER_URL: str | None = None
|
||||||
|
TIKA_SERVER_VERSION: str | None = None
|
||||||
DOCLING_SERVER_URL: str | None = None
|
DOCLING_SERVER_URL: str | None = None
|
||||||
DOCLING_API_KEY: str | None = None
|
DOCLING_API_KEY: str | None = None
|
||||||
DOCLING_PARAMS: dict | None = None
|
DOCLING_PARAMS: dict | None = None
|
||||||
@@ -1070,6 +1073,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
config.TIKA_SERVER_URL = (
|
config.TIKA_SERVER_URL = (
|
||||||
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
|
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
|
||||||
)
|
)
|
||||||
|
config.TIKA_SERVER_VERSION = (
|
||||||
|
form_data.TIKA_SERVER_VERSION if form_data.TIKA_SERVER_VERSION is not None else config.TIKA_SERVER_VERSION
|
||||||
|
)
|
||||||
config.DOCLING_SERVER_URL = (
|
config.DOCLING_SERVER_URL = (
|
||||||
form_data.DOCLING_SERVER_URL if form_data.DOCLING_SERVER_URL is not None else config.DOCLING_SERVER_URL
|
form_data.DOCLING_SERVER_URL if form_data.DOCLING_SERVER_URL is not None else config.DOCLING_SERVER_URL
|
||||||
)
|
)
|
||||||
@@ -1368,6 +1374,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
||||||
|
'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION,
|
||||||
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
||||||
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
||||||
'DOCLING_PARAMS': config.DOCLING_PARAMS,
|
'DOCLING_PARAMS': config.DOCLING_PARAMS,
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ type RAGConfigForm = {
|
|||||||
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
|
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
|
||||||
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
|
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
|
||||||
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
|
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
|
||||||
|
TIKA_SERVER_VERSION?: string | null;
|
||||||
chunk?: ChunkConfigForm;
|
chunk?: ChunkConfigForm;
|
||||||
content_extraction?: ContentExtractConfigForm;
|
content_extraction?: ContentExtractConfigForm;
|
||||||
web_loader_ssl_verification?: boolean;
|
web_loader_ssl_verification?: boolean;
|
||||||
|
|||||||
@@ -653,16 +653,27 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</AdminSettingField>
|
</AdminSettingField>
|
||||||
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
|
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
|
||||||
<AdminSettingField
|
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
|
||||||
label={$i18n.t('Tika Server URL')}
|
<AdminSettingField
|
||||||
description={$i18n.t('Tika server endpoint used for content extraction.')}
|
label={$i18n.t('Tika Server URL')}
|
||||||
>
|
description={$i18n.t('Tika server endpoint used for content extraction.')}
|
||||||
<input
|
>
|
||||||
class={inputClass}
|
<input
|
||||||
placeholder={$i18n.t('Enter Tika Server URL')}
|
class={inputClass}
|
||||||
bind:value={RAGConfig.TIKA_SERVER_URL}
|
placeholder={$i18n.t('Enter Tika Server URL')}
|
||||||
/>
|
bind:value={RAGConfig.TIKA_SERVER_URL}
|
||||||
</AdminSettingField>
|
/>
|
||||||
|
</AdminSettingField>
|
||||||
|
<AdminSettingField
|
||||||
|
label={$i18n.t('Tika Server Version')}
|
||||||
|
description={$i18n.t('Select the Tika server API version.')}
|
||||||
|
>
|
||||||
|
<SettingsSelect bind:value={RAGConfig.TIKA_SERVER_VERSION}>
|
||||||
|
<option value="3">{$i18n.t('Tika 3.x')}</option>
|
||||||
|
<option value="4">{$i18n.t('Tika 4.x')}</option>
|
||||||
|
</SettingsSelect>
|
||||||
|
</AdminSettingField>
|
||||||
|
</div>
|
||||||
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling'}
|
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling'}
|
||||||
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
|
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
|
||||||
<AdminSettingField
|
<AdminSettingField
|
||||||
|
|||||||
Reference in new issue
Block a user