refac
This commit is contained in:
1 parent
1f0948bcbe
commit
d4030a8aa5
5 files changed
+41
-2
No files matched your search
@@ -1109,6 +1109,12 @@ MINERU_PARAMS = ConfigVar(
|
|||||||
mineru_params,
|
mineru_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
MINERU_FILE_EXTENSIONS = ConfigVar(
|
||||||
|
'MINERU_FILE_EXTENSIONS',
|
||||||
|
'rag.mineru_file_extensions',
|
||||||
|
[ext.strip() for ext in os.getenv('MINERU_FILE_EXTENSIONS', 'pdf').split(',') if ext.strip()],
|
||||||
|
)
|
||||||
|
|
||||||
EXTERNAL_DOCUMENT_LOADER_URL = ConfigVar(
|
EXTERNAL_DOCUMENT_LOADER_URL = ConfigVar(
|
||||||
'EXTERNAL_DOCUMENT_LOADER_URL',
|
'EXTERNAL_DOCUMENT_LOADER_URL',
|
||||||
'rag.external_document_loader_url',
|
'rag.external_document_loader_url',
|
||||||
|
|||||||
@@ -281,6 +281,7 @@ from open_webui.config import (
|
|||||||
MINERU_API_MODE,
|
MINERU_API_MODE,
|
||||||
MINERU_API_TIMEOUT,
|
MINERU_API_TIMEOUT,
|
||||||
MINERU_API_URL,
|
MINERU_API_URL,
|
||||||
|
MINERU_FILE_EXTENSIONS,
|
||||||
MINERU_PARAMS,
|
MINERU_PARAMS,
|
||||||
MISTRAL_OCR_API_BASE_URL,
|
MISTRAL_OCR_API_BASE_URL,
|
||||||
MISTRAL_OCR_API_KEY,
|
MISTRAL_OCR_API_KEY,
|
||||||
@@ -1022,6 +1023,7 @@ app.state.config.MINERU_API_URL = MINERU_API_URL
|
|||||||
app.state.config.MINERU_API_KEY = MINERU_API_KEY
|
app.state.config.MINERU_API_KEY = MINERU_API_KEY
|
||||||
app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT
|
app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT
|
||||||
app.state.config.MINERU_PARAMS = MINERU_PARAMS
|
app.state.config.MINERU_PARAMS = MINERU_PARAMS
|
||||||
|
app.state.config.MINERU_FILE_EXTENSIONS = MINERU_FILE_EXTENSIONS
|
||||||
|
|
||||||
app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
|
app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
|
||||||
app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER
|
app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER
|
||||||
|
|||||||
@@ -371,7 +371,7 @@ class Loader:
|
|||||||
azure_credential=DefaultAzureCredential(),
|
azure_credential=DefaultAzureCredential(),
|
||||||
api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'),
|
api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'),
|
||||||
)
|
)
|
||||||
elif self.engine == 'mineru' and file_ext in ['pdf']: # MinerU currently only supports PDF
|
elif self.engine == 'mineru' and file_ext in self.kwargs.get('MINERU_FILE_EXTENSIONS', ['pdf']):
|
||||||
mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300)
|
mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300)
|
||||||
if mineru_timeout:
|
if mineru_timeout:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -466,6 +466,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
'MINERU_API_KEY': request.app.state.config.MINERU_API_KEY,
|
'MINERU_API_KEY': request.app.state.config.MINERU_API_KEY,
|
||||||
'MINERU_API_TIMEOUT': request.app.state.config.MINERU_API_TIMEOUT,
|
'MINERU_API_TIMEOUT': request.app.state.config.MINERU_API_TIMEOUT,
|
||||||
'MINERU_PARAMS': request.app.state.config.MINERU_PARAMS,
|
'MINERU_PARAMS': request.app.state.config.MINERU_PARAMS,
|
||||||
|
'MINERU_FILE_EXTENSIONS': request.app.state.config.MINERU_FILE_EXTENSIONS,
|
||||||
# Reranking settings
|
# Reranking settings
|
||||||
'RAG_RERANKING_MODEL': request.app.state.config.RAG_RERANKING_MODEL,
|
'RAG_RERANKING_MODEL': request.app.state.config.RAG_RERANKING_MODEL,
|
||||||
'RAG_RERANKING_ENGINE': request.app.state.config.RAG_RERANKING_ENGINE,
|
'RAG_RERANKING_ENGINE': request.app.state.config.RAG_RERANKING_ENGINE,
|
||||||
@@ -681,6 +682,7 @@ class ConfigForm(BaseModel):
|
|||||||
MINERU_API_KEY: str | None = None
|
MINERU_API_KEY: str | None = None
|
||||||
MINERU_API_TIMEOUT: str | None = None
|
MINERU_API_TIMEOUT: str | None = None
|
||||||
MINERU_PARAMS: dict | None = None
|
MINERU_PARAMS: dict | None = None
|
||||||
|
MINERU_FILE_EXTENSIONS: list[str] | None = None
|
||||||
|
|
||||||
# Reranking settings
|
# Reranking settings
|
||||||
RAG_RERANKING_MODEL: str | None = None
|
RAG_RERANKING_MODEL: str | None = None
|
||||||
@@ -904,6 +906,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
request.app.state.config.MINERU_PARAMS = (
|
request.app.state.config.MINERU_PARAMS = (
|
||||||
form_data.MINERU_PARAMS if form_data.MINERU_PARAMS is not None else request.app.state.config.MINERU_PARAMS
|
form_data.MINERU_PARAMS if form_data.MINERU_PARAMS is not None else request.app.state.config.MINERU_PARAMS
|
||||||
)
|
)
|
||||||
|
request.app.state.config.MINERU_FILE_EXTENSIONS = (
|
||||||
|
form_data.MINERU_FILE_EXTENSIONS
|
||||||
|
if form_data.MINERU_FILE_EXTENSIONS is not None
|
||||||
|
else request.app.state.config.MINERU_FILE_EXTENSIONS
|
||||||
|
)
|
||||||
|
|
||||||
# Reranking settings
|
# Reranking settings
|
||||||
if request.app.state.config.RAG_RERANKING_ENGINE == '':
|
if request.app.state.config.RAG_RERANKING_ENGINE == '':
|
||||||
|
|||||||
@@ -244,7 +244,10 @@
|
|||||||
MINERU_PARAMS:
|
MINERU_PARAMS:
|
||||||
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
|
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
|
||||||
? JSON.parse(RAGConfig.MINERU_PARAMS)
|
? JSON.parse(RAGConfig.MINERU_PARAMS)
|
||||||
: {}
|
: {},
|
||||||
|
MINERU_FILE_EXTENSIONS: RAGConfig.MINERU_FILE_EXTENSIONS.split(',')
|
||||||
|
.map((ext) => ext.trim())
|
||||||
|
.filter((ext) => ext !== '')
|
||||||
});
|
});
|
||||||
dispatch('save');
|
dispatch('save');
|
||||||
};
|
};
|
||||||
@@ -286,6 +289,8 @@
|
|||||||
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
|
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
|
||||||
: config.MINERU_PARAMS;
|
: config.MINERU_PARAMS;
|
||||||
|
|
||||||
|
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
|
||||||
|
|
||||||
RAGConfig = config;
|
RAGConfig = config;
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
@@ -763,6 +768,25 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- File Extensions -->
|
||||||
|
<div class="flex flex-col justify-between w-full mt-2">
|
||||||
|
<div class="text-xs font-medium mb-1">
|
||||||
|
<Tooltip
|
||||||
|
content={$i18n.t(
|
||||||
|
'Comma-separated list of file extensions MinerU will handle (e.g. pdf, docx, pptx, xlsx)'
|
||||||
|
)}
|
||||||
|
placement="top-start"
|
||||||
|
>
|
||||||
|
{$i18n.t('File Extensions')}
|
||||||
|
</Tooltip>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
class="flex-1 w-full text-sm bg-transparent outline-hidden"
|
||||||
|
placeholder={$i18n.t('pdf, docx, pptx, xlsx')}
|
||||||
|
bind:value={RAGConfig.MINERU_FILE_EXTENSIONS}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user