refac
This commit is contained in:
1 parent
3ab2026262
commit
db2d24896b
8 files changed
+103
-6
No files matched your search
@@ -853,6 +853,17 @@ ONEDRIVE_SHAREPOINT_TENANT_ID = os.getenv('ONEDRIVE_SHAREPOINT_TENANT_ID', '')
|
||||
# RAG Content Extraction
|
||||
CONTENT_EXTRACTION_ENGINE = os.getenv('CONTENT_EXTRACTION_ENGINE', '').lower()
|
||||
|
||||
content_extraction_supported_media_mime_types = os.getenv('CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES')
|
||||
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = (
|
||||
[
|
||||
mime_type.strip()
|
||||
for mime_type in content_extraction_supported_media_mime_types.split(',')
|
||||
if mime_type.strip()
|
||||
]
|
||||
if content_extraction_supported_media_mime_types is not None
|
||||
else None
|
||||
)
|
||||
|
||||
DATALAB_MARKER_API_KEY = os.getenv('DATALAB_MARKER_API_KEY', '')
|
||||
|
||||
DATALAB_MARKER_API_BASE_URL = os.getenv('DATALAB_MARKER_API_BASE_URL', '')
|
||||
@@ -1999,6 +2010,8 @@ FOLDER_MAX_FILE_COUNT = os.getenv('FOLDER_MAX_FILE_COUNT', '')
|
||||
|
||||
ENABLE_CHANNELS = os.getenv('ENABLE_CHANNELS', 'False').lower() == 'true'
|
||||
|
||||
CHANNEL_MODEL_RESPONSE_MODE = os.getenv('CHANNEL_MODEL_RESPONSE_MODE', 'thread')
|
||||
|
||||
ENABLE_CALENDAR = os.getenv('ENABLE_CALENDAR', 'True').lower() == 'true'
|
||||
|
||||
ENABLE_AUTOMATIONS = os.getenv('ENABLE_AUTOMATIONS', 'True').lower() == 'true'
|
||||
@@ -2817,6 +2830,7 @@ DEFAULT_CONFIG = {
|
||||
'onedrive.sharepoint_url': ONEDRIVE_SHAREPOINT_URL,
|
||||
'onedrive.sharepoint_tenant_id': ONEDRIVE_SHAREPOINT_TENANT_ID,
|
||||
'rag.content_extraction_engine': CONTENT_EXTRACTION_ENGINE,
|
||||
'rag.content_extraction.supported_media_mime_types': CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
|
||||
'rag.datalab_marker_api_key': DATALAB_MARKER_API_KEY,
|
||||
'rag.datalab_marker_api_base_url': DATALAB_MARKER_API_BASE_URL,
|
||||
'rag.datalab_marker_additional_config': DATALAB_MARKER_ADDITIONAL_CONFIG,
|
||||
@@ -3049,6 +3063,7 @@ DEFAULT_CONFIG = {
|
||||
'folders.enable': ENABLE_FOLDERS,
|
||||
'folders.max_file_count': FOLDER_MAX_FILE_COUNT,
|
||||
'channels.enable': ENABLE_CHANNELS,
|
||||
'channels.model_response_mode': CHANNEL_MODEL_RESPONSE_MODE,
|
||||
'calendar.enable': ENABLE_CALENDAR,
|
||||
'automations.enable': ENABLE_AUTOMATIONS,
|
||||
'subagents.enable': ENABLE_SUBAGENTS,
|
||||
|
||||
@@ -117,6 +117,7 @@ ADMIN_CONFIG_KEYS = {
|
||||
'AUTOMATION_MIN_INTERVAL': 'automations.min_interval',
|
||||
'ENABLE_AUTOMATIONS': 'automations.enable',
|
||||
'ENABLE_CHANNELS': 'channels.enable',
|
||||
'CHANNEL_MODEL_RESPONSE_MODE': 'channels.model_response_mode',
|
||||
'ENABLE_CALENDAR': 'calendar.enable',
|
||||
'ENABLE_MEMORIES': 'memories.enable',
|
||||
'ENABLE_MEMORY_SYSTEM_CONTEXT': 'memories.system_context.enable',
|
||||
@@ -1197,6 +1198,7 @@ class AdminConfig(BaseModel):
|
||||
AUTOMATION_MIN_INTERVAL: int | str | None = None
|
||||
ENABLE_AUTOMATIONS: bool
|
||||
ENABLE_CHANNELS: bool
|
||||
CHANNEL_MODEL_RESPONSE_MODE: str = 'thread'
|
||||
ENABLE_CALENDAR: bool
|
||||
ENABLE_MEMORIES: bool
|
||||
ENABLE_MEMORY_SYSTEM_CONTEXT: bool
|
||||
@@ -1220,6 +1222,9 @@ async def update_admin_config(request: Request, form_data: AdminConfig, user=Dep
|
||||
if form_data.DEFAULT_USER_ROLE not in ['pending', 'user', 'admin']:
|
||||
updates.pop('ui.default_user_role', None)
|
||||
|
||||
if form_data.CHANNEL_MODEL_RESPONSE_MODE not in ['thread', 'channel']:
|
||||
updates.pop('channels.model_response_mode', None)
|
||||
|
||||
pattern = r'^(-1|0|(-?\d+(\.\d+)?)(ms|s|m|h|d|w))$'
|
||||
|
||||
# Check if the input string matches the pattern
|
||||
|
||||
@@ -987,13 +987,18 @@ async def model_response_handler(request, channel, message, user, db=None):
|
||||
db=db,
|
||||
)
|
||||
)[::-1]
|
||||
response_parent_id = (
|
||||
message.parent_id
|
||||
if message.parent_id
|
||||
else (message.id if await Config.get('channels.model_response_mode', 'thread') == 'thread' else None)
|
||||
)
|
||||
|
||||
response_message, channel = await new_message_handler(
|
||||
request,
|
||||
channel.id,
|
||||
MessageForm(
|
||||
**{
|
||||
'parent_id': (message.parent_id if message.parent_id else message.id),
|
||||
'parent_id': response_parent_id,
|
||||
'content': f'',
|
||||
'data': {},
|
||||
'meta': {
|
||||
|
||||
@@ -108,6 +108,23 @@ def _cleanup_local_cache(file_path: str) -> None:
|
||||
log.warning(f'Failed to clean up local cache for {file_path}: {e}')
|
||||
|
||||
|
||||
def _matches_configured_mime_type(supported: list[str] | str, content_type: str) -> bool:
|
||||
if isinstance(supported, str):
|
||||
supported = supported.split(',')
|
||||
supported = [item.strip() for item in (supported or []) if item.strip()]
|
||||
if not supported:
|
||||
return False
|
||||
return bool(strict_match_mime_type(supported, content_type))
|
||||
|
||||
|
||||
def _media_supported_for_extraction(
|
||||
content_extraction_engine: str | None, supported: list[str] | str | None, content_type: str
|
||||
) -> bool:
|
||||
if supported is None:
|
||||
return content_extraction_engine == 'external'
|
||||
return bool(content_extraction_engine and _matches_configured_mime_type(supported, content_type))
|
||||
|
||||
|
||||
async def process_uploaded_file(
|
||||
request,
|
||||
file,
|
||||
@@ -127,6 +144,10 @@ async def process_uploaded_file(
|
||||
content_type = 'text/plain'
|
||||
|
||||
stt_supported = await Config.get('audio.stt.supported_content_types', [])
|
||||
content_extraction_engine = await Config.get('rag.content_extraction_engine')
|
||||
content_extraction_supported_media_mime_types = await Config.get(
|
||||
'rag.content_extraction.supported_media_mime_types'
|
||||
)
|
||||
|
||||
if content_type and strict_match_mime_type(stt_supported, content_type):
|
||||
# Audio / STT-supported files → transcribe then index
|
||||
@@ -147,9 +168,10 @@ async def process_uploaded_file(
|
||||
elif (
|
||||
content_type
|
||||
and content_type.startswith(('image/', 'video/'))
|
||||
and await Config.get('rag.content_extraction_engine') != 'external'
|
||||
and not _media_supported_for_extraction(
|
||||
content_extraction_engine, content_extraction_supported_media_mime_types, content_type
|
||||
)
|
||||
):
|
||||
# Media files without an external extraction engine
|
||||
if content_type.startswith('video/'):
|
||||
# Videos are stored as-is for downstream multimodal
|
||||
# processing (Tools, vision models). Attempting text
|
||||
@@ -165,7 +187,8 @@ async def process_uploaded_file(
|
||||
raise Exception(f'File type {content_type} is not supported for processing')
|
||||
|
||||
else:
|
||||
# Documents, or any file when an external engine is configured
|
||||
# Documents, or media files explicitly enabled for the
|
||||
# configured content extraction engine.
|
||||
if not content_type:
|
||||
log.info(f'File type {file.content_type} is not provided, but trying to process anyway')
|
||||
await process_file(
|
||||
|
||||
@@ -265,6 +265,7 @@ RETRIEVAL_CONFIG_KEYS = {
|
||||
'CHUNK_MIN_SIZE_TARGET': 'rag.chunk_min_size_target',
|
||||
'CHUNK_OVERLAP': 'rag.chunk_overlap',
|
||||
'CHUNK_SIZE': 'rag.chunk_size',
|
||||
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': 'rag.content_extraction.supported_media_mime_types',
|
||||
'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
|
||||
'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config',
|
||||
'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
|
||||
@@ -630,6 +631,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
||||
'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT,
|
||||
# Content extraction settings
|
||||
'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE,
|
||||
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
|
||||
'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES,
|
||||
'PDF_LOADER_MODE': config.PDF_LOADER_MODE,
|
||||
'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY,
|
||||
@@ -861,6 +863,7 @@ class ConfigForm(BaseModel):
|
||||
|
||||
# Content extraction settings
|
||||
CONTENT_EXTRACTION_ENGINE: str | None = None
|
||||
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES: list[str] | None = None
|
||||
PDF_EXTRACT_IMAGES: bool | None = None
|
||||
PDF_LOADER_MODE: str | None = None
|
||||
|
||||
@@ -973,6 +976,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
||||
if form_data.CONTENT_EXTRACTION_ENGINE is not None
|
||||
else config.CONTENT_EXTRACTION_ENGINE
|
||||
)
|
||||
config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = (
|
||||
form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES
|
||||
if form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES is not None
|
||||
else config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES
|
||||
)
|
||||
config.PDF_EXTRACT_IMAGES = (
|
||||
form_data.PDF_EXTRACT_IMAGES if form_data.PDF_EXTRACT_IMAGES is not None else config.PDF_EXTRACT_IMAGES
|
||||
)
|
||||
@@ -1333,6 +1341,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
||||
'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT,
|
||||
# Content extraction settings
|
||||
'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE,
|
||||
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
|
||||
'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES,
|
||||
'PDF_LOADER_MODE': config.PDF_LOADER_MODE,
|
||||
'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY,
|
||||
|
||||
Reference in new issue
Block a user