refac

backend/open_webui/env.py

@@ -753,6 +753,9 @@ BYPASS_RETRIEVAL_ACCESS_CONTROL = os.getenv('BYPASS_RETRIEVAL_ACCESS_CONTROL', '
 # for non-admin users.  When False (default), unknown collection names are
 # denied — closing the legacy unscoped namespace.
 ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS = os.getenv('ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS', 'False').lower() == 'true'
+MINERU_MAX_MARKDOWN_BYTES = (
+    int(os.getenv('MINERU_MAX_MARKDOWN_BYTES')) if os.getenv('MINERU_MAX_MARKDOWN_BYTES') else None
+)
 
 # When enabled, skips pydub-based preprocessing (format conversion, compression,
 # and chunked splitting) before sending files to processing engines. Useful when

backend/open_webui/retrieval/loaders/main.py

@@ -17,7 +17,12 @@ from langchain_community.document_loaders import (
     YoutubeLoader,
 )
 from langchain_core.documents import Document
-from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, GLOBAL_LOG_LEVEL, REQUESTS_VERIFY
+from open_webui.env import (
+    AIOHTTP_CLIENT_SESSION_SSL,
+    GLOBAL_LOG_LEVEL,
+    MINERU_MAX_MARKDOWN_BYTES,
+    REQUESTS_VERIFY,
+)
 from open_webui.retrieval.loaders.datalab_marker import DatalabMarkerLoader
 from open_webui.retrieval.loaders.external_document import ExternalDocumentLoader
 from open_webui.retrieval.loaders.mineru import MinerULoader
@@ -518,7 +523,6 @@ class Loader:
                     mineru_timeout = int(mineru_timeout)
                 except ValueError:
                     mineru_timeout = 300
-
             loader = MinerULoader(
                 file_path=file_path,
                 api_mode=self.kwargs.get('MINERU_API_MODE', 'local'),
@@ -526,6 +530,7 @@ class Loader:
                 api_key=self.kwargs.get('MINERU_API_KEY', ''),
                 params=self.kwargs.get('MINERU_PARAMS', {}),
                 timeout=mineru_timeout,
+                max_markdown_bytes=MINERU_MAX_MARKDOWN_BYTES,
             )
         elif (
             self.engine == 'mistral_ocr'