refac

backend/open_webui/env.py

@@ -822,6 +822,11 @@ BYPASS_RETRIEVAL_ACCESS_CONTROL = os.getenv('BYPASS_RETRIEVAL_ACCESS_CONTROL', '
 # for non-admin users.  When False (default), unknown collection names are
 # denied — closing the legacy unscoped namespace.
 ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS = os.getenv('ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS', 'False').lower() == 'true'
+
+RAG_METADATA_MAX_VALUE_CHARS = (
+    int(os.getenv('RAG_METADATA_MAX_VALUE_CHARS')) if os.getenv('RAG_METADATA_MAX_VALUE_CHARS') else None
+)
+
 MINERU_MAX_MARKDOWN_BYTES = (
     int(os.getenv('MINERU_MAX_MARKDOWN_BYTES')) if os.getenv('MINERU_MAX_MARKDOWN_BYTES') else None
 )

backend/open_webui/retrieval/vector/utils.py

@@ -1,6 +1,7 @@
 import datetime as dt
 from typing import Any
 
+from open_webui.env import RAG_METADATA_MAX_VALUE_CHARS
 from open_webui.retrieval.vector.main import SearchResult
 from open_webui.utils.misc import sanitize_text_for_db
 
@@ -20,8 +21,18 @@ KEYS_TO_EXCLUDE = [
 
 def filter_metadata(metadata: dict[str, any]) -> dict[str, any]:
     # Removes large/redundant fields from metadata dict.
-    metadata = {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE}
+    result = {}
-    return metadata
+    for key, value in metadata.items():
+        if key in KEYS_TO_EXCLUDE:
+            continue
+        if RAG_METADATA_MAX_VALUE_CHARS is not None and isinstance(value, (list, dict)):
+            try:
+                if len(str(value)) > RAG_METADATA_MAX_VALUE_CHARS:
+                    continue
+            except (MemoryError, RecursionError, ValueError):
+                continue
+        result[key] = value
+    return result
 
 
 def process_metadata(
@@ -36,6 +47,12 @@ def process_metadata(
             continue
         if value is None:
             continue
+        if RAG_METADATA_MAX_VALUE_CHARS is not None and isinstance(value, (list, dict)):
+            try:
+                if len(str(value)) > RAG_METADATA_MAX_VALUE_CHARS:
+                    continue
+            except (MemoryError, RecursionError, ValueError):
+                continue
         # Convert non-serializable fields to strings
         if isinstance(value, (dt.datetime, list, dict)):
             result[key] = sanitize_text_for_db(str(value))