This commit is contained in:
Timothy Jaeryang Baek committed 2026-07-23 22:45:54 -04:00
1 parent 40320c1136
commit 43e7eefa95
1 file changed
+14 -16
+14 -16
View File
@@ -17,6 +17,7 @@ import mimeparse
from open_webui.env import CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE from open_webui.env import CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
SURROGATE_RE = re.compile('[\ud800-\udfff]')
def deep_update(d, u): def deep_update(d, u):
@@ -715,18 +716,10 @@ def sanitize_text_for_db(text: str) -> str:
"""Remove null bytes and invalid UTF-8 surrogates from text for PostgreSQL storage.""" """Remove null bytes and invalid UTF-8 surrogates from text for PostgreSQL storage."""
if not isinstance(text, str): if not isinstance(text, str):
return text return text
# Fast path: skip work when there are no null bytes (the common case) # Fast path: skip work when there are no null bytes or surrogate code points.
if '\x00' not in text: if '\x00' not in text and not SURROGATE_RE.search(text):
return text return text
# Remove null bytes return SURROGATE_RE.sub('', text.replace('\x00', ''))
text = text.replace('\x00', '').replace('\u0000', '')
# Remove invalid UTF-8 surrogate characters that can cause encoding errors
# This handles cases where binary data or encoding issues introduced surrogates
try:
text = text.encode('utf-8', errors='surrogatepass').decode('utf-8', errors='ignore')
except (UnicodeEncodeError, UnicodeDecodeError):
pass
return text
def _strip_null_bytes_deep(obj): def _strip_null_bytes_deep(obj):
@@ -734,7 +727,10 @@ def _strip_null_bytes_deep(obj):
if isinstance(obj, str): if isinstance(obj, str):
return sanitize_text_for_db(obj) return sanitize_text_for_db(obj)
elif isinstance(obj, dict): elif isinstance(obj, dict):
return {k: _strip_null_bytes_deep(v) for k, v in obj.items()} cleaned = {}
for k, v in obj.items():
cleaned[sanitize_text_for_db(k) if isinstance(k, str) else k] = _strip_null_bytes_deep(v)
return cleaned
elif isinstance(obj, list): elif isinstance(obj, list):
return [_strip_null_bytes_deep(v) for v in obj] return [_strip_null_bytes_deep(v) for v in obj]
return obj return obj
@@ -744,19 +740,21 @@ def sanitize_data_for_db(obj):
"""Recursively sanitize all strings in a data structure for database storage. """Recursively sanitize all strings in a data structure for database storage.
Performs a fast pre-check: serializes the structure once and scans for Performs a fast pre-check: serializes the structure once and scans for
null bytes. If none are found (the overwhelmingly common case), the null bytes or invalid UTF-8 surrogates. If none are found, the
original object is returned immediately, skipping the expensive original object is returned immediately, skipping the expensive
recursive walk. recursive walk.
""" """
if isinstance(obj, str): if isinstance(obj, str):
return sanitize_text_for_db(obj) return sanitize_text_for_db(obj)
# Fast path: check for null bytes in the serialized form. # Fast path: check for null bytes and surrogate code points in the serialized form.
# json.dumps is implemented in C and much faster than a Python-level # json.dumps is implemented in C and much faster than a Python-level
# recursive walk over every leaf string. # recursive walk over every leaf string.
try: try:
if '\\u0000' not in json.dumps(obj, ensure_ascii=False): serialized = json.dumps(obj, ensure_ascii=False)
if '\\u0000' not in serialized:
serialized.encode('utf-8')
return obj return obj
except (TypeError, ValueError): except (TypeError, ValueError, UnicodeEncodeError):
pass pass
return _strip_null_bytes_deep(obj) return _strip_null_bytes_deep(obj)