refac

backend/open_webui/retrieval/loaders/datalab_marker.py

@@ -7,6 +7,7 @@ from typing import List, Optional
 import requests
 from fastapi import HTTPException, status
 from langchain_core.documents import Document
+from open_webui.utils.json_codec import JSONCodec
 
 log = logging.getLogger(__name__)
 
@@ -249,11 +250,11 @@ class DatalabMarkerLoader:
         images = final_result.get('images', {})
         if images:
             metadata['image_count'] = len(images)
-            metadata['images'] = json.dumps(list(images.keys()))
+            metadata['images'] = JSONCodec.dumps(list(images.keys()))
 
         for k, v in metadata.items():
             if isinstance(v, (dict, list)):
-                metadata[k] = json.dumps(v)
+                metadata[k] = JSONCodec.dumps(v)
             elif v is None:
                 metadata[k] = ''
 

backend/open_webui/retrieval/loaders/main.py

@@ -1,5 +1,4 @@
 import asyncio
-import json
 import logging
 import sys
 
@@ -28,6 +27,7 @@ from open_webui.retrieval.loaders.mineru import MinerULoader
 from open_webui.retrieval.loaders.mistral import MistralLoader
 from open_webui.retrieval.loaders.paddleocr_vl import PADDLEOCR_VL_SUPPORTED_EXTENSIONS, PaddleOCRVLLoader
 from open_webui.utils.headers import get_user_groups_for_custom_headers
+from open_webui.utils.json_codec import JSONCodec
 
 logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
 log = logging.getLogger(__name__)
@@ -508,8 +508,8 @@ class Loader:
                 params = self.kwargs.get('DOCLING_PARAMS', {})
                 if not isinstance(params, dict):
                     try:
-                        params = json.loads(params)
-                    except json.JSONDecodeError:
+                        params = JSONCodec.loads(params)
+                    except JSONCodec.JSONDecodeError:
                         log.error('Invalid DOCLING_PARAMS format, expected JSON object')
                         params = {}
 

backend/open_webui/retrieval/vector/dbs/mariadb_vector.py

@@ -5,7 +5,6 @@ NOTE: This vector database integration is community-supported and maintained on
 from __future__ import annotations
 
 import array
-import json
 import logging
 import math
 import re
@@ -30,6 +29,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from sqlalchemy import create_engine
 from sqlalchemy.pool import NullPool, QueuePool
 
@@ -72,7 +72,7 @@ def _safe_json(v: Any) -> Dict[str, Any]:
             return {}
     if isinstance(v, str):
         try:
-            j = json.loads(v)
+            j = JSONCodec.loads(v)
             return j if isinstance(j, dict) else {}
         except Exception:
             return {}
@@ -324,7 +324,7 @@ class MariaDBVectorClient(VectorDBBase):
                                 emb,
                                 collection_name,
                                 item.get('text'),
-                                json.dumps(meta),
+                                JSONCodec.dumps(meta),
                             )
                         )
                     cur.executemany(sql, params)
@@ -367,7 +367,7 @@ class MariaDBVectorClient(VectorDBBase):
                                 emb,
                                 collection_name,
                                 item.get('text'),
-                                json.dumps(meta),
+                                JSONCodec.dumps(meta),
                             )
                         )
                     cur.executemany(sql, params)

backend/open_webui/retrieval/vector/dbs/milvus.py

@@ -2,7 +2,6 @@
 NOTE: This vector database integration is community-supported and maintained on a best-effort basis.
 """
 
-import json
 import logging
 from typing import Optional
 
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from pymilvus import DataType
 from pymilvus import MilvusClient as Client
 from pymilvus.exceptions import MilvusException
@@ -354,7 +354,7 @@ class MilvusClient(VectorDBBase):
                 ids=ids,
             )
         elif filter:
-            filter_string = ' && '.join([f'metadata["{key}"] == {json.dumps(value)}' for key, value in filter.items()])
+            filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()])
             log.info(
                 f'Deleting items by filter from {self.collection_prefix}_{collection_name}. Filter: {filter_string}'
             )

backend/open_webui/retrieval/vector/dbs/pgvector.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import merge_hybrid_search_results, process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.misc import sanitize_text_for_db
 from pgvector.sqlalchemy import HALFVEC, Vector
 from sqlalchemy import (
@@ -303,7 +303,7 @@ class PgvectorClient(VectorDBBase):
                     # Use raw SQL for BYTEA/pgcrypto
                     # Ensure metadata is converted to its JSON text representation
                     # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
-                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
                     item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""
@@ -354,7 +354,7 @@ class PgvectorClient(VectorDBBase):
                 for item in items:
                     vector = self.adjust_vector_length(item['vector'])
                     # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
-                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
                     item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""

backend/open_webui/retrieval/web/bocha.py

@@ -1,9 +1,9 @@
-import json
 import logging
 from typing import Optional
 
 import requests
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.utils.json_codec import JSONCodec
 
 log = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ def search_bocha(api_key: str, query: str, count: int, filter_list: Optional[lis
     url = 'https://api.bochaai.com/v1/web-search?utm_source=ollama'
     headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
 
-    payload = json.dumps({'query': query, 'summary': True, 'freshness': 'noLimit', 'count': count})
+    payload = JSONCodec.dumps({'query': query, 'summary': True, 'freshness': 'noLimit', 'count': count})
 
     response = requests.post(url, headers=headers, data=payload, timeout=5)
     response.raise_for_status()

backend/open_webui/retrieval/web/serper.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-import json
 import logging
 
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.session_pool import get_session
 
 log = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ async def search_serper(
     headers = {'X-API-KEY': api_key, 'Content-Type': 'application/json'}
 
     session = await get_session()
-    async with session.post(url, headers=headers, data=json.dumps({'q': query})) as response:
+    async with session.post(url, headers=headers, data=JSONCodec.dumps({'q': query})) as response:
         response.raise_for_status()
         payload = await response.json()
 

backend/open_webui/retrieval/web/sougou.py

@@ -1,8 +1,8 @@
-import json
 import logging
 from typing import List, Optional
 
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.utils.json_codec import JSONCodec
 
 log = logging.getLogger(__name__)
 
@@ -28,10 +28,10 @@ def search_sougou(
         http_profile.endpoint = 'tms.tencentcloudapi.com'
         client_profile = ClientProfile()
         client_profile.http_profile = http_profile
-        params = json.dumps({'Query': query, 'Cnt': 20})
+        params = JSONCodec.dumps({'Query': query, 'Cnt': 20})
         common_client = CommonClient('tms', '2020-12-29', cred, '', profile=client_profile)
         results = [
-            json.loads(page) for page in common_client.call_json('SearchPro', json.loads(params))['Response']['Pages']
+            JSONCodec.loads(page) for page in common_client.call_json('SearchPro', JSONCodec.loads(params))['Response']['Pages']
         ]
         sorted_results = sorted(results, key=lambda x: x.get('scour', 0.0), reverse=True)
         if filter_list:

backend/open_webui/retrieval/web/yandex.py

@@ -1,6 +1,5 @@
 import base64
 import io
-import json
 import logging
 import os
 from typing import List, Optional
@@ -12,6 +11,7 @@ from fastapi import Request
 from open_webui.env import FORWARD_SESSION_INFO_HEADER_CHAT_ID
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
 from open_webui.utils.headers import include_user_info_headers
+from open_webui.utils.json_codec import JSONCodec
 
 log = logging.getLogger(__name__)
 
@@ -55,7 +55,7 @@ def search_yandex(
         if chat_id:
             headers[FORWARD_SESSION_INFO_HEADER_CHAT_ID] = str(chat_id)
 
-        payload = {} if yandex_search_config == '' else json.loads(yandex_search_config)
+        payload = {} if yandex_search_config == '' else JSONCodec.loads(yandex_search_config)
 
         if type(payload.get('query', None)) != dict:
             payload['query'] = {}