refac

backend/open_webui/retrieval/vector/dbs/mariadb_vector.py

@@ -5,7 +5,6 @@ NOTE: This vector database integration is community-supported and maintained on
 from __future__ import annotations
 
 import array
-import json
 import logging
 import math
 import re
@@ -30,6 +29,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from sqlalchemy import create_engine
 from sqlalchemy.pool import NullPool, QueuePool
 
@@ -72,7 +72,7 @@ def _safe_json(v: Any) -> Dict[str, Any]:
             return {}
     if isinstance(v, str):
         try:
-            j = json.loads(v)
+            j = JSONCodec.loads(v)
             return j if isinstance(j, dict) else {}
         except Exception:
             return {}
@@ -324,7 +324,7 @@ class MariaDBVectorClient(VectorDBBase):
                                 emb,
                                 collection_name,
                                 item.get('text'),
-                                json.dumps(meta),
+                                JSONCodec.dumps(meta),
                             )
                         )
                     cur.executemany(sql, params)
@@ -367,7 +367,7 @@ class MariaDBVectorClient(VectorDBBase):
                                 emb,
                                 collection_name,
                                 item.get('text'),
-                                json.dumps(meta),
+                                JSONCodec.dumps(meta),
                             )
                         )
                     cur.executemany(sql, params)

backend/open_webui/retrieval/vector/dbs/milvus.py

@@ -2,7 +2,6 @@
 NOTE: This vector database integration is community-supported and maintained on a best-effort basis.
 """
 
-import json
 import logging
 from typing import Optional
 
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from pymilvus import DataType
 from pymilvus import MilvusClient as Client
 from pymilvus.exceptions import MilvusException
@@ -354,7 +354,7 @@ class MilvusClient(VectorDBBase):
                 ids=ids,
             )
         elif filter:
-            filter_string = ' && '.join([f'metadata["{key}"] == {json.dumps(value)}' for key, value in filter.items()])
+            filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()])
             log.info(
                 f'Deleting items by filter from {self.collection_prefix}_{collection_name}. Filter: {filter_string}'
             )

backend/open_webui/retrieval/vector/dbs/pgvector.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
     VectorItem,
 )
 from open_webui.retrieval.vector.utils import merge_hybrid_search_results, process_metadata
+from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.misc import sanitize_text_for_db
 from pgvector.sqlalchemy import HALFVEC, Vector
 from sqlalchemy import (
@@ -303,7 +303,7 @@ class PgvectorClient(VectorDBBase):
                     # Use raw SQL for BYTEA/pgcrypto
                     # Ensure metadata is converted to its JSON text representation
                     # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
-                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
                     item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""
@@ -354,7 +354,7 @@ class PgvectorClient(VectorDBBase):
                 for item in items:
                     vector = self.adjust_vector_length(item['vector'])
                     # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
-                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
                     item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""