perf: build info log messages lazily so raising the log level actually saves work (#27837)

Raising GLOBAL_LOG_LEVEL to WARNING buys quieter output but not less work: 241 INFO call sites interpolate their payload into an f-string before the logging call gets to drop it. The heaviest is get_doc, which logs every chunk id and metadata dict in a collection, so on the full-context retrieval path that is the entire knowledge base, once per chat request.

That one line at WARNING, CPython 3.12:

| knowledge base | payload | before   | after   |
| -------------- | ------- | -------- | ------- |
| top-k of 3     | 1.2 kB  | 3.8 us   | 0.07 us |
| 500 chunks     | 201 kB  | 583.6 us | 0.08 us |
| 5000 chunks    | 2.0 MB  | 5.8 ms   | 0.15 us |

The lazy form log.info('query_doc:result %s %s', result.ids, result.metadatas) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. Output at INFO is byte-identical. Two sites that already built their message eagerly, one str concat and one % operator, move to the same lazy form.
This commit is contained in:
Classic298 authored and GitHub committed 2026-08-02 15:39:10 -05:00
1 parent 52145eede9
commit 2d18727ab8
48 files changed
+297 -247

No files matched your search

@@ -125,7 +125,7 @@ class MilvusClient(VectorDBBase):
index_type = MILVUS_INDEX_TYPE.upper()
metric_type = MILVUS_METRIC_TYPE.upper()
log.info(f'Using Milvus index type: {index_type}, metric type: {metric_type}')
log.info('Using Milvus index type: %s, metric type: %s', index_type, metric_type)
index_creation_params = {}
if index_type == 'HNSW':
@@ -133,18 +133,18 @@ class MilvusClient(VectorDBBase):
'M': MILVUS_HNSW_M,
'efConstruction': MILVUS_HNSW_EFCONSTRUCTION,
}
log.info(f'HNSW params: {index_creation_params}')
log.info('HNSW params: %s', index_creation_params)
elif index_type == 'IVF_FLAT':
index_creation_params = {'nlist': MILVUS_IVF_FLAT_NLIST}
log.info(f'IVF_FLAT params: {index_creation_params}')
log.info('IVF_FLAT params: %s', index_creation_params)
elif index_type == 'DISKANN':
index_creation_params = {
'max_degree': MILVUS_DISKANN_MAX_DEGREE,
'search_list_size': MILVUS_DISKANN_SEARCH_LIST_SIZE,
}
log.info(f'DISKANN params: {index_creation_params}')
log.info('DISKANN params: %s', index_creation_params)
elif index_type in ['FLAT', 'AUTOINDEX']:
log.info(f'Using {index_type} index with no specific build-time params.')
log.info('Using %s index with no specific build-time params.', index_type)
else:
log.warning(
f"Unsupported MILVUS_INDEX_TYPE: '{index_type}'. "
@@ -167,7 +167,11 @@ class MilvusClient(VectorDBBase):
index_params=index_params,
)
log.info(
f"Successfully created collection '{self.collection_prefix}_{collection_name}' with index type '{index_type}' and metric '{metric_type}'."
"Successfully created collection '%s_%s' with index type '%s' and metric '%s'.",
self.collection_prefix,
collection_name,
index_type,
metric_type,
)
def has_collection(self, collection_name: str) -> bool:
@@ -220,7 +224,11 @@ class MilvusClient(VectorDBBase):
try:
log.info(
f"Querying collection {self.collection_prefix}_{collection_name} with filter: '{filter_string}', limit: {limit}"
"Querying collection %s_%s with filter: '%s', limit: %s",
self.collection_prefix,
collection_name,
filter_string,
limit,
)
iterator = self.client.query_iterator(
@@ -265,7 +273,7 @@ class MilvusClient(VectorDBBase):
# Insert the items into the collection, if the collection does not exist, it will be created.
collection_name = collection_name.replace('-', '_')
if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'):
log.info(f'Collection {self.collection_prefix}_{collection_name} does not exist. Creating now.')
log.info('Collection %s_%s does not exist. Creating now.', self.collection_prefix, collection_name)
if not items:
log.error(
f'Cannot create collection {self.collection_prefix}_{collection_name} without items to determine dimension.'
@@ -273,7 +281,7 @@ class MilvusClient(VectorDBBase):
raise ValueError('Cannot create Milvus collection without items to determine vector dimension.')
self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector']))
log.info(f'Inserting {len(items)} items into collection {self.collection_prefix}_{collection_name}.')
log.info('Inserting %s items into collection %s_%s.', len(items), self.collection_prefix, collection_name)
data = []
for item in items:
text = item['text'] or ''
@@ -301,7 +309,9 @@ class MilvusClient(VectorDBBase):
# Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created.
collection_name = collection_name.replace('-', '_')
if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'):
log.info(f'Collection {self.collection_prefix}_{collection_name} does not exist for upsert. Creating now.')
log.info(
'Collection %s_%s does not exist for upsert. Creating now.', self.collection_prefix, collection_name
)
if not items:
log.error(
f'Cannot create collection {self.collection_prefix}_{collection_name} for upsert without items to determine dimension.'
@@ -311,7 +321,7 @@ class MilvusClient(VectorDBBase):
)
self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector']))
log.info(f'Upserting {len(items)} items into collection {self.collection_prefix}_{collection_name}.')
log.info('Upserting %s items into collection %s_%s.', len(items), self.collection_prefix, collection_name)
data = []
for item in items:
text = item['text'] or ''
@@ -348,7 +358,7 @@ class MilvusClient(VectorDBBase):
return None
if ids:
log.info(f'Deleting items by IDs from {self.collection_prefix}_{collection_name}. IDs: {ids}')
log.info('Deleting items by IDs from %s_%s. IDs: %s', self.collection_prefix, collection_name, ids)
return self.client.delete(
collection_name=f'{self.collection_prefix}_{collection_name}',
ids=ids,
@@ -356,7 +366,10 @@ class MilvusClient(VectorDBBase):
elif filter:
filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()])
log.info(
f'Deleting items by filter from {self.collection_prefix}_{collection_name}. Filter: {filter_string}'
'Deleting items by filter from %s_%s. Filter: %s',
self.collection_prefix,
collection_name,
filter_string,
)
return self.client.delete(
collection_name=f'{self.collection_prefix}_{collection_name}',
@@ -378,7 +391,7 @@ class MilvusClient(VectorDBBase):
try:
self.client.drop_collection(collection_name=collection_name_full)
deleted_collections.append(collection_name_full)
log.info(f'Deleted collection: {collection_name_full}')
log.info('Deleted collection: %s', collection_name_full)
except Exception as e:
log.error(f'Error deleting collection {collection_name_full}: {e}')
log.info(f'Milvus reset complete. Deleted collections: {deleted_collections}')
log.info('Milvus reset complete. Deleted collections: %s', deleted_collections)
@@ -146,7 +146,7 @@ class MilvusClient(VectorDBBase):
# The index only accelerates resource_id filters; never fail
# collection creation over it.
log.warning(f'Could not create {RESOURCE_ID_FIELD} index on {mt_collection_name}: {e}')
log.info(f'Created shared collection: {mt_collection_name}')
log.info('Created shared collection: %s', mt_collection_name)
def _ensure_collection(self, mt_collection_name: str, dimension: int):
if not self.client.has_collection(mt_collection_name):
@@ -181,7 +181,7 @@ class OpenGaussClient(VectorDBBase):
new_items.append(new_chunk)
self.session.bulk_save_objects(new_items)
self.session.commit()
log.info(f"Inserting {len(new_items)} items into collection '{collection_name}'.")
log.info("Inserting %s items into collection '%s'.", len(new_items), collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Failed to insert data: {e}')
@@ -207,7 +207,7 @@ class OpenGaussClient(VectorDBBase):
)
self.session.add(new_chunk)
self.session.commit()
log.info(f"Inserting/updating {len(items)} items in collection '{collection_name}'.")
log.info("Inserting/updating %s items in collection '%s'.", len(items), collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Failed to insert or update data.: {e}')
@@ -352,7 +352,7 @@ class OpenGaussClient(VectorDBBase):
query = query.filter(DocumentChunk.vmetadata[key].astext == str(value))
deleted = query.delete(synchronize_session=False)
self.session.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'")
log.info("Deleted %s items from collection '%s'", deleted, collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Failed to delete data: {e}')
@@ -362,7 +362,7 @@ class OpenGaussClient(VectorDBBase):
try:
deleted = self.session.query(DocumentChunk).delete()
self.session.commit()
log.info(f'Reset completed. Deleted {deleted} items')
log.info('Reset completed. Deleted %s items', deleted)
except Exception as e:
self.session.rollback()
log.exception(f'Reset failed: {e}')
@@ -386,4 +386,4 @@ class OpenGaussClient(VectorDBBase):
def delete_collection(self, collection_name: str) -> None:
self.delete(collection_name)
log.info(f"Collection '{collection_name}' has been deleted")
log.info("Collection '%s' has been deleted", collection_name)
@@ -94,10 +94,10 @@ class Oracle23aiClient(VectorDBBase):
self._create_dbcs_pool()
dsn = ORACLE_DB_DSN
log.info(f'Creating Connection Pool [{ORACLE_DB_USER}:**@{dsn}]')
log.info('Creating Connection Pool [%s:**@%s]', ORACLE_DB_USER, dsn)
with self.get_connection() as connection:
log.info(f'Connection version: {connection.version}')
log.info('Connection version: %s', connection.version)
self._initialize_database(connection)
log.info('Oracle Vector Search initialization complete.')
@@ -159,7 +159,7 @@ class Oracle23aiClient(VectorDBBase):
if attempt < max_retries - 1:
wait_time = 2**attempt
log.info(f'Retrying in {wait_time} seconds...')
log.info('Retrying in %s seconds...', wait_time)
time.sleep(wait_time)
else:
raise
@@ -184,7 +184,7 @@ class Oracle23aiClient(VectorDBBase):
thread = threading.Thread(target=_monitor, daemon=True)
thread.start()
log.info(f'Started DB health monitor every {interval_seconds} seconds.')
log.info('Started DB health monitor every %s seconds.', interval_seconds)
def _reconnect_pool(self):
"""
@@ -412,7 +412,7 @@ class Oracle23aiClient(VectorDBBase):
... ]
>>> client.insert("my_collection", items)
"""
log.info(f"Inserting {len(items)} items into collection '{collection_name}'.")
log.info("Inserting %s items into collection '%s'.", len(items), collection_name)
with self.get_connection() as connection:
try:
@@ -437,7 +437,7 @@ class Oracle23aiClient(VectorDBBase):
)
connection.commit()
log.info(f"Successfully inserted {len(items)} items into collection '{collection_name}'.")
log.info("Successfully inserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e:
connection.rollback()
@@ -466,7 +466,7 @@ class Oracle23aiClient(VectorDBBase):
... ]
>>> client.upsert("my_collection", items)
"""
log.info(f"Upserting {len(items)} items into collection '{collection_name}'.")
log.info("Upserting %s items into collection '%s'.", len(items), collection_name)
with self.get_connection() as connection:
try:
@@ -505,7 +505,7 @@ class Oracle23aiClient(VectorDBBase):
)
connection.commit()
log.info(f"Successfully upserted {len(items)} items into collection '{collection_name}'.")
log.info("Successfully upserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e:
connection.rollback()
@@ -541,7 +541,7 @@ class Oracle23aiClient(VectorDBBase):
... for i, (id, dist) in enumerate(zip(results.ids[0], results.distances[0])):
... log.info(f"Match {i+1}: id={id}, distance={dist}")
"""
log.info(f"Searching items from collection '{collection_name}' with limit {limit}.")
log.info("Searching items from collection '%s' with limit %s.", collection_name, limit)
try:
if not vectors:
@@ -587,7 +587,7 @@ class Oracle23aiClient(VectorDBBase):
metadatas[qid].append(self._json_to_metadata(metadata_str))
distances[qid].append(float(row[3]))
log.info(f'Search completed. Found {sum(len(ids[i]) for i in range(num_queries))} total results.')
log.info('Search completed. Found %s total results.', sum(len(ids[i]) for i in range(num_queries)))
return SearchResult(ids=ids, distances=distances, documents=documents, metadatas=metadatas)
@@ -616,7 +616,7 @@ class Oracle23aiClient(VectorDBBase):
>>> if results:
... print(f"Found {len(results.ids[0])} matching documents")
"""
log.info(f"Querying items from collection '{collection_name}' with filters.")
log.info("Querying items from collection '%s' with filters.", collection_name)
try:
limit = limit or 100
@@ -656,7 +656,7 @@ class Oracle23aiClient(VectorDBBase):
]
]
log.info(f'Query completed. Found {len(results)} results.')
log.info('Query completed. Found %s results.', len(results))
return GetResult(ids=ids, documents=documents, metadatas=metadatas)
@@ -747,7 +747,7 @@ class Oracle23aiClient(VectorDBBase):
>>> # Or delete by metadata filter
>>> client.delete("my_collection", filter={"source": "deprecated_source"})
"""
log.info(f"Deleting items from collection '{collection_name}'.")
log.info("Deleting items from collection '%s'.", collection_name)
try:
query = 'DELETE FROM document_chunk WHERE collection_name = :collection_name'
@@ -772,7 +772,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount
connection.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'.")
log.info("Deleted %s items from collection '%s'.", deleted, collection_name)
except Exception as e:
log.exception(f'Error during delete: {e}')
@@ -800,7 +800,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount
connection.commit()
log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.")
log.info("Reset complete. Deleted %s items from 'document_chunk' table.", deleted)
except Exception as e:
log.exception(f'Error during reset: {e}')
@@ -875,7 +875,7 @@ class Oracle23aiClient(VectorDBBase):
>>> client = Oracle23aiClient()
>>> client.delete_collection("obsolete_collection")
"""
log.info(f"Deleting collection '{collection_name}'.")
log.info("Deleting collection '%s'.", collection_name)
try:
with self.get_connection() as connection:
@@ -891,7 +891,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount
connection.commit()
log.info(f"Collection '{collection_name}' deleted. Removed {deleted} items.")
log.info("Collection '%s' deleted. Removed %s items.", collection_name, deleted)
except Exception as e:
log.exception(f"Error deleting collection '{collection_name}': {e}")
@@ -326,7 +326,7 @@ class PgvectorClient(VectorDBBase):
},
)
self.session.commit()
log.info(f"Encrypted & inserted {len(items)} into '{collection_name}'")
log.info("Encrypted & inserted %s into '%s'", len(items), collection_name)
else:
new_items = []
@@ -342,7 +342,7 @@ class PgvectorClient(VectorDBBase):
new_items.append(new_chunk)
self.session.bulk_save_objects(new_items)
self.session.commit()
log.info(f"Inserted {len(new_items)} items into collection '{collection_name}'.")
log.info("Inserted %s items into collection '%s'.", len(new_items), collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Error during insert: {e}')
@@ -381,7 +381,7 @@ class PgvectorClient(VectorDBBase):
},
)
self.session.commit()
log.info(f"Encrypted & upserted {len(items)} into '{collection_name}'")
log.info("Encrypted & upserted %s into '%s'", len(items), collection_name)
else:
for item in items:
vector = self.adjust_vector_length(item['vector'])
@@ -401,7 +401,7 @@ class PgvectorClient(VectorDBBase):
)
self.session.add(new_chunk)
self.session.commit()
log.info(f"Upserted {len(items)} items into collection '{collection_name}'.")
log.info("Upserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Error during upsert: {e}')
@@ -712,7 +712,7 @@ class PgvectorClient(VectorDBBase):
query = query.filter(DocumentChunk.vmetadata[key].astext == str(value))
deleted = query.delete(synchronize_session=False)
self.session.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'.")
log.info("Deleted %s items from collection '%s'.", deleted, collection_name)
except Exception as e:
self.session.rollback()
log.exception(f'Error during delete: {e}')
@@ -722,7 +722,7 @@ class PgvectorClient(VectorDBBase):
try:
deleted = self.session.query(DocumentChunk).delete()
self.session.commit()
log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.")
log.info("Reset complete. Deleted %s items from 'document_chunk' table.", deleted)
except Exception as e:
self.session.rollback()
log.exception(f'Error during reset: {e}')
@@ -746,4 +746,4 @@ class PgvectorClient(VectorDBBase):
def delete_collection(self, collection_name: str) -> None:
self.delete(collection_name)
log.info(f"Collection '{collection_name}' deleted.")
log.info("Collection '%s' deleted.", collection_name)
@@ -106,16 +106,16 @@ class PineconeClient(VectorDBBase):
try:
# Check if index exists
if self.index_name not in self.client.list_indexes().names():
log.info(f"Creating Pinecone index '{self.index_name}'...")
log.info("Creating Pinecone index '%s'...", self.index_name)
self.client.create_index(
name=self.index_name,
dimension=self.dimension,
metric=self.metric,
spec=ServerlessSpec(cloud=self.cloud, region=self.environment),
)
log.info(f"Successfully created Pinecone index '{self.index_name}'")
log.info("Successfully created Pinecone index '%s'", self.index_name)
else:
log.info(f"Using existing Pinecone index '{self.index_name}'")
log.info("Using existing Pinecone index '%s'", self.index_name)
# Connect to the index
self.index = self.client.Index(
@@ -245,7 +245,7 @@ class PineconeClient(VectorDBBase):
collection_name_with_prefix = self._get_collection_name_with_prefix(collection_name)
try:
self.index.delete(filter={'collection_name': collection_name_with_prefix})
log.info(f"Collection '{collection_name_with_prefix}' deleted (all vectors removed).")
log.info("Collection '%s' deleted (all vectors removed).", collection_name_with_prefix)
except Exception as e:
log.warning(f"Failed to delete collection '{collection_name_with_prefix}': {e}")
raise
@@ -276,7 +276,7 @@ class PineconeClient(VectorDBBase):
elapsed = time.time() - start_time
log.debug('Insert of %s vectors took %.2f seconds', len(points), elapsed)
log.info(
f"Successfully inserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
"Successfully inserted %s vectors in parallel batches into '%s'", len(points), collection_name_with_prefix
)
def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
@@ -305,7 +305,7 @@ class PineconeClient(VectorDBBase):
elapsed = time.time() - start_time
log.debug('Upsert of %s vectors took %.2f seconds', len(points), elapsed)
log.info(
f"Successfully upserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
"Successfully upserted %s vectors in parallel batches into '%s'", len(points), collection_name_with_prefix
)
async def insert_async(self, collection_name: str, items: List[VectorItem]) -> None:
@@ -326,7 +326,9 @@ class PineconeClient(VectorDBBase):
if isinstance(result, Exception):
log.error(f'Error in async insert batch: {result}')
raise result
log.info(f"Successfully async inserted {len(points)} vectors in batches into '{collection_name_with_prefix}'")
log.info(
"Successfully async inserted %s vectors in batches into '%s'", len(points), collection_name_with_prefix
)
async def upsert_async(self, collection_name: str, items: List[VectorItem]) -> None:
"""Async version of upsert using asyncio and run_in_executor for improved performance."""
@@ -346,7 +348,9 @@ class PineconeClient(VectorDBBase):
if isinstance(result, Exception):
log.error(f'Error in async upsert batch: {result}')
raise result
log.info(f"Successfully async upserted {len(points)} vectors in batches into '{collection_name_with_prefix}'")
log.info(
"Successfully async upserted %s vectors in batches into '%s'", len(points), collection_name_with_prefix
)
def search(
self,
@@ -477,7 +481,7 @@ class PineconeClient(VectorDBBase):
log.debug(
"Deleted batch of %s vectors by ID from '%s'", len(batch_ids), collection_name_with_prefix
)
log.info(f"Successfully deleted {len(ids)} vectors by ID from '{collection_name_with_prefix}'")
log.info("Successfully deleted %s vectors by ID from '%s'", len(ids), collection_name_with_prefix)
elif filter:
# Combine user filter with collection_name
@@ -486,7 +490,7 @@ class PineconeClient(VectorDBBase):
pinecone_filter.update(filter)
# Delete by metadata filter
self.index.delete(filter=pinecone_filter)
log.info(f"Successfully deleted vectors by filter from '{collection_name_with_prefix}'")
log.info("Successfully deleted vectors by filter from '%s'", collection_name_with_prefix)
else:
log.warning('No ids or filter provided for delete operation')
@@ -119,7 +119,7 @@ class QdrantClient(VectorDBBase):
on_disk=self.QDRANT_ON_DISK,
),
)
log.info(f'collection {collection_name_with_prefix} successfully created!')
log.info('collection %s successfully created!', collection_name_with_prefix)
def _create_collection_if_not_exists(self, collection_name, dimension):
if not self.has_collection(collection_name=collection_name):
@@ -148,7 +148,7 @@ class QdrantClient(VectorDBBase):
m=0,
),
)
log.info(f'Multi-tenant collection {mt_collection_name} created with dimension {dimension}!')
log.info('Multi-tenant collection %s created with dimension %s!', mt_collection_name, dimension)
self.client.create_payload_index(
collection_name=mt_collection_name,
@@ -36,7 +36,7 @@ class S3VectorClient(VectorDBBase):
if self.bucket_name and self.region:
try:
self.client = boto3.client('s3vectors', region_name=self.region)
log.info(f"S3Vector client initialized for bucket '{self.bucket_name}' in region '{self.region}'")
log.info("S3Vector client initialized for bucket '%s' in region '%s'", self.bucket_name, self.region)
except Exception as e:
log.error(f'Failed to initialize S3Vector client: {e}')
self.client = None
@@ -70,7 +70,9 @@ class S3VectorClient(VectorDBBase):
]
},
)
log.info(f'Created S3 index: {index_name} (dim={dimension}, type={data_type}, metric={distance_metric})')
log.info(
'Created S3 index: %s (dim=%s, type=%s, metric=%s)', index_name, dimension, data_type, distance_metric
)
except Exception as e:
log.error(f"Error creating S3 index '{index_name}': {e}")
raise
@@ -137,9 +139,9 @@ class S3VectorClient(VectorDBBase):
return
try:
log.info(f"Deleting collection '{collection_name}'")
log.info("Deleting collection '%s'", collection_name)
self.client.delete_index(vectorBucketName=self.bucket_name, indexName=collection_name)
log.info(f"Successfully deleted collection '{collection_name}'")
log.info("Successfully deleted collection '%s'", collection_name)
except Exception as e:
log.error(f"Error deleting collection '{collection_name}': {e}")
raise
@@ -156,7 +158,7 @@ class S3VectorClient(VectorDBBase):
try:
if not self.has_collection(collection_name):
log.info(f"Index '{collection_name}' does not exist. Creating index.")
log.info("Index '%s' does not exist. Creating index.", collection_name)
self._create_index(
index_name=collection_name,
dimension=dimension,
@@ -202,9 +204,11 @@ class S3VectorClient(VectorDBBase):
indexName=collection_name,
vectors=batch,
)
log.info(f"Inserted batch {i // batch_size + 1}: {len(batch)} vectors into index '{collection_name}'.")
log.info(
"Inserted batch %s: %s vectors into index '%s'.", i // batch_size + 1, len(batch), collection_name
)
log.info(f"Completed insertion of {len(vectors)} vectors into index '{collection_name}'.")
log.info("Completed insertion of %s vectors into index '%s'.", len(vectors), collection_name)
except Exception as e:
log.error(f'Error inserting vectors: {e}')
raise
@@ -218,11 +222,11 @@ class S3VectorClient(VectorDBBase):
return
dimension = len(items[0]['vector'])
log.info(f'Upsert dimension: {dimension}')
log.info('Upsert dimension: %s', dimension)
try:
if not self.has_collection(collection_name):
log.info(f"Index '{collection_name}' does not exist. Creating index for upsert.")
log.info("Index '%s' does not exist. Creating index for upsert.", collection_name)
self._create_index(
index_name=collection_name,
dimension=dimension,
@@ -264,10 +268,14 @@ class S3VectorClient(VectorDBBase):
batch = vectors[i : i + batch_size]
if i == 0: # Log sample info for first batch only
log.info(
f'Upserting batch 1: {len(batch)} vectors. First vector sample: key={batch[0]["key"]}, data_type={type(batch[0]["data"]["float32"])}, data_len={len(batch[0]["data"]["float32"])}'
'Upserting batch 1: %s vectors. First vector sample: key=%s, data_type=%s, data_len=%s',
len(batch),
batch[0]['key'],
type(batch[0]['data']['float32']),
len(batch[0]['data']['float32']),
)
else:
log.info(f'Upserting batch {i // batch_size + 1}: {len(batch)} vectors.')
log.info('Upserting batch %s: %s vectors.', i // batch_size + 1, len(batch))
self.client.put_vectors(
vectorBucketName=self.bucket_name,
@@ -275,7 +283,7 @@ class S3VectorClient(VectorDBBase):
vectors=batch,
)
log.info(f"Completed upsert of {len(vectors)} vectors into index '{collection_name}'.")
log.info("Completed upsert of %s vectors into index '%s'.", len(vectors), collection_name)
except Exception as e:
log.error(f'Error upserting vectors: {e}')
raise
@@ -300,7 +308,7 @@ class S3VectorClient(VectorDBBase):
return None
try:
log.info(f"Searching collection '{collection_name}' with {len(vectors)} query vectors, limit={limit}")
log.info("Searching collection '%s' with %s query vectors, limit=%s", collection_name, len(vectors), limit)
# Initialize result lists
all_ids = []
@@ -362,7 +370,7 @@ class S3VectorClient(VectorDBBase):
all_metadatas.append(query_metadatas)
all_distances.append(query_distances)
log.info(f'Search completed. Found results for {len(all_ids)} queries')
log.info('Search completed. Found results for %s queries', len(all_ids))
# Return SearchResult format
return SearchResult(
@@ -402,7 +410,7 @@ class S3VectorClient(VectorDBBase):
return self.get(collection_name)
try:
log.info(f"Querying collection '{collection_name}' with filter: {filter}")
log.info("Querying collection '%s' with filter: %s", collection_name, filter)
# For S3 Vector, we need to use list_vectors and then filter results
# Since S3 Vector may not support complex server-side filtering,
@@ -437,7 +445,7 @@ class S3VectorClient(VectorDBBase):
if limit and len(filtered_ids) >= limit:
break
log.info(f'Filter applied: {len(filtered_ids)} vectors match out of {len(all_ids)} total')
log.info('Filter applied: %s vectors match out of %s total', len(filtered_ids), len(all_ids))
# Return GetResult format
if filtered_ids:
@@ -472,7 +480,7 @@ class S3VectorClient(VectorDBBase):
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
try:
log.info(f"Retrieving all vectors from collection '{collection_name}'")
log.info("Retrieving all vectors from collection '%s'", collection_name)
# Initialize result lists
all_ids = []
@@ -534,7 +542,7 @@ class S3VectorClient(VectorDBBase):
if not next_token:
break
log.info(f"Retrieved {len(all_ids)} vectors from collection '{collection_name}'")
log.info("Retrieved %s vectors from collection '%s'", len(all_ids), collection_name)
# Return in GetResult format
# The Open WebUI GetResult expects lists of lists, so we wrap each list
@@ -576,17 +584,17 @@ class S3VectorClient(VectorDBBase):
try:
if ids:
# Delete by specific vector IDs/keys
log.info(f"Deleting {len(ids)} vectors by IDs from collection '{collection_name}'")
log.info("Deleting %s vectors by IDs from collection '%s'", len(ids), collection_name)
self.client.delete_vectors(
vectorBucketName=self.bucket_name,
indexName=collection_name,
keys=ids,
)
log.info(f"Deleted {len(ids)} vectors from index '{collection_name}'")
log.info("Deleted %s vectors from index '%s'", len(ids), collection_name)
elif filter:
# Handle filter-based deletion
log.info(f"Deleting vectors by filter from collection '{collection_name}': {filter}")
log.info("Deleting vectors by filter from collection '%s': %s", collection_name, filter)
# If this is a knowledge collection and we have a file_id filter,
# also clean up the corresponding file-specific collection
@@ -595,7 +603,8 @@ class S3VectorClient(VectorDBBase):
file_collection_name = f'file-{file_id}'
if self.has_collection(file_collection_name):
log.info(
f"Found related file-specific collection '{file_collection_name}', deleting it to prevent duplicates"
"Found related file-specific collection '%s', deleting it to prevent duplicates",
file_collection_name,
)
self.delete_collection(file_collection_name)
@@ -604,7 +613,7 @@ class S3VectorClient(VectorDBBase):
query_result = self.query(collection_name, filter)
if query_result and query_result.ids and query_result.ids[0]:
matching_ids = query_result.ids[0]
log.info(f'Found {len(matching_ids)} vectors matching filter, deleting them')
log.info('Found %s vectors matching filter, deleting them', len(matching_ids))
# Delete the matching vectors by ID
self.client.delete_vectors(
@@ -612,7 +621,7 @@ class S3VectorClient(VectorDBBase):
indexName=collection_name,
keys=matching_ids,
)
log.info(f"Deleted {len(matching_ids)} vectors from index '{collection_name}' using filter")
log.info("Deleted %s vectors from index '%s' using filter", len(matching_ids), collection_name)
else:
log.warning('No vectors found matching the filter criteria')
else:
@@ -645,11 +654,11 @@ class S3VectorClient(VectorDBBase):
try:
self.client.delete_index(vectorBucketName=self.bucket_name, indexName=index_name)
deleted_count += 1
log.info(f'Deleted index: {index_name}')
log.info('Deleted index: %s', index_name)
except Exception as e:
log.error(f"Error deleting index '{index_name}': {e}")
log.info(f'Reset completed: deleted {deleted_count} indexes')
log.info('Reset completed: deleted %s indexes', deleted_count)
except Exception as e:
log.error(f'Error during reset: {e}')
@@ -279,7 +279,7 @@ class ValkeyClient(VectorDBBase):
f'{self._format_version(MIN_VALKEY_VERSION)}. valkey-search 1.2.0 requires Valkey core '
'9.0.1 or later. Upgrade your server or use valkey-bundle:9.1.0-rc2+.'
)
log.info(f'Valkey core version: {self._format_version(version) if version else "unknown"}')
log.info('Valkey core version: %s', self._format_version(version) if version else 'unknown')
def _check_search_module(self) -> None:
try:
@@ -331,7 +331,7 @@ class ValkeyClient(VectorDBBase):
'TEXT field type and filter-only FT.SEARCH support required by this backend. '
'Upgrade to valkey-bundle:9.1.0-rc2+ or load valkey-search 1.2.0+ as a module.'
)
log.info(f'valkey-search version: {self._format_version(search_version) if search_version else "unknown"}')
log.info('valkey-search version: %s', self._format_version(search_version) if search_version else 'unknown')
def _index_name(self, collection_name: str) -> str:
return f'idx:{self.collection_prefix}:{collection_name}'
@@ -385,8 +385,11 @@ class ValkeyClient(VectorDBBase):
try:
g['glide_ft'].create(self.client, index_name, schema, options)
log.info(
f'Created Valkey index {index_name} with dimension={dimension}, '
f'type={self.index_type}, metric={self.distance_metric}'
'Created Valkey index %s with dimension=%s, type=%s, metric=%s',
index_name,
dimension,
self.index_type,
self.distance_metric,
)
except g['RequestError'] as e:
if 'already exists' in str(e).lower():
@@ -456,7 +459,7 @@ class ValkeyClient(VectorDBBase):
index_name = self._index_name(collection_name)
try:
self._g['glide_ft'].dropindex(self.client, index_name)
log.info(f'Dropped index {index_name}')
log.info('Dropped index %s', index_name)
except self._g['RequestError'] as e:
log.debug('Could not drop index %s: %s', index_name, e)
@@ -656,7 +659,7 @@ class ValkeyClient(VectorDBBase):
collections.append(name[len(idx_prefix) :])
try:
glide_ft.dropindex(self.client, idx)
log.info(f'Dropped index: {name}')
log.info('Dropped index: %s', name)
except Exception as e:
log.error(f'Error dropping index {name}: {e}')
except Exception as e:
@@ -664,7 +667,7 @@ class ValkeyClient(VectorDBBase):
for collection in collections:
self._delete_keys_by_prefix(self._key_prefix(collection))
log.info(f'Valkey vector store reset complete (prefix: {self.collection_prefix})')
log.info('Valkey vector store reset complete (prefix: %s)', self.collection_prefix)
def _delete_keys_by_prefix(self, prefix: str) -> None:
cursor = '0'