perf: build info log messages lazily so raising the log level actually saves work (#27837)

Raising GLOBAL_LOG_LEVEL to WARNING buys quieter output but not less work: 241 INFO call sites interpolate their payload into an f-string before the logging call gets to drop it. The heaviest is get_doc, which logs every chunk id and metadata dict in a collection, so on the full-context retrieval path that is the entire knowledge base, once per chat request.

That one line at WARNING, CPython 3.12:

| knowledge base | payload | before   | after   |
| -------------- | ------- | -------- | ------- |
| top-k of 3     | 1.2 kB  | 3.8 us   | 0.07 us |
| 500 chunks     | 201 kB  | 583.6 us | 0.08 us |
| 5000 chunks    | 2.0 MB  | 5.8 ms   | 0.15 us |

The lazy form log.info('query_doc:result %s %s', result.ids, result.metadatas) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. Output at INFO is byte-identical. Two sites that already built their message eagerly, one str concat and one % operator, move to the same lazy form.
This commit is contained in:
Classic298 authored and GitHub committed 2026-08-02 15:39:10 -05:00
1 parent 52145eede9
commit 2d18727ab8
48 files changed
+297 -247

No files matched your search

+5 -5
View File
@@ -276,9 +276,9 @@ def _resolve_ollama_base_url(url: str) -> str:
if not default.result() and fallback.result(): if not default.result() and fallback.result():
url = url.replace(':11434', ':12434') url = url.replace(':11434', ':12434')
log.info(f'Ollama port 11434 unreachable on {host}, falling back to 12434') log.info('Ollama port 11434 unreachable on %s, falling back to 12434', host)
elif not default.result(): elif not default.result():
log.info(f'Ollama ports 11434 and 12434 both unreachable on {host}') log.info('Ollama ports 11434 and 12434 both unreachable on %s', host)
return url return url
@@ -811,7 +811,7 @@ if VECTOR_DB == 'oracle23ai':
'Oracle23ai requires setting ORACLE_WALLET_DIR and ORACLE_WALLET_PASSWORD when using wallet authentication.' 'Oracle23ai requires setting ORACLE_WALLET_DIR and ORACLE_WALLET_PASSWORD when using wallet authentication.'
) )
log.info(f'VECTOR_DB: {VECTOR_DB}') log.info('VECTOR_DB: %s', VECTOR_DB)
# S3 Vector # S3 Vector
S3_VECTOR_BUCKET_NAME = os.getenv('S3_VECTOR_BUCKET_NAME', None) S3_VECTOR_BUCKET_NAME = os.getenv('S3_VECTOR_BUCKET_NAME', None)
@@ -994,7 +994,7 @@ PDF_EXTRACT_IMAGES = os.getenv('PDF_EXTRACT_IMAGES', 'False').lower() == 'true'
PDF_LOADER_MODE = os.getenv('PDF_LOADER_MODE', 'page') PDF_LOADER_MODE = os.getenv('PDF_LOADER_MODE', 'page')
RAG_EMBEDDING_MODEL = os.getenv('RAG_EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2') RAG_EMBEDDING_MODEL = os.getenv('RAG_EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2')
log.info(f'Embedding model set: {RAG_EMBEDDING_MODEL}') log.info('Embedding model set: %s', RAG_EMBEDDING_MODEL)
RAG_TOKENIZER_MODEL = os.getenv('RAG_TOKENIZER_MODEL', '') RAG_TOKENIZER_MODEL = os.getenv('RAG_TOKENIZER_MODEL', '')
@@ -1022,7 +1022,7 @@ RAG_RERANKING_ENGINE = os.getenv('RAG_RERANKING_ENGINE', '')
RAG_RERANKING_MODEL = os.getenv('RAG_RERANKING_MODEL', '') RAG_RERANKING_MODEL = os.getenv('RAG_RERANKING_MODEL', '')
if RAG_RERANKING_MODEL != '': if RAG_RERANKING_MODEL != '':
log.info(f'Reranking model set: {RAG_RERANKING_MODEL}') log.info('Reranking model set: %s', RAG_RERANKING_MODEL)
RAG_RERANKING_MODEL_AUTO_UPDATE = ( RAG_RERANKING_MODEL_AUTO_UPDATE = (
+1 -1
View File
@@ -227,7 +227,7 @@ if FROM_INIT_PY:
# Check if the data directory exists in the package directory # Check if the data directory exists in the package directory
if DATA_DIR.exists() and DATA_DIR != NEW_DATA_DIR: if DATA_DIR.exists() and DATA_DIR != NEW_DATA_DIR:
log.info(f'Moving {DATA_DIR} to {NEW_DATA_DIR}') log.info('Moving %s to %s', DATA_DIR, NEW_DATA_DIR)
for item in DATA_DIR.iterdir(): for item in DATA_DIR.iterdir():
dest = NEW_DATA_DIR / item.name dest = NEW_DATA_DIR / item.name
if item.is_dir(): if item.is_dir():
+3 -3
View File
@@ -425,13 +425,13 @@ async def lifespan(app: FastAPI):
log.info('Initializing tool servers...') log.info('Initializing tool servers...')
try: try:
await set_tool_servers(mock_request) await set_tool_servers(mock_request)
log.info(f'Initialized {len(app.state.TOOL_SERVERS)} tool server(s)') log.info('Initialized %s tool server(s)', len(app.state.TOOL_SERVERS))
except Exception as e: except Exception as e:
log.warning(f'Failed to initialize tool servers at startup: {e}') log.warning(f'Failed to initialize tool servers at startup: {e}')
try: try:
await set_terminal_servers(mock_request) await set_terminal_servers(mock_request)
log.info(f'Initialized {len(app.state.TERMINAL_SERVERS)} terminal server(s)') log.info('Initialized %s terminal server(s)', len(app.state.TERMINAL_SERVERS))
except Exception as e: except Exception as e:
log.warning(f'Failed to initialize terminal servers at startup: {e}') log.warning(f'Failed to initialize terminal servers at startup: {e}')
@@ -2594,7 +2594,7 @@ async def register_client(request, client_id: str) -> bool:
**apply_connection_oauth_options(connection, oauth_client_info.model_dump(mode='json')) **apply_connection_oauth_options(connection, oauth_client_info.model_dump(mode='json'))
) )
oauth_client_manager.add_client(client_id, oauth_client_info) oauth_client_manager.add_client(client_id, oauth_client_info)
log.info(f'Re-registered OAuth client {client_id} for tool server') log.info('Re-registered OAuth client %s for tool server', client_id)
return True return True
+4 -4
View File
@@ -1935,7 +1935,7 @@ class ChatTable:
result = await session.execute(stmt) result = await session.execute(stmt)
all_chats = result.scalars().all() all_chats = result.scalars().all()
log.info(f'The number of chats: {len(all_chats)}') log.info('The number of chats: %s', len(all_chats))
# Validate and return chats # Validate and return chats
return [ChatModel.model_validate(chat) for chat in all_chats] return [ChatModel.model_validate(chat) for chat in all_chats]
@@ -2100,7 +2100,7 @@ class ChatTable:
bind = await session.connection() bind = await session.connection()
dialect_name = bind.dialect.name dialect_name = bind.dialect.name
log.info(f'DB dialect name: {dialect_name}') log.info('DB dialect name: %s', dialect_name)
if dialect_name == 'sqlite': if dialect_name == 'sqlite':
stmt = stmt.filter( stmt = stmt.filter(
text(f"EXISTS (SELECT 1 FROM json_each(Chat.meta, '$.tags') WHERE json_each.value = :tag_id)") text(f"EXISTS (SELECT 1 FROM json_each(Chat.meta, '$.tags') WHERE json_each.value = :tag_id)")
@@ -2227,7 +2227,7 @@ class ChatTable:
result = await session.execute(stmt.where(Chat.meta['internal'].as_boolean().is_not(True))) result = await session.execute(stmt.where(Chat.meta['internal'].as_boolean().is_not(True)))
count = result.scalar() count = result.scalar()
log.info(f"Count of chats for folder '{folder_id}': {count}") log.info("Count of chats for folder '%s': %s", folder_id, count)
return count return count
async def count_chats_by_folder_ids_and_user_id( async def count_chats_by_folder_ids_and_user_id(
@@ -2241,7 +2241,7 @@ class ChatTable:
result = await session.execute(stmt.where(Chat.meta['internal'].as_boolean().is_not(True))) result = await session.execute(stmt.where(Chat.meta['internal'].as_boolean().is_not(True)))
count = result.scalar() count = result.scalar()
log.info(f"Count of chats for folders '{folder_ids}': {count}") log.info("Count of chats for folders '%s': %s", folder_ids, count)
return count return count
async def delete_tag_by_id_and_user_id_and_tag_name( async def delete_tag_by_id_and_user_id_and_tag_name(
@@ -72,7 +72,7 @@ class DatalabMarkerLoader:
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
response.raise_for_status() response.raise_for_status()
result = response.json() result = response.json()
log.info(f'Marker API status check for request {request_id}: {result}') log.info('Marker API status check for request %s: %s', request_id, result)
return result return result
except requests.HTTPError as e: except requests.HTTPError as e:
log.error(f'Error checking Marker request status: {e}') log.error(f'Error checking Marker request status: {e}')
@@ -104,7 +104,10 @@ class DatalabMarkerLoader:
form_data['additional_config'] = self.additional_config form_data['additional_config'] = self.additional_config
log.info( log.info(
f"Datalab Marker POST request parameters: {{'filename': '{filename}', 'mime_type': '{mime_type}', **{form_data}}}" "Datalab Marker POST request parameters: {'filename': '%s', 'mime_type': '%s', **%s}",
filename,
mime_type,
form_data,
) )
try: try:
@@ -168,7 +171,7 @@ class DatalabMarkerLoader:
'total_cost', 'total_cost',
) )
} }
log.info(f'Marker processing completed successfully: {json.dumps(summary, indent=2)}') log.info('Marker processing completed successfully: %s', json.dumps(summary, indent=2))
break break
if status_val == 'failed' or success_val is False: if status_val == 'failed' or success_val is False:
@@ -235,7 +238,7 @@ class DatalabMarkerLoader:
try: try:
with open(output_path, 'w', encoding='utf-8') as f: with open(output_path, 'w', encoding='utf-8') as f:
f.write(full_text) f.write(full_text)
log.info(f'Saved Marker output to: {output_path}') log.info('Saved Marker output to: %s', output_path)
except Exception as e: except Exception as e:
log.warning(f'Failed to write marker output to disk: {e}') log.warning(f'Failed to write marker output to disk: {e}')
+13 -13
View File
@@ -74,7 +74,7 @@ class MinerULoader:
Load document using Local API (synchronous). Load document using Local API (synchronous).
Posts file to /file_parse endpoint and gets immediate response. Posts file to /file_parse endpoint and gets immediate response.
""" """
log.info(f'Using MinerU Local API at {self.api_url}') log.info('Using MinerU Local API at %s', self.api_url)
filename = os.path.basename(self.file_path) filename = os.path.basename(self.file_path)
@@ -97,7 +97,7 @@ class MinerULoader:
with open(self.file_path, 'rb') as f: with open(self.file_path, 'rb') as f:
files = {'files': (filename, f, 'application/octet-stream')} files = {'files': (filename, f, 'application/octet-stream')}
log.info(f'Sending file to MinerU Local API: {filename}') log.info('Sending file to MinerU Local API: %s', filename)
log.debug('Local API parameters: %s', form_data) log.debug('Local API parameters: %s', form_data)
response = requests.post( response = requests.post(
@@ -163,7 +163,7 @@ class MinerULoader:
detail='MinerU returned empty markdown content', detail='MinerU returned empty markdown content',
) )
log.info(f'Successfully parsed document with MinerU Local API: {filename}') log.info('Successfully parsed document with MinerU Local API: %s', filename)
# Create metadata # Create metadata
metadata = { metadata = {
@@ -180,7 +180,7 @@ class MinerULoader:
Load document using Cloud API (asynchronous). Load document using Cloud API (asynchronous).
Uses batch upload endpoint to avoid need for public file URLs. Uses batch upload endpoint to avoid need for public file URLs.
""" """
log.info(f'Using MinerU Cloud API at {self.api_url}') log.info('Using MinerU Cloud API at %s', self.api_url)
filename = os.path.basename(self.file_path) filename = os.path.basename(self.file_path)
@@ -196,7 +196,7 @@ class MinerULoader:
# Step 4: Download and extract markdown from ZIP # Step 4: Download and extract markdown from ZIP
markdown_content = self._download_and_extract_zip(result['full_zip_url'], filename) markdown_content = self._download_and_extract_zip(result['full_zip_url'], filename)
log.info(f'Successfully parsed document with MinerU Cloud API: {filename}') log.info('Successfully parsed document with MinerU Cloud API: %s', filename)
# Create metadata # Create metadata
metadata = { metadata = {
@@ -232,7 +232,7 @@ class MinerULoader:
if self.page_ranges: if self.page_ranges:
request_body['files'][0]['page_ranges'] = self.page_ranges request_body['files'][0]['page_ranges'] = self.page_ranges
log.info(f'Requesting upload URL for: {filename}') log.info('Requesting upload URL for: %s', filename)
log.debug('Cloud API request body: %s', request_body) log.debug('Cloud API request body: %s', request_body)
try: try:
@@ -284,7 +284,7 @@ class MinerULoader:
) )
upload_url = file_urls[0] upload_url = file_urls[0]
log.info(f'Received upload URL for batch: {batch_id}') log.info('Received upload URL for batch: %s', batch_id)
return batch_id, upload_url return batch_id, upload_url
@@ -334,7 +334,7 @@ class MinerULoader:
max_iterations = 300 # 10 minutes max (2 seconds per iteration) max_iterations = 300 # 10 minutes max (2 seconds per iteration)
poll_interval = 2 # seconds poll_interval = 2 # seconds
log.info(f'Polling batch status: {batch_id}') log.info('Polling batch status: %s', batch_id)
for iteration in range(max_iterations): for iteration in range(max_iterations):
try: try:
@@ -393,7 +393,7 @@ class MinerULoader:
state = file_result.get('state') state = file_result.get('state')
if state == 'done': if state == 'done':
log.info(f'Processing complete for {filename}') log.info('Processing complete for %s', filename)
return file_result return file_result
elif state == 'failed': elif state == 'failed':
error_msg = file_result.get('err_msg', 'Unknown error') error_msg = file_result.get('err_msg', 'Unknown error')
@@ -404,7 +404,7 @@ class MinerULoader:
elif state in ['waiting-file', 'pending', 'running', 'converting']: elif state in ['waiting-file', 'pending', 'running', 'converting']:
# Still processing # Still processing
if iteration % 10 == 0: # Log every 20 seconds if iteration % 10 == 0: # Log every 20 seconds
log.info(f'Processing status: {state} (iteration {iteration + 1}/{max_iterations})') log.info('Processing status: %s (iteration %s/%s)', state, iteration + 1, max_iterations)
time.sleep(poll_interval) time.sleep(poll_interval)
else: else:
log.warning(f'Unknown state: {state}') log.warning(f'Unknown state: {state}')
@@ -421,7 +421,7 @@ class MinerULoader:
Download ZIP file from CDN and extract markdown content. Download ZIP file from CDN and extract markdown content.
Returns the markdown content as a string. Returns the markdown content as a string.
""" """
log.info(f'Downloading results from: {zip_url}') log.info('Downloading results from: %s', zip_url)
try: try:
response = requests.get(zip_url, timeout=60) response = requests.get(zip_url, timeout=60)
@@ -452,7 +452,7 @@ class MinerULoader:
read_errors = [] read_errors = []
for member in md_members: for member in md_members:
log.info(f'Found markdown file in ZIP: {member.filename}') log.info('Found markdown file in ZIP: %s', member.filename)
try: try:
with zip_ref.open(member, 'r') as f: with zip_ref.open(member, 'r') as f:
if self.max_markdown_bytes is None: if self.max_markdown_bytes is None:
@@ -515,5 +515,5 @@ class MinerULoader:
detail='Extracted markdown content is empty', detail='Extracted markdown content is empty',
) )
log.info(f'Successfully extracted markdown content ({len(markdown_content)} characters)') log.info('Successfully extracted markdown content (%s characters)', len(markdown_content))
return markdown_content return markdown_content
+17 -15
View File
@@ -261,7 +261,7 @@ class MistralLoader:
file_id = response_data.get('id') file_id = response_data.get('id')
if not file_id: if not file_id:
raise ValueError('File ID not found in upload response.') raise ValueError('File ID not found in upload response.')
log.info(f'File uploaded successfully. File ID: {file_id}') log.info('File uploaded successfully. File ID: %s', file_id)
return file_id return file_id
except Exception as e: except Exception as e:
log.error(f'Failed to upload file: {e}') log.error(f'Failed to upload file: {e}')
@@ -305,12 +305,12 @@ class MistralLoader:
if not file_id: if not file_id:
raise ValueError('File ID not found in upload response.') raise ValueError('File ID not found in upload response.')
log.info(f'File uploaded successfully. File ID: {file_id}') log.info('File uploaded successfully. File ID: %s', file_id)
return file_id return file_id
def _get_signed_url(self, file_id: str) -> str: def _get_signed_url(self, file_id: str) -> str:
"""Retrieves a temporary signed URL for the uploaded file (sync version).""" """Retrieves a temporary signed URL for the uploaded file (sync version)."""
log.info(f'Getting signed URL for file ID: {file_id}') log.info('Getting signed URL for file ID: %s', file_id)
url = f'{self.base_url}/files/{file_id}/url' url = f'{self.base_url}/files/{file_id}/url'
params = {'expiry': 1} params = {'expiry': 1}
signed_url_headers = {**self.headers, 'Accept': 'application/json'} signed_url_headers = {**self.headers, 'Accept': 'application/json'}
@@ -421,7 +421,7 @@ class MistralLoader:
ocr_response = await self._handle_response_async(response) ocr_response = await self._handle_response_async(response)
processing_time = time.time() - start_time processing_time = time.time() - start_time
log.info(f'OCR processing completed in {processing_time:.2f}s') log.info('OCR processing completed in %.2fs', processing_time)
return ocr_response return ocr_response
@@ -434,13 +434,13 @@ class MistralLoader:
def _delete_file(self, file_id: str) -> None: def _delete_file(self, file_id: str) -> None:
"""Deletes the file from Mistral storage (sync version).""" """Deletes the file from Mistral storage (sync version)."""
log.info(f'Deleting uploaded file ID: {file_id}') log.info('Deleting uploaded file ID: %s', file_id)
url = f'{self.base_url}/files/{file_id}' url = f'{self.base_url}/files/{file_id}'
try: try:
response = requests.delete(url, headers=self.headers, timeout=self.cleanup_timeout) response = requests.delete(url, headers=self.headers, timeout=self.cleanup_timeout)
delete_response = self._handle_response(response) delete_response = self._handle_response(response)
log.info(f'File deleted successfully: {delete_response}') log.info('File deleted successfully: %s', delete_response)
except Exception as e: except Exception as e:
# Log error but don't necessarily halt execution if deletion fails # Log error but don't necessarily halt execution if deletion fails
log.error(f'Failed to delete file ID {file_id}: {e}') log.error(f'Failed to delete file ID {file_id}: {e}')
@@ -551,7 +551,7 @@ class MistralLoader:
) )
if skipped_pages > 0: if skipped_pages > 0:
log.info(f'Processed {len(documents)} pages, skipped {skipped_pages} empty/invalid pages') log.info('Processed %s pages, skipped %s empty/invalid pages', len(documents), skipped_pages)
if not documents: if not documents:
# Case where pages existed but none had valid markdown/index # Case where pages existed but none had valid markdown/index
@@ -584,7 +584,7 @@ class MistralLoader:
if self.use_base64: if self.use_base64:
documents = self._process_results(self._process_ocr(self._get_file_data_url())) documents = self._process_results(self._process_ocr(self._get_file_data_url()))
total_time = time.time() - start_time total_time = time.time() - start_time
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents') log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents return documents
# 1. Upload file # 1. Upload file
@@ -600,7 +600,7 @@ class MistralLoader:
documents = self._process_results(ocr_response) documents = self._process_results(ocr_response)
total_time = time.time() - start_time total_time = time.time() - start_time
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents') log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents return documents
@@ -642,7 +642,7 @@ class MistralLoader:
ocr_response = await self._process_ocr_async(session, self._get_file_data_url()) ocr_response = await self._process_ocr_async(session, self._get_file_data_url())
documents = self._process_results(ocr_response) documents = self._process_results(ocr_response)
total_time = time.time() - start_time total_time = time.time() - start_time
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents') log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents return documents
# 1. Upload file with streaming # 1. Upload file with streaming
@@ -658,7 +658,7 @@ class MistralLoader:
documents = self._process_results(ocr_response) documents = self._process_results(ocr_response)
total_time = time.time() - start_time total_time = time.time() - start_time
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents') log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents return documents
@@ -701,7 +701,7 @@ class MistralLoader:
if not loaders: if not loaders:
return [] return []
log.info(f'Starting concurrent processing of {len(loaders)} files with max {max_concurrent} concurrent') log.info('Starting concurrent processing of %s files with max %s concurrent', len(loaders), max_concurrent)
start_time = time.time() start_time = time.time()
# Use semaphore to control concurrency # Use semaphore to control concurrency
@@ -741,9 +741,11 @@ class MistralLoader:
failure_count = len(results) - success_count failure_count = len(results) - success_count
log.info( log.info(
f'Batch processing completed in {total_time:.2f}s: ' 'Batch processing completed in %.2fs: %s files succeeded, %s files failed, produced %s total documents',
f'{success_count} files succeeded, {failure_count} files failed, ' total_time,
f'produced {total_docs} total documents' success_count,
failure_count,
total_docs,
) )
return processed_results return processed_results
@@ -35,7 +35,7 @@ class PaddleOCRVLLoader:
self.file_name = os.path.basename(file_path) self.file_name = os.path.basename(file_path)
def load(self) -> List[Document]: def load(self) -> List[Document]:
log.info(f'Processing with PaddleOCR-vl: {self.file_path}') log.info('Processing with PaddleOCR-vl: %s', self.file_path)
try: try:
with open(self.file_path, 'rb') as file: with open(self.file_path, 'rb') as file:
@@ -96,7 +96,7 @@ class PaddleOCRVLLoader:
) )
if skipped_pages > 0: if skipped_pages > 0:
log.info(f'PaddleOCR-vl: Processed {len(documents)} pages, skipped {skipped_pages} empty pages.') log.info('PaddleOCR-vl: Processed %s pages, skipped %s empty pages.', len(documents), skipped_pages)
if not documents: if not documents:
log.warning('No valid text content found by PaddleOCR-vl.') log.warning('No valid text content found by PaddleOCR-vl.')
@@ -138,7 +138,7 @@ class YoutubeLoader:
log.debug("No transcript found for language '%s'", lang) log.debug("No transcript found for language '%s'", lang)
continue continue
except Exception as e: except Exception as e:
log.info(f"Error finding transcript for language '{lang}'") log.info("Error finding transcript for language '%s'", lang)
raise e raise e
# If we get here, all languages failed # If we get here, all languages failed
@@ -35,8 +35,8 @@ class ExternalReranker(BaseReranker):
} }
try: try:
log.info(f'ExternalReranker:predict:model {self.model}') log.info('ExternalReranker:predict:model %s', self.model)
log.info(f'ExternalReranker:predict:query {query}') log.info('ExternalReranker:predict:query %s', query)
headers = { headers = {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
+4 -4
View File
@@ -318,7 +318,7 @@ def query_doc(collection_name: str, query_embedding: list[float], k: int, user:
) )
if result: if result:
log.info(f'query_doc:result {result.ids} {result.metadatas}') log.info('query_doc:result %s %s', result.ids, result.metadatas)
return result return result
except Exception as e: except Exception as e:
@@ -332,7 +332,7 @@ def get_doc(collection_name: str, user: UserModel = None):
result = VECTOR_DB_CLIENT.get(collection_name=collection_name) result = VECTOR_DB_CLIENT.get(collection_name=collection_name)
if result: if result:
log.info(f'query_doc:result {result.ids} {result.metadatas}') log.info('query_doc:result %s %s', result.ids, result.metadatas)
return result return result
except Exception as e: except Exception as e:
@@ -585,7 +585,7 @@ async def query_doc_with_hybrid_search(
'metadatas': [metadatas], 'metadatas': [metadatas],
} }
log.info('query_doc_with_hybrid_search:result ' + f'{result["metadatas"]} {result["distances"]}') log.info('query_doc_with_hybrid_search:result %s %s', result['metadatas'], result['distances'])
return result return result
except Exception as e: except Exception as e:
log.exception(f'Error querying doc {collection_name} with hybrid search: {e}') log.exception(f'Error querying doc {collection_name} with hybrid search: {e}')
@@ -812,7 +812,7 @@ async def query_collection_with_hybrid_search(
collection_results = dict(await asyncio.gather(*(_fetch_collection(name) for name in collection_names))) collection_results = dict(await asyncio.gather(*(_fetch_collection(name) for name in collection_names)))
log.info(f'Starting hybrid search for {len(queries)} queries in {len(collection_names)} collections...') log.info('Starting hybrid search for %s queries in %s collections...', len(queries), len(collection_names))
async def process_query(collection_name, query): async def process_query(collection_name, query):
try: try:
@@ -125,7 +125,7 @@ class MilvusClient(VectorDBBase):
index_type = MILVUS_INDEX_TYPE.upper() index_type = MILVUS_INDEX_TYPE.upper()
metric_type = MILVUS_METRIC_TYPE.upper() metric_type = MILVUS_METRIC_TYPE.upper()
log.info(f'Using Milvus index type: {index_type}, metric type: {metric_type}') log.info('Using Milvus index type: %s, metric type: %s', index_type, metric_type)
index_creation_params = {} index_creation_params = {}
if index_type == 'HNSW': if index_type == 'HNSW':
@@ -133,18 +133,18 @@ class MilvusClient(VectorDBBase):
'M': MILVUS_HNSW_M, 'M': MILVUS_HNSW_M,
'efConstruction': MILVUS_HNSW_EFCONSTRUCTION, 'efConstruction': MILVUS_HNSW_EFCONSTRUCTION,
} }
log.info(f'HNSW params: {index_creation_params}') log.info('HNSW params: %s', index_creation_params)
elif index_type == 'IVF_FLAT': elif index_type == 'IVF_FLAT':
index_creation_params = {'nlist': MILVUS_IVF_FLAT_NLIST} index_creation_params = {'nlist': MILVUS_IVF_FLAT_NLIST}
log.info(f'IVF_FLAT params: {index_creation_params}') log.info('IVF_FLAT params: %s', index_creation_params)
elif index_type == 'DISKANN': elif index_type == 'DISKANN':
index_creation_params = { index_creation_params = {
'max_degree': MILVUS_DISKANN_MAX_DEGREE, 'max_degree': MILVUS_DISKANN_MAX_DEGREE,
'search_list_size': MILVUS_DISKANN_SEARCH_LIST_SIZE, 'search_list_size': MILVUS_DISKANN_SEARCH_LIST_SIZE,
} }
log.info(f'DISKANN params: {index_creation_params}') log.info('DISKANN params: %s', index_creation_params)
elif index_type in ['FLAT', 'AUTOINDEX']: elif index_type in ['FLAT', 'AUTOINDEX']:
log.info(f'Using {index_type} index with no specific build-time params.') log.info('Using %s index with no specific build-time params.', index_type)
else: else:
log.warning( log.warning(
f"Unsupported MILVUS_INDEX_TYPE: '{index_type}'. " f"Unsupported MILVUS_INDEX_TYPE: '{index_type}'. "
@@ -167,7 +167,11 @@ class MilvusClient(VectorDBBase):
index_params=index_params, index_params=index_params,
) )
log.info( log.info(
f"Successfully created collection '{self.collection_prefix}_{collection_name}' with index type '{index_type}' and metric '{metric_type}'." "Successfully created collection '%s_%s' with index type '%s' and metric '%s'.",
self.collection_prefix,
collection_name,
index_type,
metric_type,
) )
def has_collection(self, collection_name: str) -> bool: def has_collection(self, collection_name: str) -> bool:
@@ -220,7 +224,11 @@ class MilvusClient(VectorDBBase):
try: try:
log.info( log.info(
f"Querying collection {self.collection_prefix}_{collection_name} with filter: '{filter_string}', limit: {limit}" "Querying collection %s_%s with filter: '%s', limit: %s",
self.collection_prefix,
collection_name,
filter_string,
limit,
) )
iterator = self.client.query_iterator( iterator = self.client.query_iterator(
@@ -265,7 +273,7 @@ class MilvusClient(VectorDBBase):
# Insert the items into the collection, if the collection does not exist, it will be created. # Insert the items into the collection, if the collection does not exist, it will be created.
collection_name = collection_name.replace('-', '_') collection_name = collection_name.replace('-', '_')
if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'): if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'):
log.info(f'Collection {self.collection_prefix}_{collection_name} does not exist. Creating now.') log.info('Collection %s_%s does not exist. Creating now.', self.collection_prefix, collection_name)
if not items: if not items:
log.error( log.error(
f'Cannot create collection {self.collection_prefix}_{collection_name} without items to determine dimension.' f'Cannot create collection {self.collection_prefix}_{collection_name} without items to determine dimension.'
@@ -273,7 +281,7 @@ class MilvusClient(VectorDBBase):
raise ValueError('Cannot create Milvus collection without items to determine vector dimension.') raise ValueError('Cannot create Milvus collection without items to determine vector dimension.')
self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector'])) self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector']))
log.info(f'Inserting {len(items)} items into collection {self.collection_prefix}_{collection_name}.') log.info('Inserting %s items into collection %s_%s.', len(items), self.collection_prefix, collection_name)
data = [] data = []
for item in items: for item in items:
text = item['text'] or '' text = item['text'] or ''
@@ -301,7 +309,9 @@ class MilvusClient(VectorDBBase):
# Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created. # Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created.
collection_name = collection_name.replace('-', '_') collection_name = collection_name.replace('-', '_')
if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'): if not self.client.has_collection(collection_name=f'{self.collection_prefix}_{collection_name}'):
log.info(f'Collection {self.collection_prefix}_{collection_name} does not exist for upsert. Creating now.') log.info(
'Collection %s_%s does not exist for upsert. Creating now.', self.collection_prefix, collection_name
)
if not items: if not items:
log.error( log.error(
f'Cannot create collection {self.collection_prefix}_{collection_name} for upsert without items to determine dimension.' f'Cannot create collection {self.collection_prefix}_{collection_name} for upsert without items to determine dimension.'
@@ -311,7 +321,7 @@ class MilvusClient(VectorDBBase):
) )
self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector'])) self._create_collection(collection_name=collection_name, dimension=len(items[0]['vector']))
log.info(f'Upserting {len(items)} items into collection {self.collection_prefix}_{collection_name}.') log.info('Upserting %s items into collection %s_%s.', len(items), self.collection_prefix, collection_name)
data = [] data = []
for item in items: for item in items:
text = item['text'] or '' text = item['text'] or ''
@@ -348,7 +358,7 @@ class MilvusClient(VectorDBBase):
return None return None
if ids: if ids:
log.info(f'Deleting items by IDs from {self.collection_prefix}_{collection_name}. IDs: {ids}') log.info('Deleting items by IDs from %s_%s. IDs: %s', self.collection_prefix, collection_name, ids)
return self.client.delete( return self.client.delete(
collection_name=f'{self.collection_prefix}_{collection_name}', collection_name=f'{self.collection_prefix}_{collection_name}',
ids=ids, ids=ids,
@@ -356,7 +366,10 @@ class MilvusClient(VectorDBBase):
elif filter: elif filter:
filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()]) filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()])
log.info( log.info(
f'Deleting items by filter from {self.collection_prefix}_{collection_name}. Filter: {filter_string}' 'Deleting items by filter from %s_%s. Filter: %s',
self.collection_prefix,
collection_name,
filter_string,
) )
return self.client.delete( return self.client.delete(
collection_name=f'{self.collection_prefix}_{collection_name}', collection_name=f'{self.collection_prefix}_{collection_name}',
@@ -378,7 +391,7 @@ class MilvusClient(VectorDBBase):
try: try:
self.client.drop_collection(collection_name=collection_name_full) self.client.drop_collection(collection_name=collection_name_full)
deleted_collections.append(collection_name_full) deleted_collections.append(collection_name_full)
log.info(f'Deleted collection: {collection_name_full}') log.info('Deleted collection: %s', collection_name_full)
except Exception as e: except Exception as e:
log.error(f'Error deleting collection {collection_name_full}: {e}') log.error(f'Error deleting collection {collection_name_full}: {e}')
log.info(f'Milvus reset complete. Deleted collections: {deleted_collections}') log.info('Milvus reset complete. Deleted collections: %s', deleted_collections)
@@ -146,7 +146,7 @@ class MilvusClient(VectorDBBase):
# The index only accelerates resource_id filters; never fail # The index only accelerates resource_id filters; never fail
# collection creation over it. # collection creation over it.
log.warning(f'Could not create {RESOURCE_ID_FIELD} index on {mt_collection_name}: {e}') log.warning(f'Could not create {RESOURCE_ID_FIELD} index on {mt_collection_name}: {e}')
log.info(f'Created shared collection: {mt_collection_name}') log.info('Created shared collection: %s', mt_collection_name)
def _ensure_collection(self, mt_collection_name: str, dimension: int): def _ensure_collection(self, mt_collection_name: str, dimension: int):
if not self.client.has_collection(mt_collection_name): if not self.client.has_collection(mt_collection_name):
@@ -181,7 +181,7 @@ class OpenGaussClient(VectorDBBase):
new_items.append(new_chunk) new_items.append(new_chunk)
self.session.bulk_save_objects(new_items) self.session.bulk_save_objects(new_items)
self.session.commit() self.session.commit()
log.info(f"Inserting {len(new_items)} items into collection '{collection_name}'.") log.info("Inserting %s items into collection '%s'.", len(new_items), collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Failed to insert data: {e}') log.exception(f'Failed to insert data: {e}')
@@ -207,7 +207,7 @@ class OpenGaussClient(VectorDBBase):
) )
self.session.add(new_chunk) self.session.add(new_chunk)
self.session.commit() self.session.commit()
log.info(f"Inserting/updating {len(items)} items in collection '{collection_name}'.") log.info("Inserting/updating %s items in collection '%s'.", len(items), collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Failed to insert or update data.: {e}') log.exception(f'Failed to insert or update data.: {e}')
@@ -352,7 +352,7 @@ class OpenGaussClient(VectorDBBase):
query = query.filter(DocumentChunk.vmetadata[key].astext == str(value)) query = query.filter(DocumentChunk.vmetadata[key].astext == str(value))
deleted = query.delete(synchronize_session=False) deleted = query.delete(synchronize_session=False)
self.session.commit() self.session.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'") log.info("Deleted %s items from collection '%s'", deleted, collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Failed to delete data: {e}') log.exception(f'Failed to delete data: {e}')
@@ -362,7 +362,7 @@ class OpenGaussClient(VectorDBBase):
try: try:
deleted = self.session.query(DocumentChunk).delete() deleted = self.session.query(DocumentChunk).delete()
self.session.commit() self.session.commit()
log.info(f'Reset completed. Deleted {deleted} items') log.info('Reset completed. Deleted %s items', deleted)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Reset failed: {e}') log.exception(f'Reset failed: {e}')
@@ -386,4 +386,4 @@ class OpenGaussClient(VectorDBBase):
def delete_collection(self, collection_name: str) -> None: def delete_collection(self, collection_name: str) -> None:
self.delete(collection_name) self.delete(collection_name)
log.info(f"Collection '{collection_name}' has been deleted") log.info("Collection '%s' has been deleted", collection_name)
@@ -94,10 +94,10 @@ class Oracle23aiClient(VectorDBBase):
self._create_dbcs_pool() self._create_dbcs_pool()
dsn = ORACLE_DB_DSN dsn = ORACLE_DB_DSN
log.info(f'Creating Connection Pool [{ORACLE_DB_USER}:**@{dsn}]') log.info('Creating Connection Pool [%s:**@%s]', ORACLE_DB_USER, dsn)
with self.get_connection() as connection: with self.get_connection() as connection:
log.info(f'Connection version: {connection.version}') log.info('Connection version: %s', connection.version)
self._initialize_database(connection) self._initialize_database(connection)
log.info('Oracle Vector Search initialization complete.') log.info('Oracle Vector Search initialization complete.')
@@ -159,7 +159,7 @@ class Oracle23aiClient(VectorDBBase):
if attempt < max_retries - 1: if attempt < max_retries - 1:
wait_time = 2**attempt wait_time = 2**attempt
log.info(f'Retrying in {wait_time} seconds...') log.info('Retrying in %s seconds...', wait_time)
time.sleep(wait_time) time.sleep(wait_time)
else: else:
raise raise
@@ -184,7 +184,7 @@ class Oracle23aiClient(VectorDBBase):
thread = threading.Thread(target=_monitor, daemon=True) thread = threading.Thread(target=_monitor, daemon=True)
thread.start() thread.start()
log.info(f'Started DB health monitor every {interval_seconds} seconds.') log.info('Started DB health monitor every %s seconds.', interval_seconds)
def _reconnect_pool(self): def _reconnect_pool(self):
""" """
@@ -412,7 +412,7 @@ class Oracle23aiClient(VectorDBBase):
... ] ... ]
>>> client.insert("my_collection", items) >>> client.insert("my_collection", items)
""" """
log.info(f"Inserting {len(items)} items into collection '{collection_name}'.") log.info("Inserting %s items into collection '%s'.", len(items), collection_name)
with self.get_connection() as connection: with self.get_connection() as connection:
try: try:
@@ -437,7 +437,7 @@ class Oracle23aiClient(VectorDBBase):
) )
connection.commit() connection.commit()
log.info(f"Successfully inserted {len(items)} items into collection '{collection_name}'.") log.info("Successfully inserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e: except Exception as e:
connection.rollback() connection.rollback()
@@ -466,7 +466,7 @@ class Oracle23aiClient(VectorDBBase):
... ] ... ]
>>> client.upsert("my_collection", items) >>> client.upsert("my_collection", items)
""" """
log.info(f"Upserting {len(items)} items into collection '{collection_name}'.") log.info("Upserting %s items into collection '%s'.", len(items), collection_name)
with self.get_connection() as connection: with self.get_connection() as connection:
try: try:
@@ -505,7 +505,7 @@ class Oracle23aiClient(VectorDBBase):
) )
connection.commit() connection.commit()
log.info(f"Successfully upserted {len(items)} items into collection '{collection_name}'.") log.info("Successfully upserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e: except Exception as e:
connection.rollback() connection.rollback()
@@ -541,7 +541,7 @@ class Oracle23aiClient(VectorDBBase):
... for i, (id, dist) in enumerate(zip(results.ids[0], results.distances[0])): ... for i, (id, dist) in enumerate(zip(results.ids[0], results.distances[0])):
... log.info(f"Match {i+1}: id={id}, distance={dist}") ... log.info(f"Match {i+1}: id={id}, distance={dist}")
""" """
log.info(f"Searching items from collection '{collection_name}' with limit {limit}.") log.info("Searching items from collection '%s' with limit %s.", collection_name, limit)
try: try:
if not vectors: if not vectors:
@@ -587,7 +587,7 @@ class Oracle23aiClient(VectorDBBase):
metadatas[qid].append(self._json_to_metadata(metadata_str)) metadatas[qid].append(self._json_to_metadata(metadata_str))
distances[qid].append(float(row[3])) distances[qid].append(float(row[3]))
log.info(f'Search completed. Found {sum(len(ids[i]) for i in range(num_queries))} total results.') log.info('Search completed. Found %s total results.', sum(len(ids[i]) for i in range(num_queries)))
return SearchResult(ids=ids, distances=distances, documents=documents, metadatas=metadatas) return SearchResult(ids=ids, distances=distances, documents=documents, metadatas=metadatas)
@@ -616,7 +616,7 @@ class Oracle23aiClient(VectorDBBase):
>>> if results: >>> if results:
... print(f"Found {len(results.ids[0])} matching documents") ... print(f"Found {len(results.ids[0])} matching documents")
""" """
log.info(f"Querying items from collection '{collection_name}' with filters.") log.info("Querying items from collection '%s' with filters.", collection_name)
try: try:
limit = limit or 100 limit = limit or 100
@@ -656,7 +656,7 @@ class Oracle23aiClient(VectorDBBase):
] ]
] ]
log.info(f'Query completed. Found {len(results)} results.') log.info('Query completed. Found %s results.', len(results))
return GetResult(ids=ids, documents=documents, metadatas=metadatas) return GetResult(ids=ids, documents=documents, metadatas=metadatas)
@@ -747,7 +747,7 @@ class Oracle23aiClient(VectorDBBase):
>>> # Or delete by metadata filter >>> # Or delete by metadata filter
>>> client.delete("my_collection", filter={"source": "deprecated_source"}) >>> client.delete("my_collection", filter={"source": "deprecated_source"})
""" """
log.info(f"Deleting items from collection '{collection_name}'.") log.info("Deleting items from collection '%s'.", collection_name)
try: try:
query = 'DELETE FROM document_chunk WHERE collection_name = :collection_name' query = 'DELETE FROM document_chunk WHERE collection_name = :collection_name'
@@ -772,7 +772,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount deleted = cursor.rowcount
connection.commit() connection.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'.") log.info("Deleted %s items from collection '%s'.", deleted, collection_name)
except Exception as e: except Exception as e:
log.exception(f'Error during delete: {e}') log.exception(f'Error during delete: {e}')
@@ -800,7 +800,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount deleted = cursor.rowcount
connection.commit() connection.commit()
log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.") log.info("Reset complete. Deleted %s items from 'document_chunk' table.", deleted)
except Exception as e: except Exception as e:
log.exception(f'Error during reset: {e}') log.exception(f'Error during reset: {e}')
@@ -875,7 +875,7 @@ class Oracle23aiClient(VectorDBBase):
>>> client = Oracle23aiClient() >>> client = Oracle23aiClient()
>>> client.delete_collection("obsolete_collection") >>> client.delete_collection("obsolete_collection")
""" """
log.info(f"Deleting collection '{collection_name}'.") log.info("Deleting collection '%s'.", collection_name)
try: try:
with self.get_connection() as connection: with self.get_connection() as connection:
@@ -891,7 +891,7 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount deleted = cursor.rowcount
connection.commit() connection.commit()
log.info(f"Collection '{collection_name}' deleted. Removed {deleted} items.") log.info("Collection '%s' deleted. Removed %s items.", collection_name, deleted)
except Exception as e: except Exception as e:
log.exception(f"Error deleting collection '{collection_name}': {e}") log.exception(f"Error deleting collection '{collection_name}': {e}")
@@ -326,7 +326,7 @@ class PgvectorClient(VectorDBBase):
}, },
) )
self.session.commit() self.session.commit()
log.info(f"Encrypted & inserted {len(items)} into '{collection_name}'") log.info("Encrypted & inserted %s into '%s'", len(items), collection_name)
else: else:
new_items = [] new_items = []
@@ -342,7 +342,7 @@ class PgvectorClient(VectorDBBase):
new_items.append(new_chunk) new_items.append(new_chunk)
self.session.bulk_save_objects(new_items) self.session.bulk_save_objects(new_items)
self.session.commit() self.session.commit()
log.info(f"Inserted {len(new_items)} items into collection '{collection_name}'.") log.info("Inserted %s items into collection '%s'.", len(new_items), collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Error during insert: {e}') log.exception(f'Error during insert: {e}')
@@ -381,7 +381,7 @@ class PgvectorClient(VectorDBBase):
}, },
) )
self.session.commit() self.session.commit()
log.info(f"Encrypted & upserted {len(items)} into '{collection_name}'") log.info("Encrypted & upserted %s into '%s'", len(items), collection_name)
else: else:
for item in items: for item in items:
vector = self.adjust_vector_length(item['vector']) vector = self.adjust_vector_length(item['vector'])
@@ -401,7 +401,7 @@ class PgvectorClient(VectorDBBase):
) )
self.session.add(new_chunk) self.session.add(new_chunk)
self.session.commit() self.session.commit()
log.info(f"Upserted {len(items)} items into collection '{collection_name}'.") log.info("Upserted %s items into collection '%s'.", len(items), collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Error during upsert: {e}') log.exception(f'Error during upsert: {e}')
@@ -712,7 +712,7 @@ class PgvectorClient(VectorDBBase):
query = query.filter(DocumentChunk.vmetadata[key].astext == str(value)) query = query.filter(DocumentChunk.vmetadata[key].astext == str(value))
deleted = query.delete(synchronize_session=False) deleted = query.delete(synchronize_session=False)
self.session.commit() self.session.commit()
log.info(f"Deleted {deleted} items from collection '{collection_name}'.") log.info("Deleted %s items from collection '%s'.", deleted, collection_name)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Error during delete: {e}') log.exception(f'Error during delete: {e}')
@@ -722,7 +722,7 @@ class PgvectorClient(VectorDBBase):
try: try:
deleted = self.session.query(DocumentChunk).delete() deleted = self.session.query(DocumentChunk).delete()
self.session.commit() self.session.commit()
log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.") log.info("Reset complete. Deleted %s items from 'document_chunk' table.", deleted)
except Exception as e: except Exception as e:
self.session.rollback() self.session.rollback()
log.exception(f'Error during reset: {e}') log.exception(f'Error during reset: {e}')
@@ -746,4 +746,4 @@ class PgvectorClient(VectorDBBase):
def delete_collection(self, collection_name: str) -> None: def delete_collection(self, collection_name: str) -> None:
self.delete(collection_name) self.delete(collection_name)
log.info(f"Collection '{collection_name}' deleted.") log.info("Collection '%s' deleted.", collection_name)
@@ -106,16 +106,16 @@ class PineconeClient(VectorDBBase):
try: try:
# Check if index exists # Check if index exists
if self.index_name not in self.client.list_indexes().names(): if self.index_name not in self.client.list_indexes().names():
log.info(f"Creating Pinecone index '{self.index_name}'...") log.info("Creating Pinecone index '%s'...", self.index_name)
self.client.create_index( self.client.create_index(
name=self.index_name, name=self.index_name,
dimension=self.dimension, dimension=self.dimension,
metric=self.metric, metric=self.metric,
spec=ServerlessSpec(cloud=self.cloud, region=self.environment), spec=ServerlessSpec(cloud=self.cloud, region=self.environment),
) )
log.info(f"Successfully created Pinecone index '{self.index_name}'") log.info("Successfully created Pinecone index '%s'", self.index_name)
else: else:
log.info(f"Using existing Pinecone index '{self.index_name}'") log.info("Using existing Pinecone index '%s'", self.index_name)
# Connect to the index # Connect to the index
self.index = self.client.Index( self.index = self.client.Index(
@@ -245,7 +245,7 @@ class PineconeClient(VectorDBBase):
collection_name_with_prefix = self._get_collection_name_with_prefix(collection_name) collection_name_with_prefix = self._get_collection_name_with_prefix(collection_name)
try: try:
self.index.delete(filter={'collection_name': collection_name_with_prefix}) self.index.delete(filter={'collection_name': collection_name_with_prefix})
log.info(f"Collection '{collection_name_with_prefix}' deleted (all vectors removed).") log.info("Collection '%s' deleted (all vectors removed).", collection_name_with_prefix)
except Exception as e: except Exception as e:
log.warning(f"Failed to delete collection '{collection_name_with_prefix}': {e}") log.warning(f"Failed to delete collection '{collection_name_with_prefix}': {e}")
raise raise
@@ -276,7 +276,7 @@ class PineconeClient(VectorDBBase):
elapsed = time.time() - start_time elapsed = time.time() - start_time
log.debug('Insert of %s vectors took %.2f seconds', len(points), elapsed) log.debug('Insert of %s vectors took %.2f seconds', len(points), elapsed)
log.info( log.info(
f"Successfully inserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'" "Successfully inserted %s vectors in parallel batches into '%s'", len(points), collection_name_with_prefix
) )
def upsert(self, collection_name: str, items: List[VectorItem]) -> None: def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
@@ -305,7 +305,7 @@ class PineconeClient(VectorDBBase):
elapsed = time.time() - start_time elapsed = time.time() - start_time
log.debug('Upsert of %s vectors took %.2f seconds', len(points), elapsed) log.debug('Upsert of %s vectors took %.2f seconds', len(points), elapsed)
log.info( log.info(
f"Successfully upserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'" "Successfully upserted %s vectors in parallel batches into '%s'", len(points), collection_name_with_prefix
) )
async def insert_async(self, collection_name: str, items: List[VectorItem]) -> None: async def insert_async(self, collection_name: str, items: List[VectorItem]) -> None:
@@ -326,7 +326,9 @@ class PineconeClient(VectorDBBase):
if isinstance(result, Exception): if isinstance(result, Exception):
log.error(f'Error in async insert batch: {result}') log.error(f'Error in async insert batch: {result}')
raise result raise result
log.info(f"Successfully async inserted {len(points)} vectors in batches into '{collection_name_with_prefix}'") log.info(
"Successfully async inserted %s vectors in batches into '%s'", len(points), collection_name_with_prefix
)
async def upsert_async(self, collection_name: str, items: List[VectorItem]) -> None: async def upsert_async(self, collection_name: str, items: List[VectorItem]) -> None:
"""Async version of upsert using asyncio and run_in_executor for improved performance.""" """Async version of upsert using asyncio and run_in_executor for improved performance."""
@@ -346,7 +348,9 @@ class PineconeClient(VectorDBBase):
if isinstance(result, Exception): if isinstance(result, Exception):
log.error(f'Error in async upsert batch: {result}') log.error(f'Error in async upsert batch: {result}')
raise result raise result
log.info(f"Successfully async upserted {len(points)} vectors in batches into '{collection_name_with_prefix}'") log.info(
"Successfully async upserted %s vectors in batches into '%s'", len(points), collection_name_with_prefix
)
def search( def search(
self, self,
@@ -477,7 +481,7 @@ class PineconeClient(VectorDBBase):
log.debug( log.debug(
"Deleted batch of %s vectors by ID from '%s'", len(batch_ids), collection_name_with_prefix "Deleted batch of %s vectors by ID from '%s'", len(batch_ids), collection_name_with_prefix
) )
log.info(f"Successfully deleted {len(ids)} vectors by ID from '{collection_name_with_prefix}'") log.info("Successfully deleted %s vectors by ID from '%s'", len(ids), collection_name_with_prefix)
elif filter: elif filter:
# Combine user filter with collection_name # Combine user filter with collection_name
@@ -486,7 +490,7 @@ class PineconeClient(VectorDBBase):
pinecone_filter.update(filter) pinecone_filter.update(filter)
# Delete by metadata filter # Delete by metadata filter
self.index.delete(filter=pinecone_filter) self.index.delete(filter=pinecone_filter)
log.info(f"Successfully deleted vectors by filter from '{collection_name_with_prefix}'") log.info("Successfully deleted vectors by filter from '%s'", collection_name_with_prefix)
else: else:
log.warning('No ids or filter provided for delete operation') log.warning('No ids or filter provided for delete operation')
@@ -119,7 +119,7 @@ class QdrantClient(VectorDBBase):
on_disk=self.QDRANT_ON_DISK, on_disk=self.QDRANT_ON_DISK,
), ),
) )
log.info(f'collection {collection_name_with_prefix} successfully created!') log.info('collection %s successfully created!', collection_name_with_prefix)
def _create_collection_if_not_exists(self, collection_name, dimension): def _create_collection_if_not_exists(self, collection_name, dimension):
if not self.has_collection(collection_name=collection_name): if not self.has_collection(collection_name=collection_name):
@@ -148,7 +148,7 @@ class QdrantClient(VectorDBBase):
m=0, m=0,
), ),
) )
log.info(f'Multi-tenant collection {mt_collection_name} created with dimension {dimension}!') log.info('Multi-tenant collection %s created with dimension %s!', mt_collection_name, dimension)
self.client.create_payload_index( self.client.create_payload_index(
collection_name=mt_collection_name, collection_name=mt_collection_name,
@@ -36,7 +36,7 @@ class S3VectorClient(VectorDBBase):
if self.bucket_name and self.region: if self.bucket_name and self.region:
try: try:
self.client = boto3.client('s3vectors', region_name=self.region) self.client = boto3.client('s3vectors', region_name=self.region)
log.info(f"S3Vector client initialized for bucket '{self.bucket_name}' in region '{self.region}'") log.info("S3Vector client initialized for bucket '%s' in region '%s'", self.bucket_name, self.region)
except Exception as e: except Exception as e:
log.error(f'Failed to initialize S3Vector client: {e}') log.error(f'Failed to initialize S3Vector client: {e}')
self.client = None self.client = None
@@ -70,7 +70,9 @@ class S3VectorClient(VectorDBBase):
] ]
}, },
) )
log.info(f'Created S3 index: {index_name} (dim={dimension}, type={data_type}, metric={distance_metric})') log.info(
'Created S3 index: %s (dim=%s, type=%s, metric=%s)', index_name, dimension, data_type, distance_metric
)
except Exception as e: except Exception as e:
log.error(f"Error creating S3 index '{index_name}': {e}") log.error(f"Error creating S3 index '{index_name}': {e}")
raise raise
@@ -137,9 +139,9 @@ class S3VectorClient(VectorDBBase):
return return
try: try:
log.info(f"Deleting collection '{collection_name}'") log.info("Deleting collection '%s'", collection_name)
self.client.delete_index(vectorBucketName=self.bucket_name, indexName=collection_name) self.client.delete_index(vectorBucketName=self.bucket_name, indexName=collection_name)
log.info(f"Successfully deleted collection '{collection_name}'") log.info("Successfully deleted collection '%s'", collection_name)
except Exception as e: except Exception as e:
log.error(f"Error deleting collection '{collection_name}': {e}") log.error(f"Error deleting collection '{collection_name}': {e}")
raise raise
@@ -156,7 +158,7 @@ class S3VectorClient(VectorDBBase):
try: try:
if not self.has_collection(collection_name): if not self.has_collection(collection_name):
log.info(f"Index '{collection_name}' does not exist. Creating index.") log.info("Index '%s' does not exist. Creating index.", collection_name)
self._create_index( self._create_index(
index_name=collection_name, index_name=collection_name,
dimension=dimension, dimension=dimension,
@@ -202,9 +204,11 @@ class S3VectorClient(VectorDBBase):
indexName=collection_name, indexName=collection_name,
vectors=batch, vectors=batch,
) )
log.info(f"Inserted batch {i // batch_size + 1}: {len(batch)} vectors into index '{collection_name}'.") log.info(
"Inserted batch %s: %s vectors into index '%s'.", i // batch_size + 1, len(batch), collection_name
)
log.info(f"Completed insertion of {len(vectors)} vectors into index '{collection_name}'.") log.info("Completed insertion of %s vectors into index '%s'.", len(vectors), collection_name)
except Exception as e: except Exception as e:
log.error(f'Error inserting vectors: {e}') log.error(f'Error inserting vectors: {e}')
raise raise
@@ -218,11 +222,11 @@ class S3VectorClient(VectorDBBase):
return return
dimension = len(items[0]['vector']) dimension = len(items[0]['vector'])
log.info(f'Upsert dimension: {dimension}') log.info('Upsert dimension: %s', dimension)
try: try:
if not self.has_collection(collection_name): if not self.has_collection(collection_name):
log.info(f"Index '{collection_name}' does not exist. Creating index for upsert.") log.info("Index '%s' does not exist. Creating index for upsert.", collection_name)
self._create_index( self._create_index(
index_name=collection_name, index_name=collection_name,
dimension=dimension, dimension=dimension,
@@ -264,10 +268,14 @@ class S3VectorClient(VectorDBBase):
batch = vectors[i : i + batch_size] batch = vectors[i : i + batch_size]
if i == 0: # Log sample info for first batch only if i == 0: # Log sample info for first batch only
log.info( log.info(
f'Upserting batch 1: {len(batch)} vectors. First vector sample: key={batch[0]["key"]}, data_type={type(batch[0]["data"]["float32"])}, data_len={len(batch[0]["data"]["float32"])}' 'Upserting batch 1: %s vectors. First vector sample: key=%s, data_type=%s, data_len=%s',
len(batch),
batch[0]['key'],
type(batch[0]['data']['float32']),
len(batch[0]['data']['float32']),
) )
else: else:
log.info(f'Upserting batch {i // batch_size + 1}: {len(batch)} vectors.') log.info('Upserting batch %s: %s vectors.', i // batch_size + 1, len(batch))
self.client.put_vectors( self.client.put_vectors(
vectorBucketName=self.bucket_name, vectorBucketName=self.bucket_name,
@@ -275,7 +283,7 @@ class S3VectorClient(VectorDBBase):
vectors=batch, vectors=batch,
) )
log.info(f"Completed upsert of {len(vectors)} vectors into index '{collection_name}'.") log.info("Completed upsert of %s vectors into index '%s'.", len(vectors), collection_name)
except Exception as e: except Exception as e:
log.error(f'Error upserting vectors: {e}') log.error(f'Error upserting vectors: {e}')
raise raise
@@ -300,7 +308,7 @@ class S3VectorClient(VectorDBBase):
return None return None
try: try:
log.info(f"Searching collection '{collection_name}' with {len(vectors)} query vectors, limit={limit}") log.info("Searching collection '%s' with %s query vectors, limit=%s", collection_name, len(vectors), limit)
# Initialize result lists # Initialize result lists
all_ids = [] all_ids = []
@@ -362,7 +370,7 @@ class S3VectorClient(VectorDBBase):
all_metadatas.append(query_metadatas) all_metadatas.append(query_metadatas)
all_distances.append(query_distances) all_distances.append(query_distances)
log.info(f'Search completed. Found results for {len(all_ids)} queries') log.info('Search completed. Found results for %s queries', len(all_ids))
# Return SearchResult format # Return SearchResult format
return SearchResult( return SearchResult(
@@ -402,7 +410,7 @@ class S3VectorClient(VectorDBBase):
return self.get(collection_name) return self.get(collection_name)
try: try:
log.info(f"Querying collection '{collection_name}' with filter: {filter}") log.info("Querying collection '%s' with filter: %s", collection_name, filter)
# For S3 Vector, we need to use list_vectors and then filter results # For S3 Vector, we need to use list_vectors and then filter results
# Since S3 Vector may not support complex server-side filtering, # Since S3 Vector may not support complex server-side filtering,
@@ -437,7 +445,7 @@ class S3VectorClient(VectorDBBase):
if limit and len(filtered_ids) >= limit: if limit and len(filtered_ids) >= limit:
break break
log.info(f'Filter applied: {len(filtered_ids)} vectors match out of {len(all_ids)} total') log.info('Filter applied: %s vectors match out of %s total', len(filtered_ids), len(all_ids))
# Return GetResult format # Return GetResult format
if filtered_ids: if filtered_ids:
@@ -472,7 +480,7 @@ class S3VectorClient(VectorDBBase):
return GetResult(ids=[[]], documents=[[]], metadatas=[[]]) return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
try: try:
log.info(f"Retrieving all vectors from collection '{collection_name}'") log.info("Retrieving all vectors from collection '%s'", collection_name)
# Initialize result lists # Initialize result lists
all_ids = [] all_ids = []
@@ -534,7 +542,7 @@ class S3VectorClient(VectorDBBase):
if not next_token: if not next_token:
break break
log.info(f"Retrieved {len(all_ids)} vectors from collection '{collection_name}'") log.info("Retrieved %s vectors from collection '%s'", len(all_ids), collection_name)
# Return in GetResult format # Return in GetResult format
# The Open WebUI GetResult expects lists of lists, so we wrap each list # The Open WebUI GetResult expects lists of lists, so we wrap each list
@@ -576,17 +584,17 @@ class S3VectorClient(VectorDBBase):
try: try:
if ids: if ids:
# Delete by specific vector IDs/keys # Delete by specific vector IDs/keys
log.info(f"Deleting {len(ids)} vectors by IDs from collection '{collection_name}'") log.info("Deleting %s vectors by IDs from collection '%s'", len(ids), collection_name)
self.client.delete_vectors( self.client.delete_vectors(
vectorBucketName=self.bucket_name, vectorBucketName=self.bucket_name,
indexName=collection_name, indexName=collection_name,
keys=ids, keys=ids,
) )
log.info(f"Deleted {len(ids)} vectors from index '{collection_name}'") log.info("Deleted %s vectors from index '%s'", len(ids), collection_name)
elif filter: elif filter:
# Handle filter-based deletion # Handle filter-based deletion
log.info(f"Deleting vectors by filter from collection '{collection_name}': {filter}") log.info("Deleting vectors by filter from collection '%s': %s", collection_name, filter)
# If this is a knowledge collection and we have a file_id filter, # If this is a knowledge collection and we have a file_id filter,
# also clean up the corresponding file-specific collection # also clean up the corresponding file-specific collection
@@ -595,7 +603,8 @@ class S3VectorClient(VectorDBBase):
file_collection_name = f'file-{file_id}' file_collection_name = f'file-{file_id}'
if self.has_collection(file_collection_name): if self.has_collection(file_collection_name):
log.info( log.info(
f"Found related file-specific collection '{file_collection_name}', deleting it to prevent duplicates" "Found related file-specific collection '%s', deleting it to prevent duplicates",
file_collection_name,
) )
self.delete_collection(file_collection_name) self.delete_collection(file_collection_name)
@@ -604,7 +613,7 @@ class S3VectorClient(VectorDBBase):
query_result = self.query(collection_name, filter) query_result = self.query(collection_name, filter)
if query_result and query_result.ids and query_result.ids[0]: if query_result and query_result.ids and query_result.ids[0]:
matching_ids = query_result.ids[0] matching_ids = query_result.ids[0]
log.info(f'Found {len(matching_ids)} vectors matching filter, deleting them') log.info('Found %s vectors matching filter, deleting them', len(matching_ids))
# Delete the matching vectors by ID # Delete the matching vectors by ID
self.client.delete_vectors( self.client.delete_vectors(
@@ -612,7 +621,7 @@ class S3VectorClient(VectorDBBase):
indexName=collection_name, indexName=collection_name,
keys=matching_ids, keys=matching_ids,
) )
log.info(f"Deleted {len(matching_ids)} vectors from index '{collection_name}' using filter") log.info("Deleted %s vectors from index '%s' using filter", len(matching_ids), collection_name)
else: else:
log.warning('No vectors found matching the filter criteria') log.warning('No vectors found matching the filter criteria')
else: else:
@@ -645,11 +654,11 @@ class S3VectorClient(VectorDBBase):
try: try:
self.client.delete_index(vectorBucketName=self.bucket_name, indexName=index_name) self.client.delete_index(vectorBucketName=self.bucket_name, indexName=index_name)
deleted_count += 1 deleted_count += 1
log.info(f'Deleted index: {index_name}') log.info('Deleted index: %s', index_name)
except Exception as e: except Exception as e:
log.error(f"Error deleting index '{index_name}': {e}") log.error(f"Error deleting index '{index_name}': {e}")
log.info(f'Reset completed: deleted {deleted_count} indexes') log.info('Reset completed: deleted %s indexes', deleted_count)
except Exception as e: except Exception as e:
log.error(f'Error during reset: {e}') log.error(f'Error during reset: {e}')
@@ -279,7 +279,7 @@ class ValkeyClient(VectorDBBase):
f'{self._format_version(MIN_VALKEY_VERSION)}. valkey-search 1.2.0 requires Valkey core ' f'{self._format_version(MIN_VALKEY_VERSION)}. valkey-search 1.2.0 requires Valkey core '
'9.0.1 or later. Upgrade your server or use valkey-bundle:9.1.0-rc2+.' '9.0.1 or later. Upgrade your server or use valkey-bundle:9.1.0-rc2+.'
) )
log.info(f'Valkey core version: {self._format_version(version) if version else "unknown"}') log.info('Valkey core version: %s', self._format_version(version) if version else 'unknown')
def _check_search_module(self) -> None: def _check_search_module(self) -> None:
try: try:
@@ -331,7 +331,7 @@ class ValkeyClient(VectorDBBase):
'TEXT field type and filter-only FT.SEARCH support required by this backend. ' 'TEXT field type and filter-only FT.SEARCH support required by this backend. '
'Upgrade to valkey-bundle:9.1.0-rc2+ or load valkey-search 1.2.0+ as a module.' 'Upgrade to valkey-bundle:9.1.0-rc2+ or load valkey-search 1.2.0+ as a module.'
) )
log.info(f'valkey-search version: {self._format_version(search_version) if search_version else "unknown"}') log.info('valkey-search version: %s', self._format_version(search_version) if search_version else 'unknown')
def _index_name(self, collection_name: str) -> str: def _index_name(self, collection_name: str) -> str:
return f'idx:{self.collection_prefix}:{collection_name}' return f'idx:{self.collection_prefix}:{collection_name}'
@@ -385,8 +385,11 @@ class ValkeyClient(VectorDBBase):
try: try:
g['glide_ft'].create(self.client, index_name, schema, options) g['glide_ft'].create(self.client, index_name, schema, options)
log.info( log.info(
f'Created Valkey index {index_name} with dimension={dimension}, ' 'Created Valkey index %s with dimension=%s, type=%s, metric=%s',
f'type={self.index_type}, metric={self.distance_metric}' index_name,
dimension,
self.index_type,
self.distance_metric,
) )
except g['RequestError'] as e: except g['RequestError'] as e:
if 'already exists' in str(e).lower(): if 'already exists' in str(e).lower():
@@ -456,7 +459,7 @@ class ValkeyClient(VectorDBBase):
index_name = self._index_name(collection_name) index_name = self._index_name(collection_name)
try: try:
self._g['glide_ft'].dropindex(self.client, index_name) self._g['glide_ft'].dropindex(self.client, index_name)
log.info(f'Dropped index {index_name}') log.info('Dropped index %s', index_name)
except self._g['RequestError'] as e: except self._g['RequestError'] as e:
log.debug('Could not drop index %s: %s', index_name, e) log.debug('Could not drop index %s: %s', index_name, e)
@@ -656,7 +659,7 @@ class ValkeyClient(VectorDBBase):
collections.append(name[len(idx_prefix) :]) collections.append(name[len(idx_prefix) :])
try: try:
glide_ft.dropindex(self.client, idx) glide_ft.dropindex(self.client, idx)
log.info(f'Dropped index: {name}') log.info('Dropped index: %s', name)
except Exception as e: except Exception as e:
log.error(f'Error dropping index {name}: {e}') log.error(f'Error dropping index {name}: {e}')
except Exception as e: except Exception as e:
@@ -664,7 +667,7 @@ class ValkeyClient(VectorDBBase):
for collection in collections: for collection in collections:
self._delete_keys_by_prefix(self._key_prefix(collection)) self._delete_keys_by_prefix(self._key_prefix(collection))
log.info(f'Valkey vector store reset complete (prefix: {self.collection_prefix})') log.info('Valkey vector store reset complete (prefix: %s)', self.collection_prefix)
def _delete_keys_by_prefix(self, prefix: str) -> None: def _delete_keys_by_prefix(self, prefix: str) -> None:
cursor = '0' cursor = '0'
+2 -2
View File
@@ -31,7 +31,7 @@ def search_exa(
count (int): Number of results to return count (int): Number of results to return
filter_list (Optional[list[str]]): List of domains to filter results by filter_list (Optional[list[str]]): List of domains to filter results by
""" """
log.info(f'Searching with Exa for query: {query}') log.info('Searching with Exa for query: %s', query)
headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'} headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
@@ -58,7 +58,7 @@ def search_exa(
) )
) )
log.info(f'Found {len(results)} results') log.info('Found %s results', len(results))
return [ return [
SearchResult( SearchResult(
link=result.url, link=result.url,
+1 -1
View File
@@ -53,7 +53,7 @@ def search_external(
) )
for result in results[:count] for result in results[:count]
] ]
log.info(f'External search results: {results}') log.info('External search results: %s', results)
return results return results
except Exception as e: except Exception as e:
log.error(f'Error in External search: {e}') log.error(f'Error in External search: {e}')
@@ -225,7 +225,7 @@ def search_firecrawl(
) )
) )
log.info(f'FireCrawl search results: {search_results}') log.info('FireCrawl search results: %s', search_results)
return search_results return search_results
except Exception as e: except Exception as e:
log.error(f'Error in FireCrawl search: {e}') log.error(f'Error in FireCrawl search: {e}')
+2 -2
View File
@@ -23,7 +23,7 @@ def search_ollama_cloud(
count (int): Number of results to return count (int): Number of results to return
filter_list (Optional[list[str]]): List of domains to filter results by filter_list (Optional[list[str]]): List of domains to filter results by
""" """
log.info(f'Searching with Ollama for query: {query}') log.info('Searching with Ollama for query: %s', query)
headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'} headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
payload = {'query': query, 'max_results': count} payload = {'query': query, 'max_results': count}
@@ -34,7 +34,7 @@ def search_ollama_cloud(
data = response.json() data = response.json()
results = data.get('results', []) results = data.get('results', [])
log.info(f'Found {len(results)} results') log.info('Found %s results', len(results))
if filter_list: if filter_list:
results = get_filtered_results(results, filter_list) results = get_filtered_results(results, filter_list)
@@ -31,7 +31,7 @@ def search_searchapi(
response = requests.request('GET', url) response = requests.request('GET', url)
json_response = response.json() json_response = response.json()
log.info(f'results from searchapi search: {json_response}') log.info('results from searchapi search: %s', json_response)
results = sorted(json_response.get('organic_results', []), key=lambda x: x.get('position', 0)) results = sorted(json_response.get('organic_results', []), key=lambda x: x.get('position', 0))
if filter_list: if filter_list:
+1 -1
View File
@@ -31,7 +31,7 @@ def search_serpapi(
response = requests.request('GET', url) response = requests.request('GET', url)
json_response = response.json() json_response = response.json()
log.info(f'results from serpapi search: {json_response}') log.info('results from serpapi search: %s', json_response)
results = sorted(json_response.get('organic_results', []), key=lambda x: x.get('position', 0)) results = sorted(json_response.get('organic_results', []), key=lambda x: x.get('position', 0))
if filter_list: if filter_list:
+1 -1
View File
@@ -51,7 +51,7 @@ def search_serply(
response.raise_for_status() response.raise_for_status()
json_response = response.json() json_response = response.json()
log.info(f'results from serply search: {json_response}') log.info('results from serply search: %s', json_response)
results = sorted(json_response.get('results', []), key=lambda x: x.get('realPosition', 0)) results = sorted(json_response.get('results', []), key=lambda x: x.get('realPosition', 0))
if filter_list: if filter_list:
+1 -1
View File
@@ -112,7 +112,7 @@ def search_yandex(
for result in results[:count] for result in results[:count]
] ]
log.info(f'Yandex search results: {results}') log.info('Yandex search results: %s', results)
return results return results
except Exception as e: except Exception as e:
+8 -6
View File
@@ -157,7 +157,7 @@ def convert_audio_to_mp3(file_path):
output_path = os.path.splitext(file_path)[0] + '.mp3' output_path = os.path.splitext(file_path)[0] + '.mp3'
audio = AudioSegment.from_file(file_path) audio = AudioSegment.from_file(file_path)
audio.export(output_path, format='mp3') audio.export(output_path, format='mp3')
log.info(f'Converted {file_path} to {output_path}') log.info('Converted %s to %s', file_path, output_path)
return output_path return output_path
except Exception as e: except Exception as e:
log.error(f'Error converting audio file: {e}') log.error(f'Error converting audio file: {e}')
@@ -208,7 +208,7 @@ def transcode_audio_to_mp3(audio_data: bytes, content_type_header: str, output_p
audio_segment = AudioSegment.from_file(io.BytesIO(audio_data)) audio_segment = AudioSegment.from_file(io.BytesIO(audio_data))
audio_segment.export(str(output_path), format='mp3') audio_segment.export(str(output_path), format='mp3')
log.info(f'Transcoded {mime_type} audio to MP3: {output_path}') log.info('Transcoded %s audio to MP3: %s', mime_type, output_path)
return True return True
@@ -631,7 +631,7 @@ async def _transcribe_whisper(request, file_path, languages, file_dir, id):
language=languages[0], language=languages[0],
multilingual=WHISPER_MULTILINGUAL, multilingual=WHISPER_MULTILINGUAL,
) )
log.info("Detected language '%s' with probability %f" % (info.language, info.language_probability)) log.info("Detected language '%s' with probability %f", info.language, info.language_probability)
return ''.join([segment.text for segment in list(segments)]) return ''.join([segment.text for segment in list(segments)])
transcript = await asyncio.to_thread(_run) transcript = await asyncio.to_thread(_run)
@@ -952,7 +952,9 @@ async def _transcribe_mistral(request, file_path, filename, metadata, file_dir,
try: try:
model = await Config.get('audio.stt.model') or 'voxtral-mini-latest' model = await Config.get('audio.stt.model') or 'voxtral-mini-latest'
log.info( log.info(
f'Mistral STT - model: {model}, method: {"chat_completions" if use_chat_completions else "transcriptions"}' 'Mistral STT - model: %s, method: %s',
model,
'chat_completions' if use_chat_completions else 'transcriptions',
) )
session = await get_session() session = await get_session()
@@ -1075,7 +1077,7 @@ async def _transcribe_mistral(request, file_path, filename, metadata, file_dir,
async def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None, user=None): async def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None, user=None):
log.info(f'transcribe: {file_path} {metadata}') log.info('transcribe: %s %s', file_path, metadata)
if BYPASS_PYDUB_PREPROCESSING: if BYPASS_PYDUB_PREPROCESSING:
log.info('Bypassing pydub preprocessing (BYPASS_PYDUB_PREPROCESSING=true)') log.info('Bypassing pydub preprocessing (BYPASS_PYDUB_PREPROCESSING=true)')
@@ -1202,7 +1204,7 @@ async def transcription(
status_code=status.HTTP_403_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN,
detail=ERROR_MESSAGES.ACCESS_PROHIBITED, detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
) )
log.info(f'file.content_type: {file.content_type}') log.info('file.content_type: %s', file.content_type)
stt_supported_content_types = await Config.get('audio.stt.supported_content_types', []) stt_supported_content_types = await Config.get('audio.stt.supported_content_types', [])
if not strict_match_mime_type(stt_supported_content_types, file.content_type): if not strict_match_mime_type(stt_supported_content_types, file.content_type):
+14 -14
View File
@@ -548,8 +548,8 @@ async def ldap_auth(
] ]
if ENABLE_LDAP_GROUP_MANAGEMENT: if ENABLE_LDAP_GROUP_MANAGEMENT:
search_attributes.append(f'{LDAP_ATTRIBUTE_FOR_GROUPS}') search_attributes.append(f'{LDAP_ATTRIBUTE_FOR_GROUPS}')
log.info(f'LDAP Group Management enabled. Adding {LDAP_ATTRIBUTE_FOR_GROUPS} to search attributes') log.info('LDAP Group Management enabled. Adding %s to search attributes', LDAP_ATTRIBUTE_FOR_GROUPS)
log.info(f'LDAP search attributes: {search_attributes}') log.info('LDAP search attributes: %s', search_attributes)
search_success = await asyncio.to_thread( search_success = await asyncio.to_thread(
connection_app.search, connection_app.search,
@@ -586,30 +586,30 @@ async def ldap_auth(
user_groups = [] user_groups = []
if ENABLE_LDAP_GROUP_MANAGEMENT and LDAP_ATTRIBUTE_FOR_GROUPS in entry: if ENABLE_LDAP_GROUP_MANAGEMENT and LDAP_ATTRIBUTE_FOR_GROUPS in entry:
group_dns = entry[LDAP_ATTRIBUTE_FOR_GROUPS] group_dns = entry[LDAP_ATTRIBUTE_FOR_GROUPS]
log.info(f'LDAP raw group DNs for user {username_list}: {group_dns}') log.info('LDAP raw group DNs for user %s: %s', username_list, group_dns)
if group_dns: if group_dns:
log.info(f'LDAP group_dns original: {group_dns}') log.info('LDAP group_dns original: %s', group_dns)
log.info(f'LDAP group_dns type: {type(group_dns)}') log.info('LDAP group_dns type: %s', type(group_dns))
log.info(f'LDAP group_dns length: {len(group_dns)}') log.info('LDAP group_dns length: %s', len(group_dns))
if hasattr(group_dns, 'value'): if hasattr(group_dns, 'value'):
group_dns = group_dns.value group_dns = group_dns.value
log.info(f'Extracted .value property: {group_dns}') log.info('Extracted .value property: %s', group_dns)
elif hasattr(group_dns, '__iter__') and not isinstance(group_dns, (str, bytes)): elif hasattr(group_dns, '__iter__') and not isinstance(group_dns, (str, bytes)):
group_dns = list(group_dns) group_dns = list(group_dns)
log.info(f'Converted to list: {group_dns}') log.info('Converted to list: %s', group_dns)
if isinstance(group_dns, list): if isinstance(group_dns, list):
group_dns = [str(item) for item in group_dns] group_dns = [str(item) for item in group_dns]
else: else:
group_dns = [str(group_dns)] group_dns = [str(group_dns)]
log.info(f'LDAP group_dns after processing - type: {type(group_dns)}, length: {len(group_dns)}') log.info('LDAP group_dns after processing - type: %s, length: %s', type(group_dns), len(group_dns))
for group_idx, group_dn in enumerate(group_dns): for group_idx, group_dn in enumerate(group_dns):
group_dn = str(group_dn) group_dn = str(group_dn)
log.info(f'Processing group DN #{group_idx + 1}: {group_dn}') log.info('Processing group DN #%s: %s', group_idx + 1, group_dn)
try: try:
group_cn = extract_group_cn_from_dn(group_dn) group_cn = extract_group_cn_from_dn(group_dn)
@@ -621,9 +621,9 @@ async def ldap_auth(
except Exception as e: except Exception as e:
log.warning(f'Failed to extract group name from DN {group_dn}: {e}') log.warning(f'Failed to extract group name from DN {group_dn}: {e}')
log.info(f'LDAP groups for user {username_list}: {user_groups} (total: {len(user_groups)})') log.info('LDAP groups for user %s: %s (total: %s)', username_list, user_groups, len(user_groups))
else: else:
log.info(f'No groups found for user {username_list}') log.info('No groups found for user %s', username_list)
elif ENABLE_LDAP_GROUP_MANAGEMENT: elif ENABLE_LDAP_GROUP_MANAGEMENT:
log.warning( log.warning(
f'LDAP Group Management enabled but {LDAP_ATTRIBUTE_FOR_GROUPS} attribute not found in user entry' f'LDAP Group Management enabled but {LDAP_ATTRIBUTE_FOR_GROUPS} attribute not found in user entry'
@@ -691,7 +691,7 @@ async def ldap_auth(
if ENABLE_LDAP_GROUP_CREATION: if ENABLE_LDAP_GROUP_CREATION:
await Groups.create_groups_by_group_names(user.id, user_groups, db=db) await Groups.create_groups_by_group_names(user.id, user_groups, db=db)
await Groups.sync_groups_by_group_names(user.id, user_groups, db=db) await Groups.sync_groups_by_group_names(user.id, user_groups, db=db)
log.info(f'Successfully synced groups for user {user.id}: {user_groups}') log.info('Successfully synced groups for user %s: %s', user.id, user_groups)
except Exception as e: except Exception as e:
log.error(f'Failed to sync groups for user {user.id}: {e}') log.error(f'Failed to sync groups for user {user.id}: {e}')
@@ -1156,7 +1156,7 @@ async def get_admin_details(
admin_email = await Config.get('auth.admin.email') admin_email = await Config.get('auth.admin.email')
admin_name = None admin_name = None
log.info(f'Admin details - Email: {admin_email}, Name: {admin_name}') log.info('Admin details - Email: %s, Name: %s', admin_email, admin_name)
if admin_email: if admin_email:
admin = await Users.get_user_by_email(admin_email, db=db) admin = await Users.get_user_by_email(admin_email, db=db)
+1 -1
View File
@@ -575,7 +575,7 @@ async def verify_tool_servers_config(request: Request, form_data: ToolServerConn
'oauth_server_metadata': oauth_server_metadata.model_dump(mode='json'), 'oauth_server_metadata': oauth_server_metadata.model_dump(mode='json'),
} }
except Exception as e: except Exception as e:
log.info(f'Failed to parse OAuth 2.1 discovery document: {e}') log.info('Failed to parse OAuth 2.1 discovery document: %s', e)
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail=f'Failed to parse OAuth 2.1 discovery document from {discovery_url}', detail=f'Failed to parse OAuth 2.1 discovery document from {discovery_url}',
+5 -5
View File
@@ -177,7 +177,7 @@ async def process_uploaded_file(
# processing (Tools, vision models). Attempting text # processing (Tools, vision models). Attempting text
# extraction causes "Timeout reached while detecting # extraction causes "Timeout reached while detecting
# encoding" errors. # encoding" errors.
log.info(f'Video file detected ({content_type}), skipping text extraction') log.info('Video file detected (%s), skipping text extraction', content_type)
await Files.update_file_data_by_id( await Files.update_file_data_by_id(
file_item.id, file_item.id,
{'status': 'completed'}, {'status': 'completed'},
@@ -190,7 +190,7 @@ async def process_uploaded_file(
# Documents, or media files explicitly enabled for the # Documents, or media files explicitly enabled for the
# configured content extraction engine. # configured content extraction engine.
if not content_type: if not content_type:
log.info(f'File type {file.content_type} is not provided, but trying to process anyway') log.info('File type %s is not provided, but trying to process anyway', file.content_type)
await process_file( await process_file(
request, request,
ProcessFileForm(file_id=file_item.id), ProcessFileForm(file_id=file_item.id),
@@ -242,7 +242,7 @@ async def process_uploaded_file(
) )
if not knowledge_file: if not knowledge_file:
raise Exception(f'Failed to link file {file_item.id} to knowledge {knowledge_id}') raise Exception(f'Failed to link file {file_item.id} to knowledge {knowledge_id}')
log.info(f'Linked file {file_item.id} to knowledge {knowledge_id}') log.info('Linked file %s to knowledge %s', file_item.id, knowledge_id)
except Exception as e: except Exception as e:
log.warning(f'Failed to link file {file_item.id} to knowledge {knowledge_id}: {e}') log.warning(f'Failed to link file {file_item.id} to knowledge {knowledge_id}: {e}')
raise raise
@@ -322,7 +322,7 @@ async def upload_file_handler(
background_tasks: Optional[BackgroundTasks] = None, background_tasks: Optional[BackgroundTasks] = None,
db: Optional[AsyncSession] = None, db: Optional[AsyncSession] = None,
): ):
log.info(f'file.content_type: {file.content_type} {process}') log.info('file.content_type: %s %s', file.content_type, process)
if isinstance(metadata, str): if isinstance(metadata, str):
try: try:
@@ -868,7 +868,7 @@ async def get_html_file_content_by_id(
# Check if the file already exists in the cache # Check if the file already exists in the cache
if file_path.is_file(): if file_path.is_file():
log.info(f'file_path: {file_path}') log.info('file_path: %s', file_path)
return FileResponse(file_path) return FileResponse(file_path)
else: else:
raise HTTPException( raise HTTPException(
+1 -1
View File
@@ -170,7 +170,7 @@ def get_image_file_item(base64_string, param_name='image'):
async def set_image_model(request: Request, model: str): async def set_image_model(request: Request, model: str):
log.info(f'Setting image model to {model}') log.info('Setting image model to %s', model)
await Config.upsert({'image_generation.model': model}) await Config.upsert({'image_generation.model': model})
image_config = await get_image_config() image_config = await get_image_config()
if image_config.IMAGE_GENERATION_ENGINE in ['', 'automatic1111']: if image_config.IMAGE_GENERATION_ENGINE in ['', 'automatic1111']:
+15 -10
View File
@@ -351,7 +351,7 @@ async def reindex_knowledge_files(
failed_files = [] failed_files = []
start_time = time.monotonic() start_time = time.monotonic()
log.info(f'Starting reindexing for {len(knowledge_bases)} knowledge bases ({total_files} files)') log.info('Starting reindexing for %s knowledge bases (%s files)', len(knowledge_bases), total_files)
for kb_idx, (knowledge_base, files) in enumerate(knowledge_base_files, start=1): for kb_idx, (knowledge_base, files) in enumerate(knowledge_base_files, start=1):
try: try:
@@ -371,8 +371,13 @@ async def reindex_knowledge_files(
eta = f', ETA: {round(elapsed / (processed_files - 1) * remaining_files)}s' eta = f', ETA: {round(elapsed / (processed_files - 1) * remaining_files)}s'
log.info( log.info(
f'Reindexing knowledge base {kb_idx}/{len(knowledge_bases)} ' 'Reindexing knowledge base %s/%s file %s/%s%s: %s',
f'file {processed_files}/{total_files}{eta}: {file.filename}' kb_idx,
len(knowledge_bases),
processed_files,
total_files,
eta,
file.filename,
) )
try: try:
@@ -397,7 +402,7 @@ async def reindex_knowledge_files(
for failed in failed_files: for failed in failed_files:
log.warning(f'File ID: {failed["file_id"]}, Error: {failed["error"]}') log.warning(f'File ID: {failed["file_id"]}, Error: {failed["error"]}')
log.info(f'Reindexing completed in {round(time.monotonic() - start_time)}s.') log.info('Reindexing completed in %ss.', round(time.monotonic() - start_time))
await publish_event( await publish_event(
request, request,
EVENTS.KNOWLEDGE_REINDEXED, EVENTS.KNOWLEDGE_REINDEXED,
@@ -426,14 +431,14 @@ async def reindex_knowledge_base_metadata_embeddings(
this entire operation would exhaust the connection pool. this entire operation would exhaust the connection pool.
""" """
knowledge_bases = await Knowledges.get_knowledge_bases() knowledge_bases = await Knowledges.get_knowledge_bases()
log.info(f'Reindexing embeddings for {len(knowledge_bases)} knowledge bases') log.info('Reindexing embeddings for %s knowledge bases', len(knowledge_bases))
success_count = 0 success_count = 0
for kb in knowledge_bases: for kb in knowledge_bases:
if await embed_knowledge_base_metadata(request, kb.id, kb.name, kb.description): if await embed_knowledge_base_metadata(request, kb.id, kb.name, kb.description):
success_count += 1 success_count += 1
log.info(f'Embedding reindex complete: {success_count}/{len(knowledge_bases)}') log.info('Embedding reindex complete: %s/%s', success_count, len(knowledge_bases))
return {'total': len(knowledge_bases), 'success': success_count} return {'total': len(knowledge_bases), 'success': success_count}
@@ -1687,11 +1692,11 @@ async def delete_knowledge_by_id(
detail=ERROR_MESSAGES.ACCESS_PROHIBITED, detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
) )
log.info(f'Deleting knowledge base: {id} (name: {knowledge.name})') log.info('Deleting knowledge base: %s (name: %s)', id, knowledge.name)
# Get all models # Get all models
models = await Models.get_all_models(db=db) models = await Models.get_all_models(db=db)
log.info(f'Found {len(models)} models to check for knowledge base {id}') log.info('Found %s models to check for knowledge base %s', len(models), id)
# Update models that reference this knowledge base # Update models that reference this knowledge base
for model in models: for model in models:
@@ -1702,7 +1707,7 @@ async def delete_knowledge_by_id(
# If the knowledge list changed, update the model # If the knowledge list changed, update the model
if len(updated_knowledge) != len(knowledge_list): if len(updated_knowledge) != len(knowledge_list):
log.info(f'Updating model {model.id} to remove knowledge base {id}') log.info('Updating model %s to remove knowledge base %s', model.id, id)
model.meta.knowledge = updated_knowledge model.meta.knowledge = updated_knowledge
model_form = ModelForm(**model.model_dump()) model_form = ModelForm(**model.model_dump())
await Models.update_model_by_id(model.id, model_form, db=db) await Models.update_model_by_id(model.id, model_form, db=db)
@@ -2022,7 +2027,7 @@ async def add_files_to_knowledge_batch(
) )
# Batch-fetch all files to avoid N+1 queries # Batch-fetch all files to avoid N+1 queries
log.info(f'files/batch/add - {len(form_data)} files') log.info('files/batch/add - %s files', len(form_data))
file_ids = [form.file_id for form in form_data] file_ids = [form.file_id for form in form_data]
files = await Files.get_files_by_ids(file_ids, db=db) files = await Files.get_files_by_ids(file_ids, db=db)
+7 -7
View File
@@ -650,7 +650,7 @@ async def pull_model(
form_data['model'] = form_data.get('model', form_data.get('name')) form_data['model'] = form_data.get('model', form_data.get('name'))
url = (await Config.get('ollama.base_urls', []))[url_idx] url = (await Config.get('ollama.base_urls', []))[url_idx]
log.info(f'url: {url}') log.info('url: %s', url)
# Admins may pull from any registry # Admins may pull from any registry
return await send_request( return await send_request(
@@ -881,7 +881,7 @@ async def embed(
if not await Config.get('ollama.enable'): if not await Config.get('ollama.enable'):
raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED) raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
log.info(f'generate_ollama_batch_embeddings {form_data}') log.info('generate_ollama_batch_embeddings %s', form_data)
await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL) await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL)
await validate_ollama_backend_idx(request, form_data.model, url_idx, user) await validate_ollama_backend_idx(request, form_data.model, url_idx, user)
@@ -932,7 +932,7 @@ async def embeddings(
if not await Config.get('ollama.enable'): if not await Config.get('ollama.enable'):
raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED) raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
log.info(f'generate_ollama_embeddings {form_data}') log.info('generate_ollama_embeddings %s', form_data)
await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL) await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL)
await validate_ollama_backend_idx(request, form_data.model, url_idx, user) await validate_ollama_backend_idx(request, form_data.model, url_idx, user)
@@ -1637,11 +1637,11 @@ async def upload_model(
async def file_process_stream(): async def file_process_stream():
nonlocal ollama_url nonlocal ollama_url
total_size = os.path.getsize(file_path) total_size = os.path.getsize(file_path)
log.info(f'Total Model Size: {total_size}') log.info('Total Model Size: %s', total_size)
# Stage 2: hash the file and emit SSE progress # Stage 2: hash the file and emit SSE progress
file_hash = await asyncio.to_thread(calculate_sha256, file_path, chunk_size) file_hash = await asyncio.to_thread(calculate_sha256, file_path, chunk_size)
log.info(f'Model Hash: {file_hash}') log.info('Model Hash: %s', file_hash)
try: try:
bytes_read = 0 bytes_read = 0
@@ -1675,13 +1675,13 @@ async def upload_model(
# Stage 4: create the model # Stage 4: create the model
model, _ext = os.path.splitext(filename) model, _ext = os.path.splitext(filename)
log.info(f'Created Model: {model}') log.info('Created Model: %s', model)
create_payload = { create_payload = {
'model': model, 'model': model,
'files': {filename: f'sha256:{file_hash}'}, 'files': {filename: f'sha256:{file_hash}'},
} }
log.info(f'Model Payload: {create_payload}') log.info('Model Payload: %s', create_payload)
async with session.post( async with session.post(
f'{ollama_url}/api/create', f'{ollama_url}/api/create',
+1 -1
View File
@@ -221,7 +221,7 @@ async def upload_pipeline(
file: UploadFile = File(...), file: UploadFile = File(...),
user=Depends(get_admin_user), user=Depends(get_admin_user),
): ):
log.info(f'upload_pipeline: urlIdx={urlIdx}, filename={file.filename}') log.info('upload_pipeline: urlIdx=%s, filename=%s', urlIdx, file.filename)
filename = os.path.basename(file.filename) filename = os.path.basename(file.filename)
# Check if the uploaded file is a python file # Check if the uploaded file is a python file
+12 -12
View File
@@ -519,7 +519,7 @@ async def unload_embedding_model(request: Request):
@router.post('/embedding/update') @router.post('/embedding/update')
async def update_embedding_config(request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)): async def update_embedding_config(request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)):
config = await get_retrieval_config() config = await get_retrieval_config()
log.info(f'Updating embedding model: {config.RAG_EMBEDDING_MODEL} to {form_data.RAG_EMBEDDING_MODEL}') log.info('Updating embedding model: %s to %s', config.RAG_EMBEDDING_MODEL, form_data.RAG_EMBEDDING_MODEL)
await unload_embedding_model(request) await unload_embedding_model(request)
try: try:
config.RAG_EMBEDDING_ENGINE = form_data.RAG_EMBEDDING_ENGINE config.RAG_EMBEDDING_ENGINE = form_data.RAG_EMBEDDING_ENGINE
@@ -1159,7 +1159,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
else config.RAG_RERANKING_BATCH_SIZE else config.RAG_RERANKING_BATCH_SIZE
) )
log.info(f'Updating reranking model: {config.RAG_RERANKING_MODEL} to {form_data.RAG_RERANKING_MODEL}') log.info('Updating reranking model: %s to %s', config.RAG_RERANKING_MODEL, form_data.RAG_RERANKING_MODEL)
try: try:
config.RAG_RERANKING_MODEL = ( config.RAG_RERANKING_MODEL = (
form_data.RAG_RERANKING_MODEL if form_data.RAG_RERANKING_MODEL is not None else config.RAG_RERANKING_MODEL form_data.RAG_RERANKING_MODEL if form_data.RAG_RERANKING_MODEL is not None else config.RAG_RERANKING_MODEL
@@ -1663,7 +1663,7 @@ def save_docs_to_vector_db(
existing_file_id = result.metadatas[0][0].get('file_id') existing_file_id = result.metadatas[0][0].get('file_id')
if existing_file_id != metadata.get('file_id'): if existing_file_id != metadata.get('file_id'):
log.info(f'Document with hash {metadata["hash"]} already exists') log.info('Document with hash %s already exists', metadata['hash'])
raise ValueError(ERROR_MESSAGES.DUPLICATE_CONTENT) raise ValueError(ERROR_MESSAGES.DUPLICATE_CONTENT)
if split: if split:
@@ -1706,7 +1706,7 @@ def save_docs_to_vector_db(
) )
docs = text_splitter.split_documents(docs) docs = text_splitter.split_documents(docs)
elif config.TEXT_SPLITTER == 'token': elif config.TEXT_SPLITTER == 'token':
log.info(f'Using token text splitter: {config.TIKTOKEN_ENCODING_NAME}') log.info('Using token text splitter: %s', config.TIKTOKEN_ENCODING_NAME)
tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME)) tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
text_splitter = TokenTextSplitter( text_splitter = TokenTextSplitter(
@@ -1748,16 +1748,16 @@ def save_docs_to_vector_db(
try: try:
if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name): if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name):
log.info(f'collection {collection_name} already exists') log.info('collection %s already exists', collection_name)
if overwrite: if overwrite:
VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name)
log.info(f'deleting existing collection {collection_name}') log.info('deleting existing collection %s', collection_name)
elif add is False: elif add is False:
log.info(f'collection {collection_name} already exists, overwrite is False and add is False') log.info('collection %s already exists, overwrite is False and add is False', collection_name)
return True return True
log.info(f'generating embeddings for {collection_name}') log.info('generating embeddings for %s', collection_name)
embedding_function = get_embedding_function( embedding_function = get_embedding_function(
config.RAG_EMBEDDING_ENGINE, config.RAG_EMBEDDING_ENGINE,
config.RAG_EMBEDDING_MODEL, config.RAG_EMBEDDING_MODEL,
@@ -1801,7 +1801,7 @@ def save_docs_to_vector_db(
request.app.state.main_loop, request.app.state.main_loop,
) )
embeddings = future.result(timeout=embedding_timeout) embeddings = future.result(timeout=embedding_timeout)
log.info(f'embeddings generated {len(embeddings)} for {len(texts)} items') log.info('embeddings generated %s for %s items', len(embeddings), len(texts))
items = [ items = [
{ {
@@ -1813,13 +1813,13 @@ def save_docs_to_vector_db(
for idx, text in enumerate(texts) for idx, text in enumerate(texts)
] ]
log.info(f'adding to collection {collection_name}') log.info('adding to collection %s', collection_name)
VECTOR_DB_CLIENT.insert( VECTOR_DB_CLIENT.insert(
collection_name=collection_name, collection_name=collection_name,
items=items, items=items,
) )
log.info(f'added {len(items)} items to collection {collection_name}') log.info('added %s items to collection %s', len(items), collection_name)
return True return True
except Exception as e: except Exception as e:
log.exception(e) log.exception(e)
@@ -2011,7 +2011,7 @@ async def process_file(
add=(True if form_data.collection_name else False), add=(True if form_data.collection_name else False),
user=user, user=user,
) )
log.info(f'added {len(docs)} items to collection {collection_name}') log.info('added %s items to collection %s', len(docs), collection_name)
if result: if result:
# Fresh session for the final update. # Fresh session for the final update.
+1 -1
View File
@@ -258,7 +258,7 @@ def get_scim_auth(request: Request, authorization: Optional[str] = Header(None))
# Check if SCIM is enabled # Check if SCIM is enabled
enable_scim = getattr(request.app.state, 'ENABLE_SCIM', False) enable_scim = getattr(request.app.state, 'ENABLE_SCIM', False)
log.info(f'SCIM auth check - raw ENABLE_SCIM: {enable_scim}, type: {type(enable_scim)}') log.info('SCIM auth check - raw ENABLE_SCIM: %s, type: %s', enable_scim, type(enable_scim))
if not enable_scim: if not enable_scim:
raise HTTPException( raise HTTPException(
+1 -1
View File
@@ -412,7 +412,7 @@ async def generate_queries(request: Request, form_data: dict, user=Depends(get_v
) )
if getattr(request.state, 'cached_queries', None): if getattr(request.state, 'cached_queries', None):
log.info(f'Reusing cached queries: {request.state.cached_queries}') log.info('Reusing cached queries: %s', request.state.cached_queries)
return request.state.cached_queries return request.state.cached_queries
if getattr(request.state, 'direct', False) and hasattr(request.state, 'model'): if getattr(request.state, 'direct', False) and hasattr(request.state, 'model'):
+5 -5
View File
@@ -331,7 +331,7 @@ async def disconnect_user_sessions(user_id: str):
for sid in session_ids: for sid in session_ids:
await sio.disconnect(sid) await sio.disconnect(sid)
if session_ids: if session_ids:
log.info(f'Disconnected {len(session_ids)} session(s) for user {user_id}') log.info('Disconnected %s session(s) for user %s', len(session_ids), user_id)
except Exception as e: except Exception as e:
log.warning(f'Failed to disconnect sessions for user {user_id}: {e}') log.warning(f'Failed to disconnect sessions for user {user_id}: {e}')
@@ -626,7 +626,7 @@ async def ydoc_document_join(sid, data):
user_name = data.get('user_name', 'Anonymous') user_name = data.get('user_name', 'Anonymous')
user_color = data.get('user_color', '#000000') user_color = data.get('user_color', '#000000')
log.info(f'User {user_id} joining document {document_id}') log.info('User %s joining document %s', user_id, document_id)
await YDOC_MANAGER.add_user(document_id=document_id, user_id=sid) await YDOC_MANAGER.add_user(document_id=document_id, user_id=sid)
# Join Socket.IO room # Join Socket.IO room
@@ -665,7 +665,7 @@ async def ydoc_document_join(sid, data):
skip_sid=sid, skip_sid=sid,
) )
log.info(f'User {user_id} successfully joined document {document_id}') log.info('User %s successfully joined document %s', user_id, document_id)
except Exception as e: except Exception as e:
log.error(f'Error in yjs_document_join: {e}') log.error(f'Error in yjs_document_join: {e}')
@@ -826,7 +826,7 @@ async def yjs_document_leave(sid, data):
try: try:
document_id = normalize_document_id(data['document_id']) document_id = normalize_document_id(data['document_id'])
log.info(f'User {user["id"]} leaving document {document_id}') log.info('User %s leaving document %s', user['id'], document_id)
# Remove user from the document # Remove user from the document
await YDOC_MANAGER.remove_user(document_id=document_id, user_id=sid) await YDOC_MANAGER.remove_user(document_id=document_id, user_id=sid)
@@ -842,7 +842,7 @@ async def yjs_document_leave(sid, data):
) )
if await YDOC_MANAGER.document_exists(document_id) and len(await YDOC_MANAGER.get_users(document_id)) == 0: if await YDOC_MANAGER.document_exists(document_id) and len(await YDOC_MANAGER.get_users(document_id)) == 0:
log.info(f'Cleaning up document {document_id} as no users are left') log.info('Cleaning up document %s as no users are left', document_id)
await YDOC_MANAGER.clear_document(document_id) await YDOC_MANAGER.clear_document(document_id)
except Exception as e: except Exception as e:
+2 -2
View File
@@ -542,7 +542,7 @@ async def create_admin_user(email: str, password: str, name: str = 'Admin'):
log.debug('Users already exist, skipping admin creation') log.debug('Users already exist, skipping admin creation')
return None return None
log.info(f'Creating admin account from environment variables: {email}') log.info('Creating admin account from environment variables: %s', email)
try: try:
hashed = await get_password_hash(password) hashed = await get_password_hash(password)
user = await Auths.insert_new_auth( user = await Auths.insert_new_auth(
@@ -552,7 +552,7 @@ async def create_admin_user(email: str, password: str, name: str = 'Admin'):
role='admin', role='admin',
) )
if user: if user:
log.info(f'Admin account created successfully: {email}') log.info('Admin account created successfully: %s', email)
return user return user
else: else:
log.error('Failed to create admin account from environment variables') log.error('Failed to create admin account from environment variables')
+4 -3
View File
@@ -211,8 +211,9 @@ async def scheduler_worker_loop(app) -> None:
SCHEDULER_POLL_INTERVAL env var (default: 10 seconds). SCHEDULER_POLL_INTERVAL env var (default: 10 seconds).
""" """
log.info( log.info(
f'Scheduler worker started (timer poll interval: {TIMER_POLL_INTERVAL}s, ' 'Scheduler worker started (timer poll interval: %ss, scheduler poll interval: %ss)',
f'scheduler poll interval: {SCHEDULER_POLL_INTERVAL}s)' TIMER_POLL_INTERVAL,
SCHEDULER_POLL_INTERVAL,
) )
next_scheduler_poll = 0.0 next_scheduler_poll = 0.0
@@ -240,7 +241,7 @@ async def scheduler_worker_loop(app) -> None:
async with get_async_db() as db: async with get_async_db() as db:
batch = await Automations.claim_due(int(time.time_ns()), limit=10, db=db) batch = await Automations.claim_due(int(time.time_ns()), limit=10, db=db)
if batch: if batch:
log.info(f'Claimed {len(batch)} due automation(s)') log.info('Claimed %s due automation(s)', len(batch))
for automation in batch: for automation in batch:
asyncio.create_task(execute_automation(app, automation)) asyncio.create_task(execute_automation(app, automation))
except Exception: except Exception:
+2 -2
View File
@@ -68,7 +68,7 @@ async def generate_direct_chat_completion(
) )
channel = f'{user_id}:{session_id}:{request_id}' channel = f'{user_id}:{session_id}:{request_id}'
logging.info(f'WebSocket channel: {channel}') logging.info('WebSocket channel: %s', channel)
if form_data.get('stream'): if form_data.get('stream'):
q = asyncio.Queue() q = asyncio.Queue()
@@ -95,7 +95,7 @@ async def generate_direct_chat_completion(
} }
) )
log.info(f'res: {res}') log.info('res: %s', res)
if res.get('status', False): if res.get('status', False):
# Define a generator to stream responses # Define a generator to stream responses
+2 -2
View File
@@ -1193,7 +1193,7 @@ def stream_chunks_handler(stream: aiohttp.StreamReader):
if len(line) > max_buffer_size: if len(line) > max_buffer_size:
skip_mode = True skip_mode = True
yield b'data: {}\n' yield b'data: {}\n'
log.info(f'Skip mode triggered, line size: {len(line)}') log.info('Skip mode triggered, line size: %s', len(line))
else: else:
yield line + b'\n' yield line + b'\n'
@@ -1203,7 +1203,7 @@ def stream_chunks_handler(stream: aiohttp.StreamReader):
# Check if buffer exceeds limit # Check if buffer exceeds limit
if not skip_mode and len(buffer) > max_buffer_size: if not skip_mode and len(buffer) > max_buffer_size:
skip_mode = True skip_mode = True
log.info(f'Skip mode triggered, buffer size: {len(buffer)}') log.info('Skip mode triggered, buffer size: %s', len(buffer))
# Clear oversized buffer to prevent unlimited growth # Clear oversized buffer to prevent unlimited growth
buffer = b'' buffer = b''
+4 -4
View File
@@ -377,13 +377,13 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
if items is None: if items is None:
action_function = functions_by_id.get(action_id) action_function = functions_by_id.get(action_id)
if action_function is None: if action_function is None:
log.info(f'Action not found: {action_id}') log.info('Action not found: %s', action_id)
action_items_by_id[action_id] = [] action_items_by_id[action_id] = []
continue continue
function_module = functions_cache.get(action_id) function_module = functions_cache.get(action_id)
if function_module is None: if function_module is None:
log.info(f'Failed to load action module: {action_id}') log.info('Failed to load action module: %s', action_id)
action_items_by_id[action_id] = [] action_items_by_id[action_id] = []
continue continue
items = get_action_items_from_module(action_function, function_module) items = get_action_items_from_module(action_function, function_module)
@@ -397,13 +397,13 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
if items is None: if items is None:
filter_function = functions_by_id.get(filter_id) filter_function = functions_by_id.get(filter_id)
if filter_function is None: if filter_function is None:
log.info(f'Filter not found: {filter_id}') log.info('Filter not found: %s', filter_id)
filter_items_by_id[filter_id] = [] filter_items_by_id[filter_id] = []
continue continue
function_module = functions_cache.get(filter_id) function_module = functions_cache.get(filter_id)
if function_module is None: if function_module is None:
log.info(f'Failed to load filter module: {filter_id}') log.info('Failed to load filter module: %s', filter_id)
filter_items_by_id[filter_id] = [] filter_items_by_id[filter_id] = []
continue continue
if getattr(function_module, 'toggle', None): if getattr(function_module, 'toggle', None):
+21 -13
View File
@@ -608,7 +608,9 @@ async def get_oauth_client_info_with_dynamic_client_registration(
} }
) )
log.info( log.info(
f'Dynamic client registration successful at {registration_url}, client_id: {oauth_client_info.client_id}' 'Dynamic client registration successful at %s, client_id: %s',
registration_url,
oauth_client_info.client_id,
) )
return oauth_client_info return oauth_client_info
except Exception as e: except Exception as e:
@@ -701,7 +703,7 @@ async def get_oauth_client_info_with_static_credentials(
) )
log.info( log.info(
f'Static OAuth client info built for {oauth_client_id} using metadata from {oauth_server_metadata_url}' 'Static OAuth client info built for %s using metadata from %s', oauth_client_id, oauth_server_metadata_url
) )
return oauth_client_info return oauth_client_info
except Exception as e: except Exception as e:
@@ -807,7 +809,7 @@ async def recover_static_oauth_client_metadata(connection: dict, oauth_client_in
recovered = {**oauth_client_info} recovered = {**oauth_client_info}
if not recovered.get('scope') and resource_metadata.scopes_supported: if not recovered.get('scope') and resource_metadata.scopes_supported:
recovered['scope'] = ' '.join(resource_metadata.scopes_supported) recovered['scope'] = ' '.join(resource_metadata.scopes_supported)
log.info(f'Recovered static OAuth scopes for {server_url} from protected resource metadata') log.info('Recovered static OAuth scopes for %s from protected resource metadata', server_url)
if not recovered.get('resource') and resource_metadata.resource: if not recovered.get('resource') and resource_metadata.resource:
recovered['resource'] = resource_metadata.resource recovered['resource'] = resource_metadata.resource
@@ -916,7 +918,7 @@ class OAuthClientManager:
def remove_client(self, client_id): def remove_client(self, client_id):
if client_id in self.clients: if client_id in self.clients:
del self.clients[client_id] del self.clients[client_id]
log.info(f'Removed OAuth client {client_id}') log.info('Removed OAuth client %s', client_id)
if hasattr(self.oauth, '_clients'): if hasattr(self.oauth, '_clients'):
if client_id in self.oauth._clients: if client_id in self.oauth._clients:
@@ -1074,7 +1076,7 @@ class OAuthClientManager:
if refreshed_token: if refreshed_token:
# Update the session with new token data # Update the session with new token data
session = await OAuthSessions.update_session_by_id(session.id, refreshed_token) session = await OAuthSessions.update_session_by_id(session.id, refreshed_token)
log.info(f'Successfully refreshed token for session {session.id}') log.info('Successfully refreshed token for session %s', session.id)
return session.token return session.token
else: else:
log.error(f'Failed to refresh token for session {session.id}') log.error(f'Failed to refresh token for session {session.id}')
@@ -1242,7 +1244,7 @@ class OAuthClientManager:
provider=client_id, provider=client_id,
token=token, token=token,
) )
log.info(f'Stored OAuth session server-side for user {user_id}, client_id {client_id}') log.info('Stored OAuth session server-side for user %s, client_id %s', user_id, client_id)
except Exception as e: except Exception as e:
error_message = 'Failed to store OAuth session server-side' error_message = 'Failed to store OAuth session server-side'
log.error(f'Failed to store OAuth session server-side: {e}') log.error(f'Failed to store OAuth session server-side: {e}')
@@ -1369,7 +1371,7 @@ class OAuthManager:
if refreshed_token: if refreshed_token:
# Update the session with new token data # Update the session with new token data
session = await OAuthSessions.update_session_by_id(session.id, refreshed_token) session = await OAuthSessions.update_session_by_id(session.id, refreshed_token)
log.info(f'Successfully refreshed token for session {session.id}') log.info('Successfully refreshed token for session %s', session.id)
return session.token return session.token
else: else:
log.error(f'Failed to refresh token for session {session.id}') log.error(f'Failed to refresh token for session {session.id}')
@@ -1594,7 +1596,7 @@ class OAuthManager:
for group_name in user_oauth_groups: for group_name in user_oauth_groups:
if group_name not in all_group_names: if group_name not in all_group_names:
log.info(f"Group '{group_name}' not found via OAuth claim. Creating group...") log.info("Group '%s' not found via OAuth claim. Creating group...", group_name)
try: try:
new_group_form = GroupForm( new_group_form = GroupForm(
name=group_name, name=group_name,
@@ -1606,7 +1608,10 @@ class OAuthManager:
created_group = await Groups.insert_new_group(creator_id, new_group_form, db=db) created_group = await Groups.insert_new_group(creator_id, new_group_form, db=db)
if created_group: if created_group:
log.info( log.info(
f"Successfully created group '{group_name}' with ID {created_group.id} using creator ID {creator_id}" "Successfully created group '%s' with ID %s using creator ID %s",
group_name,
created_group.id,
creator_id,
) )
groups_created = True groups_created = True
# Add to local set to prevent duplicate creation attempts in this run # Add to local set to prevent duplicate creation attempts in this run
@@ -2096,7 +2101,7 @@ class OAuthManager:
**({'max_age': cookie_max_age} if cookie_max_age is not None else {}), **({'max_age': cookie_max_age} if cookie_max_age is not None else {}),
) )
log.info(f'Stored OAuth session server-side for user {user.id}, provider {provider}') log.info('Stored OAuth session server-side for user %s, provider %s', user.id, provider)
else: else:
log.warning(f'Failed to create OAuth session for user {user.id}, provider {provider}') log.warning(f'Failed to create OAuth session for user {user.id}, provider {provider}')
except Exception as e: except Exception as e:
@@ -2280,11 +2285,14 @@ class OAuthManager:
revoked_count += 1 revoked_count += 1
log.info( log.info(
f'Back-channel logout: revoked sessions for user {user.id} ' 'Back-channel logout: revoked sessions for user %s (email=%s, provider=%s, sessions_deleted=%s)',
f'(email={user.email}, provider={matched_provider}, sessions_deleted={len(sessions)})' user.id,
user.email,
matched_provider,
len(sessions),
) )
log.info( log.info(
f'Back-channel logout: completed for {len(users_to_logout)} user(s), {revoked_count} revocation(s) set' 'Back-channel logout: completed for %s user(s), %s revocation(s) set', len(users_to_logout), revoked_count
) )
return JSONResponse(status_code=200, content={}) return JSONResponse(status_code=200, content={})
+3 -3
View File
@@ -241,7 +241,7 @@ async def load_tool_module_by_id(tool_id, content=None):
exec(content, module.__dict__) exec(content, module.__dict__)
if frontmatter is None: if frontmatter is None:
frontmatter = extract_frontmatter(content) frontmatter = extract_frontmatter(content)
log.info(f'Loaded module: {module.__name__}') log.info('Loaded module: %s', module.__name__)
# Create and return the object if the class 'Tools' is found in the module # Create and return the object if the class 'Tools' is found in the module
if hasattr(module, 'Tools'): if hasattr(module, 'Tools'):
@@ -291,7 +291,7 @@ async def load_function_module_by_id(function_id: str, content: str | None = Non
exec(content, module.__dict__) exec(content, module.__dict__)
if frontmatter is None: if frontmatter is None:
frontmatter = extract_frontmatter(content) frontmatter = extract_frontmatter(content)
log.info(f'Loaded module: {module.__name__}') log.info('Loaded module: %s', module.__name__)
# Create appropriate object based on available class type in the module # Create appropriate object based on available class type in the module
if hasattr(module, 'Pipe'): if hasattr(module, 'Pipe'):
@@ -437,7 +437,7 @@ def install_frontmatter_requirements(requirements: str):
if not new_reqs: if not new_reqs:
return return
log.info(f'Installing requirements: {" ".join(new_reqs)}') log.info('Installing requirements: %s', ' '.join(new_reqs))
subprocess.check_call( subprocess.check_call(
[sys.executable, '-m', 'pip', 'install'] + PIP_OPTIONS + new_reqs + PIP_PACKAGE_INDEX_OPTIONS [sys.executable, '-m', 'pip', 'install'] + PIP_OPTIONS + new_reqs + PIP_PACKAGE_INDEX_OPTIONS
) )