perf: build info log messages lazily so raising the log level actually saves work (#27837)

Raising GLOBAL_LOG_LEVEL to WARNING buys quieter output but not less work: 241 INFO call sites interpolate their payload into an f-string before the logging call gets to drop it. The heaviest is get_doc, which logs every chunk id and metadata dict in a collection, so on the full-context retrieval path that is the entire knowledge base, once per chat request.

That one line at WARNING, CPython 3.12:

| knowledge base | payload | before   | after   |
| -------------- | ------- | -------- | ------- |
| top-k of 3     | 1.2 kB  | 3.8 us   | 0.07 us |
| 500 chunks     | 201 kB  | 583.6 us | 0.08 us |
| 5000 chunks    | 2.0 MB  | 5.8 ms   | 0.15 us |

The lazy form log.info('query_doc:result %s %s', result.ids, result.metadatas) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. Output at INFO is byte-identical. Two sites that already built their message eagerly, one str concat and one % operator, move to the same lazy form.
This commit is contained in:
Classic298 authored and GitHub committed 2026-08-02 15:39:10 -05:00
1 parent 52145eede9
commit 2d18727ab8
48 files changed
+297 -247

No files matched your search

+17 -15
View File
@@ -261,7 +261,7 @@ class MistralLoader:
file_id = response_data.get('id')
if not file_id:
raise ValueError('File ID not found in upload response.')
log.info(f'File uploaded successfully. File ID: {file_id}')
log.info('File uploaded successfully. File ID: %s', file_id)
return file_id
except Exception as e:
log.error(f'Failed to upload file: {e}')
@@ -305,12 +305,12 @@ class MistralLoader:
if not file_id:
raise ValueError('File ID not found in upload response.')
log.info(f'File uploaded successfully. File ID: {file_id}')
log.info('File uploaded successfully. File ID: %s', file_id)
return file_id
def _get_signed_url(self, file_id: str) -> str:
"""Retrieves a temporary signed URL for the uploaded file (sync version)."""
log.info(f'Getting signed URL for file ID: {file_id}')
log.info('Getting signed URL for file ID: %s', file_id)
url = f'{self.base_url}/files/{file_id}/url'
params = {'expiry': 1}
signed_url_headers = {**self.headers, 'Accept': 'application/json'}
@@ -421,7 +421,7 @@ class MistralLoader:
ocr_response = await self._handle_response_async(response)
processing_time = time.time() - start_time
log.info(f'OCR processing completed in {processing_time:.2f}s')
log.info('OCR processing completed in %.2fs', processing_time)
return ocr_response
@@ -434,13 +434,13 @@ class MistralLoader:
def _delete_file(self, file_id: str) -> None:
"""Deletes the file from Mistral storage (sync version)."""
log.info(f'Deleting uploaded file ID: {file_id}')
log.info('Deleting uploaded file ID: %s', file_id)
url = f'{self.base_url}/files/{file_id}'
try:
response = requests.delete(url, headers=self.headers, timeout=self.cleanup_timeout)
delete_response = self._handle_response(response)
log.info(f'File deleted successfully: {delete_response}')
log.info('File deleted successfully: %s', delete_response)
except Exception as e:
# Log error but don't necessarily halt execution if deletion fails
log.error(f'Failed to delete file ID {file_id}: {e}')
@@ -551,7 +551,7 @@ class MistralLoader:
)
if skipped_pages > 0:
log.info(f'Processed {len(documents)} pages, skipped {skipped_pages} empty/invalid pages')
log.info('Processed %s pages, skipped %s empty/invalid pages', len(documents), skipped_pages)
if not documents:
# Case where pages existed but none had valid markdown/index
@@ -584,7 +584,7 @@ class MistralLoader:
if self.use_base64:
documents = self._process_results(self._process_ocr(self._get_file_data_url()))
total_time = time.time() - start_time
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents
# 1. Upload file
@@ -600,7 +600,7 @@ class MistralLoader:
documents = self._process_results(ocr_response)
total_time = time.time() - start_time
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents
@@ -642,7 +642,7 @@ class MistralLoader:
ocr_response = await self._process_ocr_async(session, self._get_file_data_url())
documents = self._process_results(ocr_response)
total_time = time.time() - start_time
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents
# 1. Upload file with streaming
@@ -658,7 +658,7 @@ class MistralLoader:
documents = self._process_results(ocr_response)
total_time = time.time() - start_time
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
return documents
@@ -701,7 +701,7 @@ class MistralLoader:
if not loaders:
return []
log.info(f'Starting concurrent processing of {len(loaders)} files with max {max_concurrent} concurrent')
log.info('Starting concurrent processing of %s files with max %s concurrent', len(loaders), max_concurrent)
start_time = time.time()
# Use semaphore to control concurrency
@@ -741,9 +741,11 @@ class MistralLoader:
failure_count = len(results) - success_count
log.info(
f'Batch processing completed in {total_time:.2f}s: '
f'{success_count} files succeeded, {failure_count} files failed, '
f'produced {total_docs} total documents'
'Batch processing completed in %.2fs: %s files succeeded, %s files failed, produced %s total documents',
total_time,
success_count,
failure_count,
total_docs,
)
return processed_results