perf: build info log messages lazily so raising the log level actually saves work (#27837)
Raising GLOBAL_LOG_LEVEL to WARNING buys quieter output but not less work: 241 INFO call sites interpolate their payload into an f-string before the logging call gets to drop it. The heaviest is get_doc, which logs every chunk id and metadata dict in a collection, so on the full-context retrieval path that is the entire knowledge base, once per chat request.
That one line at WARNING, CPython 3.12:
| knowledge base | payload | before | after |
| -------------- | ------- | -------- | ------- |
| top-k of 3 | 1.2 kB | 3.8 us | 0.07 us |
| 500 chunks | 201 kB | 583.6 us | 0.08 us |
| 5000 chunks | 2.0 MB | 5.8 ms | 0.15 us |
The lazy form log.info('query_doc:result %s %s', result.ids, result.metadatas) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. Output at INFO is byte-identical. Two sites that already built their message eagerly, one str concat and one % operator, move to the same lazy form.
This commit is contained in:
1 parent
52145eede9
commit
2d18727ab8
48 files changed
+297
-247
No files matched your search
@@ -72,7 +72,7 @@ class DatalabMarkerLoader:
|
||||
response = requests.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
log.info(f'Marker API status check for request {request_id}: {result}')
|
||||
log.info('Marker API status check for request %s: %s', request_id, result)
|
||||
return result
|
||||
except requests.HTTPError as e:
|
||||
log.error(f'Error checking Marker request status: {e}')
|
||||
@@ -104,7 +104,10 @@ class DatalabMarkerLoader:
|
||||
form_data['additional_config'] = self.additional_config
|
||||
|
||||
log.info(
|
||||
f"Datalab Marker POST request parameters: {{'filename': '{filename}', 'mime_type': '{mime_type}', **{form_data}}}"
|
||||
"Datalab Marker POST request parameters: {'filename': '%s', 'mime_type': '%s', **%s}",
|
||||
filename,
|
||||
mime_type,
|
||||
form_data,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -168,7 +171,7 @@ class DatalabMarkerLoader:
|
||||
'total_cost',
|
||||
)
|
||||
}
|
||||
log.info(f'Marker processing completed successfully: {json.dumps(summary, indent=2)}')
|
||||
log.info('Marker processing completed successfully: %s', json.dumps(summary, indent=2))
|
||||
break
|
||||
|
||||
if status_val == 'failed' or success_val is False:
|
||||
@@ -235,7 +238,7 @@ class DatalabMarkerLoader:
|
||||
try:
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write(full_text)
|
||||
log.info(f'Saved Marker output to: {output_path}')
|
||||
log.info('Saved Marker output to: %s', output_path)
|
||||
except Exception as e:
|
||||
log.warning(f'Failed to write marker output to disk: {e}')
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class MinerULoader:
|
||||
Load document using Local API (synchronous).
|
||||
Posts file to /file_parse endpoint and gets immediate response.
|
||||
"""
|
||||
log.info(f'Using MinerU Local API at {self.api_url}')
|
||||
log.info('Using MinerU Local API at %s', self.api_url)
|
||||
|
||||
filename = os.path.basename(self.file_path)
|
||||
|
||||
@@ -97,7 +97,7 @@ class MinerULoader:
|
||||
with open(self.file_path, 'rb') as f:
|
||||
files = {'files': (filename, f, 'application/octet-stream')}
|
||||
|
||||
log.info(f'Sending file to MinerU Local API: {filename}')
|
||||
log.info('Sending file to MinerU Local API: %s', filename)
|
||||
log.debug('Local API parameters: %s', form_data)
|
||||
|
||||
response = requests.post(
|
||||
@@ -163,7 +163,7 @@ class MinerULoader:
|
||||
detail='MinerU returned empty markdown content',
|
||||
)
|
||||
|
||||
log.info(f'Successfully parsed document with MinerU Local API: {filename}')
|
||||
log.info('Successfully parsed document with MinerU Local API: %s', filename)
|
||||
|
||||
# Create metadata
|
||||
metadata = {
|
||||
@@ -180,7 +180,7 @@ class MinerULoader:
|
||||
Load document using Cloud API (asynchronous).
|
||||
Uses batch upload endpoint to avoid need for public file URLs.
|
||||
"""
|
||||
log.info(f'Using MinerU Cloud API at {self.api_url}')
|
||||
log.info('Using MinerU Cloud API at %s', self.api_url)
|
||||
|
||||
filename = os.path.basename(self.file_path)
|
||||
|
||||
@@ -196,7 +196,7 @@ class MinerULoader:
|
||||
# Step 4: Download and extract markdown from ZIP
|
||||
markdown_content = self._download_and_extract_zip(result['full_zip_url'], filename)
|
||||
|
||||
log.info(f'Successfully parsed document with MinerU Cloud API: {filename}')
|
||||
log.info('Successfully parsed document with MinerU Cloud API: %s', filename)
|
||||
|
||||
# Create metadata
|
||||
metadata = {
|
||||
@@ -232,7 +232,7 @@ class MinerULoader:
|
||||
if self.page_ranges:
|
||||
request_body['files'][0]['page_ranges'] = self.page_ranges
|
||||
|
||||
log.info(f'Requesting upload URL for: {filename}')
|
||||
log.info('Requesting upload URL for: %s', filename)
|
||||
log.debug('Cloud API request body: %s', request_body)
|
||||
|
||||
try:
|
||||
@@ -284,7 +284,7 @@ class MinerULoader:
|
||||
)
|
||||
|
||||
upload_url = file_urls[0]
|
||||
log.info(f'Received upload URL for batch: {batch_id}')
|
||||
log.info('Received upload URL for batch: %s', batch_id)
|
||||
|
||||
return batch_id, upload_url
|
||||
|
||||
@@ -334,7 +334,7 @@ class MinerULoader:
|
||||
max_iterations = 300 # 10 minutes max (2 seconds per iteration)
|
||||
poll_interval = 2 # seconds
|
||||
|
||||
log.info(f'Polling batch status: {batch_id}')
|
||||
log.info('Polling batch status: %s', batch_id)
|
||||
|
||||
for iteration in range(max_iterations):
|
||||
try:
|
||||
@@ -393,7 +393,7 @@ class MinerULoader:
|
||||
state = file_result.get('state')
|
||||
|
||||
if state == 'done':
|
||||
log.info(f'Processing complete for {filename}')
|
||||
log.info('Processing complete for %s', filename)
|
||||
return file_result
|
||||
elif state == 'failed':
|
||||
error_msg = file_result.get('err_msg', 'Unknown error')
|
||||
@@ -404,7 +404,7 @@ class MinerULoader:
|
||||
elif state in ['waiting-file', 'pending', 'running', 'converting']:
|
||||
# Still processing
|
||||
if iteration % 10 == 0: # Log every 20 seconds
|
||||
log.info(f'Processing status: {state} (iteration {iteration + 1}/{max_iterations})')
|
||||
log.info('Processing status: %s (iteration %s/%s)', state, iteration + 1, max_iterations)
|
||||
time.sleep(poll_interval)
|
||||
else:
|
||||
log.warning(f'Unknown state: {state}')
|
||||
@@ -421,7 +421,7 @@ class MinerULoader:
|
||||
Download ZIP file from CDN and extract markdown content.
|
||||
Returns the markdown content as a string.
|
||||
"""
|
||||
log.info(f'Downloading results from: {zip_url}')
|
||||
log.info('Downloading results from: %s', zip_url)
|
||||
|
||||
try:
|
||||
response = requests.get(zip_url, timeout=60)
|
||||
@@ -452,7 +452,7 @@ class MinerULoader:
|
||||
read_errors = []
|
||||
|
||||
for member in md_members:
|
||||
log.info(f'Found markdown file in ZIP: {member.filename}')
|
||||
log.info('Found markdown file in ZIP: %s', member.filename)
|
||||
try:
|
||||
with zip_ref.open(member, 'r') as f:
|
||||
if self.max_markdown_bytes is None:
|
||||
@@ -515,5 +515,5 @@ class MinerULoader:
|
||||
detail='Extracted markdown content is empty',
|
||||
)
|
||||
|
||||
log.info(f'Successfully extracted markdown content ({len(markdown_content)} characters)')
|
||||
log.info('Successfully extracted markdown content (%s characters)', len(markdown_content))
|
||||
return markdown_content
|
||||
@@ -261,7 +261,7 @@ class MistralLoader:
|
||||
file_id = response_data.get('id')
|
||||
if not file_id:
|
||||
raise ValueError('File ID not found in upload response.')
|
||||
log.info(f'File uploaded successfully. File ID: {file_id}')
|
||||
log.info('File uploaded successfully. File ID: %s', file_id)
|
||||
return file_id
|
||||
except Exception as e:
|
||||
log.error(f'Failed to upload file: {e}')
|
||||
@@ -305,12 +305,12 @@ class MistralLoader:
|
||||
if not file_id:
|
||||
raise ValueError('File ID not found in upload response.')
|
||||
|
||||
log.info(f'File uploaded successfully. File ID: {file_id}')
|
||||
log.info('File uploaded successfully. File ID: %s', file_id)
|
||||
return file_id
|
||||
|
||||
def _get_signed_url(self, file_id: str) -> str:
|
||||
"""Retrieves a temporary signed URL for the uploaded file (sync version)."""
|
||||
log.info(f'Getting signed URL for file ID: {file_id}')
|
||||
log.info('Getting signed URL for file ID: %s', file_id)
|
||||
url = f'{self.base_url}/files/{file_id}/url'
|
||||
params = {'expiry': 1}
|
||||
signed_url_headers = {**self.headers, 'Accept': 'application/json'}
|
||||
@@ -421,7 +421,7 @@ class MistralLoader:
|
||||
ocr_response = await self._handle_response_async(response)
|
||||
|
||||
processing_time = time.time() - start_time
|
||||
log.info(f'OCR processing completed in {processing_time:.2f}s')
|
||||
log.info('OCR processing completed in %.2fs', processing_time)
|
||||
|
||||
return ocr_response
|
||||
|
||||
@@ -434,13 +434,13 @@ class MistralLoader:
|
||||
|
||||
def _delete_file(self, file_id: str) -> None:
|
||||
"""Deletes the file from Mistral storage (sync version)."""
|
||||
log.info(f'Deleting uploaded file ID: {file_id}')
|
||||
log.info('Deleting uploaded file ID: %s', file_id)
|
||||
url = f'{self.base_url}/files/{file_id}'
|
||||
|
||||
try:
|
||||
response = requests.delete(url, headers=self.headers, timeout=self.cleanup_timeout)
|
||||
delete_response = self._handle_response(response)
|
||||
log.info(f'File deleted successfully: {delete_response}')
|
||||
log.info('File deleted successfully: %s', delete_response)
|
||||
except Exception as e:
|
||||
# Log error but don't necessarily halt execution if deletion fails
|
||||
log.error(f'Failed to delete file ID {file_id}: {e}')
|
||||
@@ -551,7 +551,7 @@ class MistralLoader:
|
||||
)
|
||||
|
||||
if skipped_pages > 0:
|
||||
log.info(f'Processed {len(documents)} pages, skipped {skipped_pages} empty/invalid pages')
|
||||
log.info('Processed %s pages, skipped %s empty/invalid pages', len(documents), skipped_pages)
|
||||
|
||||
if not documents:
|
||||
# Case where pages existed but none had valid markdown/index
|
||||
@@ -584,7 +584,7 @@ class MistralLoader:
|
||||
if self.use_base64:
|
||||
documents = self._process_results(self._process_ocr(self._get_file_data_url()))
|
||||
total_time = time.time() - start_time
|
||||
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
|
||||
log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
|
||||
return documents
|
||||
|
||||
# 1. Upload file
|
||||
@@ -600,7 +600,7 @@ class MistralLoader:
|
||||
documents = self._process_results(ocr_response)
|
||||
|
||||
total_time = time.time() - start_time
|
||||
log.info(f'Sync OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
|
||||
log.info('Sync OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
|
||||
|
||||
return documents
|
||||
|
||||
@@ -642,7 +642,7 @@ class MistralLoader:
|
||||
ocr_response = await self._process_ocr_async(session, self._get_file_data_url())
|
||||
documents = self._process_results(ocr_response)
|
||||
total_time = time.time() - start_time
|
||||
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
|
||||
log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
|
||||
return documents
|
||||
|
||||
# 1. Upload file with streaming
|
||||
@@ -658,7 +658,7 @@ class MistralLoader:
|
||||
documents = self._process_results(ocr_response)
|
||||
|
||||
total_time = time.time() - start_time
|
||||
log.info(f'Async OCR workflow completed in {total_time:.2f}s, produced {len(documents)} documents')
|
||||
log.info('Async OCR workflow completed in %.2fs, produced %s documents', total_time, len(documents))
|
||||
|
||||
return documents
|
||||
|
||||
@@ -701,7 +701,7 @@ class MistralLoader:
|
||||
if not loaders:
|
||||
return []
|
||||
|
||||
log.info(f'Starting concurrent processing of {len(loaders)} files with max {max_concurrent} concurrent')
|
||||
log.info('Starting concurrent processing of %s files with max %s concurrent', len(loaders), max_concurrent)
|
||||
start_time = time.time()
|
||||
|
||||
# Use semaphore to control concurrency
|
||||
@@ -741,9 +741,11 @@ class MistralLoader:
|
||||
failure_count = len(results) - success_count
|
||||
|
||||
log.info(
|
||||
f'Batch processing completed in {total_time:.2f}s: '
|
||||
f'{success_count} files succeeded, {failure_count} files failed, '
|
||||
f'produced {total_docs} total documents'
|
||||
'Batch processing completed in %.2fs: %s files succeeded, %s files failed, produced %s total documents',
|
||||
total_time,
|
||||
success_count,
|
||||
failure_count,
|
||||
total_docs,
|
||||
)
|
||||
|
||||
return processed_results
|
||||
@@ -35,7 +35,7 @@ class PaddleOCRVLLoader:
|
||||
self.file_name = os.path.basename(file_path)
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
log.info(f'Processing with PaddleOCR-vl: {self.file_path}')
|
||||
log.info('Processing with PaddleOCR-vl: %s', self.file_path)
|
||||
|
||||
try:
|
||||
with open(self.file_path, 'rb') as file:
|
||||
@@ -96,7 +96,7 @@ class PaddleOCRVLLoader:
|
||||
)
|
||||
|
||||
if skipped_pages > 0:
|
||||
log.info(f'PaddleOCR-vl: Processed {len(documents)} pages, skipped {skipped_pages} empty pages.')
|
||||
log.info('PaddleOCR-vl: Processed %s pages, skipped %s empty pages.', len(documents), skipped_pages)
|
||||
|
||||
if not documents:
|
||||
log.warning('No valid text content found by PaddleOCR-vl.')
|
||||
|
||||
@@ -138,7 +138,7 @@ class YoutubeLoader:
|
||||
log.debug("No transcript found for language '%s'", lang)
|
||||
continue
|
||||
except Exception as e:
|
||||
log.info(f"Error finding transcript for language '{lang}'")
|
||||
log.info("Error finding transcript for language '%s'", lang)
|
||||
raise e
|
||||
|
||||
# If we get here, all languages failed
|
||||
|
||||
Reference in new issue
Block a user