perf: build debug log messages lazily so disabled debug logs cost nothing (#27834)
GLOBAL_LOG_LEVEL defaults to INFO, so every log.debug(...) in the backend is discarded, but the message is built first: 187 call sites interpolate their payload into an f-string before the logging call runs, so the work happens on every request and the result is thrown away. The worst one sits in process_chat_payload and stringifies the whole request body, full conversation history included, once per chat completion.
That one line with DEBUG disabled, CPython 3.12:
| conversation | payload | before | after |
| ------------ | ------- | -------- | ------- |
| 4 messages | 1.2 kB | 3.4 us | 0.07 us |
| 20 messages | 17 kB | 24.8 us | 0.07 us |
| 60 messages | 123 kB | 216.6 us | 0.07 us |
The lazy form log.debug('form_data: %s', form_data) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. With DEBUG enabled the emitted lines are byte-identical, f'{x=}' sites included: those map to %r. MistralLoader._debug_log callers get the same treatment, since that wrapper already forwards *args.
This commit is contained in:
1 parent
b67804f2b6
commit
52cfb02c72
40 files changed
+198
-190
No files matched your search
@@ -100,7 +100,7 @@ async def get_function_models(request):
|
|||||||
log.exception(e)
|
log.exception(e)
|
||||||
sub_pipes = []
|
sub_pipes = []
|
||||||
|
|
||||||
log.debug(f"get_function_models: function '{pipe.id}' is a manifold of {sub_pipes}")
|
log.debug("get_function_models: function '%s' is a manifold of %s", pipe.id, sub_pipes)
|
||||||
|
|
||||||
for p in sub_pipes:
|
for p in sub_pipes:
|
||||||
sub_pipe_id = f'{pipe.id}.{p["id"]}'
|
sub_pipe_id = f'{pipe.id}.{p["id"]}'
|
||||||
@@ -126,7 +126,10 @@ async def get_function_models(request):
|
|||||||
pipe_flag = {'type': 'pipe'}
|
pipe_flag = {'type': 'pipe'}
|
||||||
|
|
||||||
log.debug(
|
log.debug(
|
||||||
f"get_function_models: function '{pipe.id}' is a single pipe {{ 'id': {pipe.id}, 'name': {pipe.name} }}"
|
"get_function_models: function '%s' is a single pipe { 'id': %s, 'name': %s }",
|
||||||
|
pipe.id,
|
||||||
|
pipe.id,
|
||||||
|
pipe.name,
|
||||||
)
|
)
|
||||||
|
|
||||||
pipe_models.append(
|
pipe_models.append(
|
||||||
|
|||||||
@@ -878,7 +878,7 @@ async def get_models(request: Request, refresh: bool = False, user=Depends(get_v
|
|||||||
tags = list(set(model_tags + tags))
|
tags = list(set(model_tags + tags))
|
||||||
model['tags'] = [{'name': tag} for tag in tags]
|
model['tags'] = [{'name': tag} for tag in tags]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error processing model tags: {e}')
|
log.debug('Error processing model tags: %s', e)
|
||||||
model['tags'] = []
|
model['tags'] = []
|
||||||
|
|
||||||
model_order_list = await Config.get('ui.model_order_list')
|
model_order_list = await Config.get('ui.model_order_list')
|
||||||
@@ -1377,7 +1377,7 @@ async def chat_completion(
|
|||||||
user.id,
|
user.id,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error inserting chat files: {e}')
|
log.debug('Error inserting chat files: %s', e)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if initial_title_generation is not None and all_assistant_ids:
|
if initial_title_generation is not None and all_assistant_ids:
|
||||||
@@ -1399,7 +1399,7 @@ async def chat_completion(
|
|||||||
try:
|
try:
|
||||||
await background_tasks_handler(title_ctx)
|
await background_tasks_handler(title_ctx)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error generating initial chat title: {e}')
|
log.debug('Error generating initial chat title: %s', e)
|
||||||
|
|
||||||
asyncio.create_task(run_initial_title_generation())
|
asyncio.create_task(run_initial_title_generation())
|
||||||
else:
|
else:
|
||||||
@@ -1487,7 +1487,7 @@ async def chat_completion(
|
|||||||
user.id,
|
user.id,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error inserting chat files: {e}')
|
log.debug('Error inserting chat files: %s', e)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Save ALL assistant placeholders
|
# Save ALL assistant placeholders
|
||||||
@@ -1653,9 +1653,9 @@ async def chat_completion(
|
|||||||
try:
|
try:
|
||||||
await client.disconnect()
|
await client.disconnect()
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
log.debug(f'Error disconnecting MCP client: {e}')
|
log.debug('Error disconnecting MCP client: %s', e)
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
log.debug(f'Error cleaning up MCP clients: {e}')
|
log.debug('Error cleaning up MCP clients: %s', e)
|
||||||
|
|
||||||
# Deregister this task, then emit chat:active=false if no others remain
|
# Deregister this task, then emit chat:active=false if no others remain
|
||||||
try:
|
try:
|
||||||
@@ -2051,7 +2051,7 @@ async def list_tasks_by_chat_id_endpoint(request: Request, chat_id: str, user=De
|
|||||||
|
|
||||||
task_ids = await list_task_ids_by_item_id(request.app.state.redis, chat_id)
|
task_ids = await list_task_ids_by_item_id(request.app.state.redis, chat_id)
|
||||||
|
|
||||||
log.debug(f'Task IDs for chat {chat_id}: {task_ids}')
|
log.debug('Task IDs for chat %s: %s', chat_id, task_ids)
|
||||||
return {'task_ids': task_ids}
|
return {'task_ids': task_ids}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ class TagTable:
|
|||||||
async with get_async_db_context(db) as db:
|
async with get_async_db_context(db) as db:
|
||||||
id = name.replace(' ', '_').lower()
|
id = name.replace(' ', '_').lower()
|
||||||
result = await db.execute(delete(Tag).filter_by(id=id, user_id=user_id))
|
result = await db.execute(delete(Tag).filter_by(id=id, user_id=user_id))
|
||||||
log.debug(f'res: {result.rowcount}')
|
log.debug('res: %s', result.rowcount)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ class MinerULoader:
|
|||||||
files = {'files': (filename, f, 'application/octet-stream')}
|
files = {'files': (filename, f, 'application/octet-stream')}
|
||||||
|
|
||||||
log.info(f'Sending file to MinerU Local API: {filename}')
|
log.info(f'Sending file to MinerU Local API: {filename}')
|
||||||
log.debug(f'Local API parameters: {form_data}')
|
log.debug('Local API parameters: %s', form_data)
|
||||||
|
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
f'{self.api_url}/file_parse',
|
f'{self.api_url}/file_parse',
|
||||||
@@ -233,7 +233,7 @@ class MinerULoader:
|
|||||||
request_body['files'][0]['page_ranges'] = self.page_ranges
|
request_body['files'][0]['page_ranges'] = self.page_ranges
|
||||||
|
|
||||||
log.info(f'Requesting upload URL for: {filename}')
|
log.info(f'Requesting upload URL for: {filename}')
|
||||||
log.debug(f'Cloud API request body: {request_body}')
|
log.debug('Cloud API request body: %s', request_body)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
|
|||||||
@@ -338,7 +338,7 @@ class MistralLoader:
|
|||||||
headers = {**self.headers, 'Accept': 'application/json'}
|
headers = {**self.headers, 'Accept': 'application/json'}
|
||||||
|
|
||||||
async def url_request():
|
async def url_request():
|
||||||
self._debug_log(f'Getting signed URL for file ID: {file_id}')
|
self._debug_log('Getting signed URL for file ID: %s', file_id)
|
||||||
async with session.get(
|
async with session.get(
|
||||||
url,
|
url,
|
||||||
headers=headers,
|
headers=headers,
|
||||||
@@ -450,7 +450,7 @@ class MistralLoader:
|
|||||||
try:
|
try:
|
||||||
|
|
||||||
async def delete_request():
|
async def delete_request():
|
||||||
self._debug_log(f'Deleting file ID: {file_id}')
|
self._debug_log('Deleting file ID: %s', file_id)
|
||||||
async with session.delete(
|
async with session.delete(
|
||||||
url=f'{self.base_url}/files/{file_id}',
|
url=f'{self.base_url}/files/{file_id}',
|
||||||
headers=self.headers,
|
headers=self.headers,
|
||||||
@@ -460,7 +460,7 @@ class MistralLoader:
|
|||||||
return await self._handle_response_async(response)
|
return await self._handle_response_async(response)
|
||||||
|
|
||||||
await self._retry_request_async(delete_request)
|
await self._retry_request_async(delete_request)
|
||||||
self._debug_log(f'File {file_id} deleted successfully')
|
self._debug_log('File %s deleted successfully', file_id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Don't fail the entire process if cleanup fails
|
# Don't fail the entire process if cleanup fails
|
||||||
@@ -519,7 +519,7 @@ class MistralLoader:
|
|||||||
if page_content is None or page_index is None:
|
if page_content is None or page_index is None:
|
||||||
skipped_pages += 1
|
skipped_pages += 1
|
||||||
self._debug_log(
|
self._debug_log(
|
||||||
f"Skipping page due to missing 'markdown' or 'index'. Data keys: {list(page_data.keys())}"
|
"Skipping page due to missing 'markdown' or 'index'. Data keys: %s", list(page_data.keys())
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -531,7 +531,7 @@ class MistralLoader:
|
|||||||
|
|
||||||
if not cleaned_content:
|
if not cleaned_content:
|
||||||
skipped_pages += 1
|
skipped_pages += 1
|
||||||
self._debug_log(f'Skipping empty page {page_index}')
|
self._debug_log('Skipping empty page %s', page_index)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Create document with optimized metadata
|
# Create document with optimized metadata
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ class YoutubeLoader:
|
|||||||
|
|
||||||
if self.proxy_url:
|
if self.proxy_url:
|
||||||
youtube_proxies = GenericProxyConfig(http_url=self.proxy_url, https_url=self.proxy_url)
|
youtube_proxies = GenericProxyConfig(http_url=self.proxy_url, https_url=self.proxy_url)
|
||||||
log.debug(f'Using proxy URL: {self.proxy_url[:14]}...')
|
log.debug('Using proxy URL: %s...', self.proxy_url[:14])
|
||||||
else:
|
else:
|
||||||
youtube_proxies = None
|
youtube_proxies = None
|
||||||
|
|
||||||
@@ -106,23 +106,23 @@ class YoutubeLoader:
|
|||||||
try:
|
try:
|
||||||
transcript = transcript_list.find_transcript([lang])
|
transcript = transcript_list.find_transcript([lang])
|
||||||
if transcript.is_generated:
|
if transcript.is_generated:
|
||||||
log.debug(f"Found generated transcript for language '{lang}'")
|
log.debug("Found generated transcript for language '%s'", lang)
|
||||||
try:
|
try:
|
||||||
transcript = transcript_list.find_manually_created_transcript([lang])
|
transcript = transcript_list.find_manually_created_transcript([lang])
|
||||||
log.debug(f"Found manual transcript for language '{lang}'")
|
log.debug("Found manual transcript for language '%s'", lang)
|
||||||
except NoTranscriptFound:
|
except NoTranscriptFound:
|
||||||
log.debug(f"No manual transcript found for language '{lang}', using generated")
|
log.debug("No manual transcript found for language '%s', using generated", lang)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
log.debug(f"Found transcript for language '{lang}'")
|
log.debug("Found transcript for language '%s'", lang)
|
||||||
try:
|
try:
|
||||||
transcript_pieces: List[Dict[str, Any]] = transcript.fetch()
|
transcript_pieces: List[Dict[str, Any]] = transcript.fetch()
|
||||||
except ParseError:
|
except ParseError:
|
||||||
log.debug(f"Empty or invalid transcript for language '{lang}'")
|
log.debug("Empty or invalid transcript for language '%s'", lang)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not transcript_pieces:
|
if not transcript_pieces:
|
||||||
log.debug(f"Empty transcript for language '{lang}'")
|
log.debug("Empty transcript for language '%s'", lang)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
transcript_text = ' '.join(
|
transcript_text = ' '.join(
|
||||||
@@ -135,7 +135,7 @@ class YoutubeLoader:
|
|||||||
)
|
)
|
||||||
return [Document(page_content=transcript_text, metadata=self._metadata)]
|
return [Document(page_content=transcript_text, metadata=self._metadata)]
|
||||||
except NoTranscriptFound:
|
except NoTranscriptFound:
|
||||||
log.debug(f"No transcript found for language '{lang}'")
|
log.debug("No transcript found for language '%s'", lang)
|
||||||
continue
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info(f"Error finding transcript for language '{lang}'")
|
log.info(f"Error finding transcript for language '{lang}'")
|
||||||
|
|||||||
@@ -310,7 +310,7 @@ class VectorSearchRetriever(BaseRetriever):
|
|||||||
|
|
||||||
def query_doc(collection_name: str, query_embedding: list[float], k: int, user: UserModel = None):
|
def query_doc(collection_name: str, query_embedding: list[float], k: int, user: UserModel = None):
|
||||||
try:
|
try:
|
||||||
log.debug(f'query_doc:doc {collection_name}')
|
log.debug('query_doc:doc %s', collection_name)
|
||||||
result = VECTOR_DB_CLIENT.search(
|
result = VECTOR_DB_CLIENT.search(
|
||||||
collection_name=collection_name,
|
collection_name=collection_name,
|
||||||
vectors=[query_embedding],
|
vectors=[query_embedding],
|
||||||
@@ -328,7 +328,7 @@ def query_doc(collection_name: str, query_embedding: list[float], k: int, user:
|
|||||||
|
|
||||||
def get_doc(collection_name: str, user: UserModel = None):
|
def get_doc(collection_name: str, user: UserModel = None):
|
||||||
try:
|
try:
|
||||||
log.debug(f'get_doc:doc {collection_name}')
|
log.debug('get_doc:doc %s', collection_name)
|
||||||
result = VECTOR_DB_CLIENT.get(collection_name=collection_name)
|
result = VECTOR_DB_CLIENT.get(collection_name=collection_name)
|
||||||
|
|
||||||
if result:
|
if result:
|
||||||
@@ -457,7 +457,7 @@ async def query_doc_with_native_hybrid_search(
|
|||||||
'metadatas': [metadatas],
|
'metadatas': [metadatas],
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Native hybrid search failed for {collection_name}, falling back to legacy hybrid search: {e}')
|
log.debug('Native hybrid search failed for %s, falling back to legacy hybrid search: %s', collection_name, e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -510,7 +510,7 @@ async def query_doc_with_hybrid_search(
|
|||||||
log.warning(f'query_doc_with_hybrid_search:no_docs {collection_name}')
|
log.warning(f'query_doc_with_hybrid_search:no_docs {collection_name}')
|
||||||
return {'documents': [], 'metadatas': [], 'distances': []}
|
return {'documents': [], 'metadatas': [], 'distances': []}
|
||||||
|
|
||||||
log.debug(f'query_doc_with_hybrid_search:doc {collection_name}')
|
log.debug('query_doc_with_hybrid_search:doc %s', collection_name)
|
||||||
|
|
||||||
original_texts = collection_result.documents[0]
|
original_texts = collection_result.documents[0]
|
||||||
bm25_metadatas = [
|
bm25_metadatas = [
|
||||||
@@ -707,7 +707,7 @@ async def query_collection(
|
|||||||
enable_enriched_texts=config.get('rag.enable_hybrid_search_enriched_texts'),
|
enable_enriched_texts=config.get('rag.enable_hybrid_search_enriched_texts'),
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Hybrid search failed, falling back to vector search: {e}')
|
log.debug('Hybrid search failed, falling back to vector search: %s', e)
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
error = False
|
error = False
|
||||||
@@ -736,7 +736,7 @@ async def query_collection(
|
|||||||
|
|
||||||
# Generate all query embeddings (in one call)
|
# Generate all query embeddings (in one call)
|
||||||
query_embeddings = await embedding_function(queries, prefix=RAG_EMBEDDING_QUERY_PREFIX)
|
query_embeddings = await embedding_function(queries, prefix=RAG_EMBEDDING_QUERY_PREFIX)
|
||||||
log.debug(f'query_collection: processing {len(queries)} queries across {len(collection_names)} collections')
|
log.debug('query_collection: processing %s queries across %s collections', len(queries), len(collection_names))
|
||||||
|
|
||||||
task_results = await asyncio.gather(
|
task_results = await asyncio.gather(
|
||||||
*[
|
*[
|
||||||
@@ -866,7 +866,7 @@ def generate_openai_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'generate_openai_batch_embeddings:model {model} batch size: {len(texts)}')
|
log.debug('generate_openai_batch_embeddings:model %s batch size: %s', model, len(texts))
|
||||||
json_data = {'input': texts, 'model': model}
|
json_data = {'input': texts, 'model': model}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -899,7 +899,7 @@ async def agenerate_openai_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'agenerate_openai_batch_embeddings:model {model} batch size: {len(texts)}')
|
log.debug('agenerate_openai_batch_embeddings:model %s batch size: %s', model, len(texts))
|
||||||
form_data = {'input': texts, 'model': model}
|
form_data = {'input': texts, 'model': model}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -937,7 +937,7 @@ def generate_azure_openai_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'generate_azure_openai_batch_embeddings:deployment {model} batch size: {len(texts)}')
|
log.debug('generate_azure_openai_batch_embeddings:deployment %s batch size: %s', model, len(texts))
|
||||||
json_data = {'input': texts}
|
json_data = {'input': texts}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -979,7 +979,7 @@ async def agenerate_azure_openai_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'agenerate_azure_openai_batch_embeddings:deployment {model} batch size: {len(texts)}')
|
log.debug('agenerate_azure_openai_batch_embeddings:deployment %s batch size: %s', model, len(texts))
|
||||||
form_data = {'input': texts}
|
form_data = {'input': texts}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -1018,7 +1018,7 @@ def generate_ollama_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'generate_ollama_batch_embeddings:model {model} batch size: {len(texts)}')
|
log.debug('generate_ollama_batch_embeddings:model %s batch size: %s', model, len(texts))
|
||||||
json_data = {'input': texts, 'model': model, 'truncate': True}
|
json_data = {'input': texts, 'model': model, 'truncate': True}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -1054,7 +1054,7 @@ async def agenerate_ollama_batch_embeddings(
|
|||||||
prefix: str = None,
|
prefix: str = None,
|
||||||
user: UserModel = None,
|
user: UserModel = None,
|
||||||
) -> list[list[float]]:
|
) -> list[list[float]]:
|
||||||
log.debug(f'agenerate_ollama_batch_embeddings:model {model} batch size: {len(texts)}')
|
log.debug('agenerate_ollama_batch_embeddings:model %s batch size: %s', model, len(texts))
|
||||||
form_data = {'input': texts, 'model': model, 'truncate': True}
|
form_data = {'input': texts, 'model': model, 'truncate': True}
|
||||||
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
|
||||||
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
|
||||||
@@ -1138,7 +1138,7 @@ def get_embedding_function(
|
|||||||
batches = [query[i : i + embedding_batch_size] for i in range(0, len(query), embedding_batch_size)]
|
batches = [query[i : i + embedding_batch_size] for i in range(0, len(query), embedding_batch_size)]
|
||||||
|
|
||||||
if enable_async:
|
if enable_async:
|
||||||
log.debug(f'generate_multiple_async: Processing {len(batches)} batches in parallel')
|
log.debug('generate_multiple_async: Processing %s batches in parallel', len(batches))
|
||||||
# Use semaphore to limit concurrent embedding API requests
|
# Use semaphore to limit concurrent embedding API requests
|
||||||
# 0 = unlimited (no semaphore)
|
# 0 = unlimited (no semaphore)
|
||||||
if concurrent_requests:
|
if concurrent_requests:
|
||||||
@@ -1153,7 +1153,7 @@ def get_embedding_function(
|
|||||||
tasks = [embedding_function(batch, prefix=prefix, user=user) for batch in batches]
|
tasks = [embedding_function(batch, prefix=prefix, user=user) for batch in batches]
|
||||||
batch_results = await asyncio.gather(*tasks)
|
batch_results = await asyncio.gather(*tasks)
|
||||||
else:
|
else:
|
||||||
log.debug(f'generate_multiple_async: Processing {len(batches)} batches sequentially')
|
log.debug('generate_multiple_async: Processing %s batches sequentially', len(batches))
|
||||||
batch_results = []
|
batch_results = []
|
||||||
for batch in batches:
|
for batch in batches:
|
||||||
batch_results.append(await embedding_function(batch, prefix=prefix, user=user))
|
batch_results.append(await embedding_function(batch, prefix=prefix, user=user))
|
||||||
@@ -1166,7 +1166,9 @@ def get_embedding_function(
|
|||||||
embeddings.extend(batch_embeddings)
|
embeddings.extend(batch_embeddings)
|
||||||
|
|
||||||
log.debug(
|
log.debug(
|
||||||
f'generate_multiple_async: Generated {len(embeddings)} embeddings from {len(batches)} parallel batches'
|
'generate_multiple_async: Generated %s embeddings from %s parallel batches',
|
||||||
|
len(embeddings),
|
||||||
|
len(batches),
|
||||||
)
|
)
|
||||||
return embeddings
|
return embeddings
|
||||||
else:
|
else:
|
||||||
@@ -1606,14 +1608,14 @@ async def get_sources_from_items(
|
|||||||
if query_result is None and collection_names:
|
if query_result is None and collection_names:
|
||||||
collection_names = set(collection_names).difference(extracted_collections)
|
collection_names = set(collection_names).difference(extracted_collections)
|
||||||
if not collection_names:
|
if not collection_names:
|
||||||
log.debug(f'skipping {item} as it has already been extracted')
|
log.debug('skipping %s as it has already been extracted', item)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Filter out collections the user cannot read
|
# Filter out collections the user cannot read
|
||||||
if user and (item.get('type'), item.get('id')) not in folder_items:
|
if user and (item.get('type'), item.get('id')) not in folder_items:
|
||||||
collection_names = await filter_accessible_collections(collection_names, user)
|
collection_names = await filter_accessible_collections(collection_names, user)
|
||||||
if not collection_names:
|
if not collection_names:
|
||||||
log.debug(f'access denied for all collections in item {item}')
|
log.debug('access denied for all collections in item %s', item)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -1672,8 +1674,8 @@ def get_model_path(model: str, update_model: bool = False):
|
|||||||
'local_files_only': local_files_only,
|
'local_files_only': local_files_only,
|
||||||
}
|
}
|
||||||
|
|
||||||
log.debug(f'model: {model}')
|
log.debug('model: %s', model)
|
||||||
log.debug(f'snapshot_kwargs: {snapshot_kwargs}')
|
log.debug('snapshot_kwargs: %s', snapshot_kwargs)
|
||||||
|
|
||||||
# Inspiration from upstream sentence_transformers
|
# Inspiration from upstream sentence_transformers
|
||||||
if os.path.exists(model) or ('\\' in model or model.count('/') > 1) and local_files_only:
|
if os.path.exists(model) or ('\\' in model or model.count('/') > 1) and local_files_only:
|
||||||
@@ -1688,7 +1690,7 @@ def get_model_path(model: str, update_model: bool = False):
|
|||||||
# Attempt to query the huggingface_hub library to determine the local path and/or to update
|
# Attempt to query the huggingface_hub library to determine the local path and/or to update
|
||||||
try:
|
try:
|
||||||
model_repo_path = snapshot_download(**snapshot_kwargs)
|
model_repo_path = snapshot_download(**snapshot_kwargs)
|
||||||
log.debug(f'model_repo_path: {model_repo_path}')
|
log.debug('model_repo_path: %s', model_repo_path)
|
||||||
return model_repo_path
|
return model_repo_path
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.exception(f'Cannot determine model snapshot path: {e}')
|
log.exception(f'Cannot determine model snapshot path: {e}')
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ class ChromaClient(VectorDBBase):
|
|||||||
collection.delete(where=filter)
|
collection.delete(where=filter)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# If collection doesn't exist, that's fine - nothing to delete
|
# If collection doesn't exist, that's fine - nothing to delete
|
||||||
log.debug(f'Attempted to delete from non-existent collection {collection_name}. Ignoring.')
|
log.debug('Attempted to delete from non-existent collection %s. Ignoring.', collection_name)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def reset(self):
|
def reset(self):
|
||||||
|
|||||||
@@ -242,7 +242,7 @@ class MilvusClient(VectorDBBase):
|
|||||||
break
|
break
|
||||||
all_results.extend(batch)
|
all_results.extend(batch)
|
||||||
|
|
||||||
log.debug(f'Total results from query: {len(all_results)}')
|
log.debug('Total results from query: %s', len(all_results))
|
||||||
return self._result_to_get_result([all_results] if all_results else [[]])
|
return self._result_to_get_result([all_results] if all_results else [[]])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -274,7 +274,7 @@ class PineconeClient(VectorDBBase):
|
|||||||
log.error(f'Error inserting batch: {e}')
|
log.error(f'Error inserting batch: {e}')
|
||||||
raise
|
raise
|
||||||
elapsed = time.time() - start_time
|
elapsed = time.time() - start_time
|
||||||
log.debug(f'Insert of {len(points)} vectors took {elapsed:.2f} seconds')
|
log.debug('Insert of %s vectors took %.2f seconds', len(points), elapsed)
|
||||||
log.info(
|
log.info(
|
||||||
f"Successfully inserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
|
f"Successfully inserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
|
||||||
)
|
)
|
||||||
@@ -303,7 +303,7 @@ class PineconeClient(VectorDBBase):
|
|||||||
log.error(f'Error upserting batch: {e}')
|
log.error(f'Error upserting batch: {e}')
|
||||||
raise
|
raise
|
||||||
elapsed = time.time() - start_time
|
elapsed = time.time() - start_time
|
||||||
log.debug(f'Upsert of {len(points)} vectors took {elapsed:.2f} seconds')
|
log.debug('Upsert of %s vectors took %.2f seconds', len(points), elapsed)
|
||||||
log.info(
|
log.info(
|
||||||
f"Successfully upserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
|
f"Successfully upserted {len(points)} vectors in parallel batches into '{collection_name_with_prefix}'"
|
||||||
)
|
)
|
||||||
@@ -474,7 +474,9 @@ class PineconeClient(VectorDBBase):
|
|||||||
# Note: When deleting by ID, we can't filter by collection_name
|
# Note: When deleting by ID, we can't filter by collection_name
|
||||||
# This is a limitation of Pinecone - be careful with ID uniqueness
|
# This is a limitation of Pinecone - be careful with ID uniqueness
|
||||||
self.index.delete(ids=batch_ids)
|
self.index.delete(ids=batch_ids)
|
||||||
log.debug(f"Deleted batch of {len(batch_ids)} vectors by ID from '{collection_name_with_prefix}'")
|
log.debug(
|
||||||
|
"Deleted batch of %s vectors by ID from '%s'", len(batch_ids), collection_name_with_prefix
|
||||||
|
)
|
||||||
log.info(f"Successfully deleted {len(ids)} vectors by ID from '{collection_name_with_prefix}'")
|
log.info(f"Successfully deleted {len(ids)} vectors by ID from '{collection_name_with_prefix}'")
|
||||||
|
|
||||||
elif filter:
|
elif filter:
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ class QdrantClient(VectorDBBase):
|
|||||||
|
|
||||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||||
if not self.client.collection_exists(collection_name=mt_collection):
|
if not self.client.collection_exists(collection_name=mt_collection):
|
||||||
log.debug(f"Collection {mt_collection} doesn't exist, nothing to delete")
|
log.debug("Collection %s doesn't exist, nothing to delete", mt_collection)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
must_conditions = [_tenant_filter(tenant_id)]
|
must_conditions = [_tenant_filter(tenant_id)]
|
||||||
@@ -255,7 +255,7 @@ class QdrantClient(VectorDBBase):
|
|||||||
return None
|
return None
|
||||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||||
if not self.client.collection_exists(collection_name=mt_collection):
|
if not self.client.collection_exists(collection_name=mt_collection):
|
||||||
log.debug(f"Collection {mt_collection} doesn't exist, search returns None")
|
log.debug("Collection %s doesn't exist, search returns None", mt_collection)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
tenant_filter = _tenant_filter(tenant_id)
|
tenant_filter = _tenant_filter(tenant_id)
|
||||||
@@ -281,7 +281,7 @@ class QdrantClient(VectorDBBase):
|
|||||||
return None
|
return None
|
||||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||||
if not self.client.collection_exists(collection_name=mt_collection):
|
if not self.client.collection_exists(collection_name=mt_collection):
|
||||||
log.debug(f"Collection {mt_collection} doesn't exist, query returns None")
|
log.debug("Collection %s doesn't exist, query returns None", mt_collection)
|
||||||
return None
|
return None
|
||||||
if limit is None:
|
if limit is None:
|
||||||
limit = NO_LIMIT
|
limit = NO_LIMIT
|
||||||
@@ -303,7 +303,7 @@ class QdrantClient(VectorDBBase):
|
|||||||
return None
|
return None
|
||||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||||
if not self.client.collection_exists(collection_name=mt_collection):
|
if not self.client.collection_exists(collection_name=mt_collection):
|
||||||
log.debug(f"Collection {mt_collection} doesn't exist, get returns None")
|
log.debug("Collection %s doesn't exist, get returns None", mt_collection)
|
||||||
return None
|
return None
|
||||||
tenant_filter = _tenant_filter(tenant_id)
|
tenant_filter = _tenant_filter(tenant_id)
|
||||||
points = self.client.scroll(
|
points = self.client.scroll(
|
||||||
@@ -350,7 +350,7 @@ class QdrantClient(VectorDBBase):
|
|||||||
return None
|
return None
|
||||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||||
if not self.client.collection_exists(collection_name=mt_collection):
|
if not self.client.collection_exists(collection_name=mt_collection):
|
||||||
log.debug(f"Collection {mt_collection} doesn't exist, nothing to delete")
|
log.debug("Collection %s doesn't exist, nothing to delete", mt_collection)
|
||||||
return None
|
return None
|
||||||
self.client.delete(
|
self.client.delete(
|
||||||
collection_name=mt_collection,
|
collection_name=mt_collection,
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ class S3VectorClient(VectorDBBase):
|
|||||||
Create a new index in the S3 vector bucket for the given collection if it does not exist.
|
Create a new index in the S3 vector bucket for the given collection if it does not exist.
|
||||||
"""
|
"""
|
||||||
if self.has_collection(index_name):
|
if self.has_collection(index_name):
|
||||||
log.debug(f"Index '{index_name}' already exists, skipping creation")
|
log.debug("Index '%s' already exists, skipping creation", index_name)
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -310,7 +310,7 @@ class S3VectorClient(VectorDBBase):
|
|||||||
|
|
||||||
# Process each query vector
|
# Process each query vector
|
||||||
for i, query_vector in enumerate(vectors):
|
for i, query_vector in enumerate(vectors):
|
||||||
log.debug(f'Processing query vector {i + 1}/{len(vectors)}')
|
log.debug('Processing query vector %s/%s', i + 1, len(vectors))
|
||||||
|
|
||||||
# Prepare the query vector in S3 Vector format
|
# Prepare the query vector in S3 Vector format
|
||||||
query_vector_dict = {'float32': [float(x) for x in query_vector]}
|
query_vector_dict = {'float32': [float(x) for x in query_vector]}
|
||||||
@@ -521,7 +521,7 @@ class S3VectorClient(VectorDBBase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Log the actual content for debugging
|
# Log the actual content for debugging
|
||||||
log.debug(f'Document text preview (first 200 chars): {str(document_text)[:200]}')
|
log.debug('Document text preview (first 200 chars): %s', str(document_text)[:200])
|
||||||
else:
|
else:
|
||||||
document_text = vector_id
|
document_text = vector_id
|
||||||
|
|
||||||
|
|||||||
@@ -390,7 +390,7 @@ class ValkeyClient(VectorDBBase):
|
|||||||
)
|
)
|
||||||
except g['RequestError'] as e:
|
except g['RequestError'] as e:
|
||||||
if 'already exists' in str(e).lower():
|
if 'already exists' in str(e).lower():
|
||||||
log.debug(f'Index {index_name} already exists, skipping creation.')
|
log.debug('Index %s already exists, skipping creation.', index_name)
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -458,7 +458,7 @@ class ValkeyClient(VectorDBBase):
|
|||||||
self._g['glide_ft'].dropindex(self.client, index_name)
|
self._g['glide_ft'].dropindex(self.client, index_name)
|
||||||
log.info(f'Dropped index {index_name}')
|
log.info(f'Dropped index {index_name}')
|
||||||
except self._g['RequestError'] as e:
|
except self._g['RequestError'] as e:
|
||||||
log.debug(f'Could not drop index {index_name}: {e}')
|
log.debug('Could not drop index %s: %s', index_name, e)
|
||||||
|
|
||||||
self._delete_keys_by_prefix(self._key_prefix(collection_name))
|
self._delete_keys_by_prefix(self._key_prefix(collection_name))
|
||||||
|
|
||||||
@@ -492,7 +492,7 @@ class ValkeyClient(VectorDBBase):
|
|||||||
}
|
}
|
||||||
self.batch_client.hset(self._item_key(collection_name, item['id']), mapping)
|
self.batch_client.hset(self._item_key(collection_name, item['id']), mapping)
|
||||||
|
|
||||||
log.debug(f'Inserted {len(items)} items into collection {collection_name}')
|
log.debug('Inserted %s items into collection %s', len(items), collection_name)
|
||||||
|
|
||||||
def upsert(self, collection_name: str, items: list[VectorItem]):
|
def upsert(self, collection_name: str, items: list[VectorItem]):
|
||||||
self.insert(collection_name, items)
|
self.insert(collection_name, items)
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
|
|||||||
if validate_url(u):
|
if validate_url(u):
|
||||||
valid_urls.append(u)
|
valid_urls.append(u)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Invalid URL {u}: {str(e)}')
|
log.debug('Invalid URL %s: %s', u, e)
|
||||||
continue
|
continue
|
||||||
return valid_urls
|
return valid_urls
|
||||||
|
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ def search_yacy(
|
|||||||
# Strip all query parameters from the URL
|
# Strip all query parameters from the URL
|
||||||
query_url = query_url.rstrip('/') + '/yacysearch.json'
|
query_url = query_url.rstrip('/') + '/yacysearch.json'
|
||||||
|
|
||||||
log.debug(f'searching {query_url}')
|
log.debug('searching %s', query_url)
|
||||||
|
|
||||||
response = requests.get(
|
response = requests.get(
|
||||||
query_url,
|
query_url,
|
||||||
|
|||||||
@@ -486,7 +486,7 @@ async def _tts_transformers(request, payload, file_path, file_body_path, user):
|
|||||||
try:
|
try:
|
||||||
idx = embeddings['filename'].index(model_name)
|
idx = embeddings['filename'].index(model_name)
|
||||||
except (ValueError, KeyError):
|
except (ValueError, KeyError):
|
||||||
log.debug(f'Speaker embedding not found for {model_name}, using default index {idx}')
|
log.debug('Speaker embedding not found for %s, using default index %s', model_name, idx)
|
||||||
|
|
||||||
def _run_pipeline():
|
def _run_pipeline():
|
||||||
speaker_embedding = torch.tensor(embeddings[idx]['xvector']).unsqueeze(0)
|
speaker_embedding = torch.tensor(embeddings[idx]['xvector']).unsqueeze(0)
|
||||||
@@ -1303,7 +1303,7 @@ async def get_available_models(request: Request) -> list[dict]:
|
|||||||
data = await resp.json()
|
data = await resp.json()
|
||||||
available_models = data.get('models', [])
|
available_models = data.get('models', [])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'/audio/models not available, trying /models fallback: {e}')
|
log.debug('/audio/models not available, trying /models fallback: %s', e)
|
||||||
try:
|
try:
|
||||||
async with session.get(
|
async with session.get(
|
||||||
f'{base_url}/models',
|
f'{base_url}/models',
|
||||||
|
|||||||
@@ -448,7 +448,7 @@ def _process_chat_for_export(chat) -> ChatStatsExport | None:
|
|||||||
history_models[model] = 0
|
history_models[model] = 0
|
||||||
history_models[model] += 1
|
history_models[model] += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error processing message {key}: {e}')
|
log.debug('Error processing message %s: %s', key, e)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Calculate Averages
|
# Calculate Averages
|
||||||
@@ -614,7 +614,7 @@ async def export_chat_stats(
|
|||||||
return ChatStatsExportList(items=chat_stats_export_list, total=total, page=page)
|
return ChatStatsExportList(items=chat_stats_export_list, total=total, page=page)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error exporting chat stats: {e}')
|
log.debug('Error exporting chat stats: %s', e)
|
||||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
||||||
|
|
||||||
|
|
||||||
@@ -671,7 +671,7 @@ async def export_single_chat_stats(
|
|||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error exporting single chat stats: {e}')
|
log.debug('Error exporting single chat stats: %s', e)
|
||||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
||||||
|
|
||||||
|
|
||||||
@@ -877,7 +877,7 @@ async def search_user_chats(
|
|||||||
tag_id = words[0].replace('tag:', '')
|
tag_id = words[0].replace('tag:', '')
|
||||||
if len(chat_list) == 0:
|
if len(chat_list) == 0:
|
||||||
if await Tags.get_tag_by_name_and_user_id(tag_id, user.id, db=db):
|
if await Tags.get_tag_by_name_and_user_id(tag_id, user.id, db=db):
|
||||||
log.debug(f'deleting tag: {tag_id}')
|
log.debug('deleting tag: %s', tag_id)
|
||||||
await Tags.delete_tag_by_name_and_user_id(tag_id, user.id, db=db)
|
await Tags.delete_tag_by_name_and_user_id(tag_id, user.id, db=db)
|
||||||
|
|
||||||
return await add_active_state_to_chat_list(request, chat_list)
|
return await add_active_state_to_chat_list(request, chat_list)
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ async def register_oauth_client(
|
|||||||
'oauth_client_info': encrypt_data(oauth_client_info.model_dump(mode='json')),
|
'oauth_client_info': encrypt_data(oauth_client_info.model_dump(mode='json')),
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to register OAuth client: {e}')
|
log.debug('Failed to register OAuth client: %s', e)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=f'Failed to register OAuth client: {e}',
|
detail=f'Failed to register OAuth client: {e}',
|
||||||
@@ -279,7 +279,7 @@ async def set_tool_servers_config(
|
|||||||
OAuthClientInformationFull(**oauth_client_info),
|
OAuthClientInformationFull(**oauth_client_info),
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to add OAuth client for MCP tool server: {e}')
|
log.debug('Failed to add OAuth client for MCP tool server: %s', e)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
await publish_event(
|
await publish_event(
|
||||||
@@ -390,7 +390,7 @@ async def verify_terminal_server_connection(
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to connect to the terminal server: {e}')
|
log.debug('Failed to connect to the terminal server: %s', e)
|
||||||
|
|
||||||
raise HTTPException(status_code=400, detail='Failed to connect to the terminal server')
|
raise HTTPException(status_code=400, detail='Failed to connect to the terminal server')
|
||||||
|
|
||||||
@@ -454,7 +454,7 @@ async def put_terminal_server_policy(
|
|||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to access policy on terminal server: {e}')
|
log.debug('Failed to access policy on terminal server: %s', e)
|
||||||
raise HTTPException(status_code=400, detail='Failed to access policy on terminal server')
|
raise HTTPException(status_code=400, detail='Failed to access policy on terminal server')
|
||||||
|
|
||||||
|
|
||||||
@@ -491,7 +491,7 @@ async def put_terminal_server_lifecycle(
|
|||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to access lifecycle on terminal server: {e}')
|
log.debug('Failed to access lifecycle on terminal server: %s', e)
|
||||||
raise HTTPException(status_code=400, detail='Failed to access lifecycle on terminal server')
|
raise HTTPException(status_code=400, detail='Failed to access lifecycle on terminal server')
|
||||||
|
|
||||||
|
|
||||||
@@ -538,7 +538,7 @@ async def refresh_terminal_server_terminals(
|
|||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to refresh terminals: {e}')
|
log.debug('Failed to refresh terminals: %s', e)
|
||||||
raise HTTPException(status_code=400, detail='Failed to refresh terminals')
|
raise HTTPException(status_code=400, detail='Failed to refresh terminals')
|
||||||
|
|
||||||
|
|
||||||
@@ -557,7 +557,7 @@ async def verify_tool_servers_config(request: Request, form_data: ToolServerConn
|
|||||||
)
|
)
|
||||||
discovery_urls = await get_discovery_urls(oauth_server_url)
|
discovery_urls = await get_discovery_urls(oauth_server_url)
|
||||||
for discovery_url in discovery_urls:
|
for discovery_url in discovery_urls:
|
||||||
log.debug(f'Trying to fetch OAuth 2.1 discovery document from {discovery_url}')
|
log.debug('Trying to fetch OAuth 2.1 discovery document from %s', discovery_url)
|
||||||
async with aiohttp.ClientSession(
|
async with aiohttp.ClientSession(
|
||||||
trust_env=True,
|
trust_env=True,
|
||||||
timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT),
|
timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT),
|
||||||
@@ -624,7 +624,7 @@ async def verify_tool_servers_config(request: Request, form_data: ToolServerConn
|
|||||||
'specs': specs,
|
'specs': specs,
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to create MCP client: {e}')
|
log.debug('Failed to create MCP client: %s', e)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=f'Failed to create MCP client',
|
detail=f'Failed to create MCP client',
|
||||||
@@ -667,7 +667,7 @@ async def verify_tool_servers_config(request: Request, form_data: ToolServerConn
|
|||||||
except HTTPException as e:
|
except HTTPException as e:
|
||||||
raise e
|
raise e
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to connect to the tool server: {e}')
|
log.debug('Failed to connect to the tool server: %s', e)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=f'Failed to connect to the tool server',
|
detail=f'Failed to connect to the tool server',
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ def _cleanup_local_cache(file_path: str) -> None:
|
|||||||
local_path = os.path.join(UPLOAD_DIR, local_filename)
|
local_path = os.path.join(UPLOAD_DIR, local_filename)
|
||||||
if os.path.isfile(local_path):
|
if os.path.isfile(local_path):
|
||||||
os.remove(local_path)
|
os.remove(local_path)
|
||||||
log.debug(f'Cleaned up local cache: {local_path}')
|
log.debug('Cleaned up local cache: %s', local_path)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
log.warning(f'Failed to clean up local cache for {file_path}: {e}')
|
log.warning(f'Failed to clean up local cache for {file_path}: {e}')
|
||||||
|
|
||||||
@@ -1021,7 +1021,7 @@ async def delete_file_by_id(
|
|||||||
if file.hash:
|
if file.hash:
|
||||||
await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=knowledge.id, filter={'hash': file.hash})
|
await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=knowledge.id, filter={'hash': file.hash})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'KB embedding cleanup for {knowledge.id}: {e}')
|
log.debug('KB embedding cleanup for %s: %s', knowledge.id, e)
|
||||||
|
|
||||||
result = await Files.delete_file_by_id(id, db=db)
|
result = await Files.delete_file_by_id(id, db=db)
|
||||||
if result:
|
if result:
|
||||||
|
|||||||
@@ -149,7 +149,7 @@ def normalize_openai_edit_image_data_url(data_url: str) -> str:
|
|||||||
normalized_image = base64.b64encode(output.getvalue()).decode('utf-8')
|
normalized_image = base64.b64encode(output.getvalue()).decode('utf-8')
|
||||||
return f'data:image/jpeg;base64,{normalized_image}'
|
return f'data:image/jpeg;base64,{normalized_image}'
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Image edit normalization skipped: {e}')
|
log.debug('Image edit normalization skipped: %s', e)
|
||||||
|
|
||||||
return data_url
|
return data_url
|
||||||
|
|
||||||
@@ -194,7 +194,7 @@ async def set_image_model(request: Request, model: str):
|
|||||||
) as r:
|
) as r:
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'{e}')
|
log.debug('%s', e)
|
||||||
|
|
||||||
return image_config.IMAGE_GENERATION_MODEL
|
return image_config.IMAGE_GENERATION_MODEL
|
||||||
|
|
||||||
@@ -488,7 +488,7 @@ async def get_image_data(data: str, headers=None, trusted_base_url: str | None =
|
|||||||
# ENABLE_LOCAL_WEB_FETCH hammer and a blanket trust flag
|
# ENABLE_LOCAL_WEB_FETCH hammer and a blanket trust flag
|
||||||
# that would follow arbitrary redirects.
|
# that would follow arbitrary redirects.
|
||||||
if trusted_base_url and _is_same_origin(data, trusted_base_url):
|
if trusted_base_url and _is_same_origin(data, trusted_base_url):
|
||||||
log.debug(f'Skipping URL validation for trusted backend: {data}')
|
log.debug('Skipping URL validation for trusted backend: %s', data)
|
||||||
else:
|
else:
|
||||||
await asyncio.to_thread(validate_url, data)
|
await asyncio.to_thread(validate_url, data)
|
||||||
session = await get_session()
|
session = await get_session()
|
||||||
@@ -762,7 +762,7 @@ async def image_generations(
|
|||||||
image_config.COMFYUI_BASE_URL,
|
image_config.COMFYUI_BASE_URL,
|
||||||
image_config.COMFYUI_API_KEY,
|
image_config.COMFYUI_API_KEY,
|
||||||
)
|
)
|
||||||
log.debug(f'res: {res}')
|
log.debug('res: %s', res)
|
||||||
|
|
||||||
images = []
|
images = []
|
||||||
|
|
||||||
@@ -817,7 +817,7 @@ async def image_generations(
|
|||||||
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
ssl=AIOHTTP_CLIENT_SESSION_SSL,
|
||||||
) as r:
|
) as r:
|
||||||
res = await r.json(content_type=None)
|
res = await r.json(content_type=None)
|
||||||
log.debug(f'res: {res}')
|
log.debug('res: %s', res)
|
||||||
|
|
||||||
images = []
|
images = []
|
||||||
|
|
||||||
@@ -1119,7 +1119,7 @@ async def image_edits(
|
|||||||
)
|
)
|
||||||
comfyui_images.append(res.get('name', file_item[1][0]))
|
comfyui_images.append(res.get('name', file_item[1][0]))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error uploading images to ComfyUI: {e}')
|
log.debug('Error uploading images to ComfyUI: %s', e)
|
||||||
raise Exception('Failed to upload images to ComfyUI.')
|
raise Exception('Failed to upload images to ComfyUI.')
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
@@ -1148,7 +1148,7 @@ async def image_edits(
|
|||||||
image_config.IMAGES_EDIT_COMFYUI_BASE_URL,
|
image_config.IMAGES_EDIT_COMFYUI_BASE_URL,
|
||||||
image_config.IMAGES_EDIT_COMFYUI_API_KEY,
|
image_config.IMAGES_EDIT_COMFYUI_API_KEY,
|
||||||
)
|
)
|
||||||
log.debug(f'res: {res}')
|
log.debug('res: %s', res)
|
||||||
|
|
||||||
image_urls = set()
|
image_urls = set()
|
||||||
for image in res['data']:
|
for image in res['data']:
|
||||||
@@ -1160,7 +1160,7 @@ async def image_edits(
|
|||||||
if output_type_urls:
|
if output_type_urls:
|
||||||
image_urls = output_type_urls
|
image_urls = output_type_urls
|
||||||
|
|
||||||
log.debug(f'Image URLs: {image_urls}')
|
log.debug('Image URLs: %s', image_urls)
|
||||||
images = []
|
images = []
|
||||||
|
|
||||||
for image_url in image_urls:
|
for image_url in image_urls:
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ async def remove_knowledge_base_metadata_embedding(knowledge_base_id: str) -> bo
|
|||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to remove embedding for {knowledge_base_id}: {e}')
|
log.debug('Failed to remove embedding for %s: %s', knowledge_base_id, e)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -443,7 +443,7 @@ async def get_all_models(request: Request, user: UserModel | None = None):
|
|||||||
dt = datetime.fromisoformat(expires_map[m['model']])
|
dt = datetime.fromisoformat(expires_map[m['model']])
|
||||||
m['expires_at'] = int(dt.timestamp())
|
m['expires_at'] = int(dt.timestamp())
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.debug(f'Failed to get loaded models: {exc}')
|
log.debug('Failed to get loaded models: %s', exc)
|
||||||
|
|
||||||
request.app.state.OLLAMA_MODELS = {m['model']: m for m in models_dict['models']}
|
request.app.state.OLLAMA_MODELS = {m['model']: m for m in models_dict['models']}
|
||||||
return models_dict
|
return models_dict
|
||||||
@@ -691,7 +691,7 @@ async def push_model(
|
|||||||
url_idx = models[form_data.model]['urls'][0]
|
url_idx = models[form_data.model]['urls'][0]
|
||||||
|
|
||||||
url = (await Config.get('ollama.base_urls', []))[url_idx]
|
url = (await Config.get('ollama.base_urls', []))[url_idx]
|
||||||
log.debug(f'url: {url}')
|
log.debug('url: %s', url)
|
||||||
|
|
||||||
return await send_request(
|
return await send_request(
|
||||||
f'{url}/api/push',
|
f'{url}/api/push',
|
||||||
@@ -723,7 +723,7 @@ async def create_model(
|
|||||||
if not await Config.get('ollama.enable'):
|
if not await Config.get('ollama.enable'):
|
||||||
raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
|
raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
|
||||||
|
|
||||||
log.debug(f'form_data: {form_data}')
|
log.debug('form_data: %s', form_data)
|
||||||
url = (await Config.get('ollama.base_urls', []))[url_idx]
|
url = (await Config.get('ollama.base_urls', []))[url_idx]
|
||||||
|
|
||||||
return await send_request(
|
return await send_request(
|
||||||
|
|||||||
@@ -613,7 +613,7 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list:
|
|||||||
if provider:
|
if provider:
|
||||||
model['provider'] = provider
|
model['provider'] = provider
|
||||||
|
|
||||||
log.debug(f'get_all_models:responses() {responses}')
|
log.debug('get_all_models:responses() %s', responses)
|
||||||
return responses
|
return responses
|
||||||
|
|
||||||
|
|
||||||
@@ -669,7 +669,7 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
|
|||||||
return not any(name in model_id for name in _UNSUPPORTED_OPENAI_MODEL_KEYWORDS)
|
return not any(name in model_id for name in _UNSUPPORTED_OPENAI_MODEL_KEYWORDS)
|
||||||
|
|
||||||
def get_merged_models(model_lists):
|
def get_merged_models(model_lists):
|
||||||
log.debug(f'merge_models_lists {model_lists}')
|
log.debug('merge_models_lists %s', model_lists)
|
||||||
models = {}
|
models = {}
|
||||||
|
|
||||||
for idx, model_list in enumerate(model_lists):
|
for idx, model_list in enumerate(model_lists):
|
||||||
@@ -710,7 +710,7 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
|
|||||||
return models
|
return models
|
||||||
|
|
||||||
models = get_merged_models(map(extract_data, responses))
|
models = get_merged_models(map(extract_data, responses))
|
||||||
log.debug(f'models: {models}')
|
log.debug('models: %s', models)
|
||||||
|
|
||||||
request.app.state.OPENAI_MODELS = models
|
request.app.state.OPENAI_MODELS = models
|
||||||
return {'data': list(models.values())}
|
return {'data': list(models.values())}
|
||||||
@@ -923,7 +923,7 @@ def get_azure_allowed_params(api_version: str) -> set[str]:
|
|||||||
if api_version >= '2024-09-01-preview':
|
if api_version >= '2024-09-01-preview':
|
||||||
allowed_params.add('stream_options')
|
allowed_params.add('stream_options')
|
||||||
except ValueError:
|
except ValueError:
|
||||||
log.debug(f'Invalid API version {api_version} for Azure OpenAI. Defaulting to allowed parameters.')
|
log.debug('Invalid API version %s for Azure OpenAI. Defaulting to allowed parameters.', api_version)
|
||||||
|
|
||||||
return allowed_params
|
return allowed_params
|
||||||
|
|
||||||
@@ -971,7 +971,7 @@ def convert_to_azure_payload(url, payload: dict, api_version: str):
|
|||||||
# Remove temperature if not 1 for o-series models
|
# Remove temperature if not 1 for o-series models
|
||||||
if 'temperature' in payload and payload['temperature'] != 1:
|
if 'temperature' in payload and payload['temperature'] != 1:
|
||||||
log.debug(
|
log.debug(
|
||||||
f'Removing temperature parameter for o-series model {model} as only default value (1) is supported'
|
'Removing temperature parameter for o-series model %s as only default value (1) is supported', model
|
||||||
)
|
)
|
||||||
del payload['temperature']
|
del payload['temperature']
|
||||||
|
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ router = APIRouter()
|
|||||||
@router.get('/list')
|
@router.get('/list')
|
||||||
async def get_pipelines_list(request: Request, user=Depends(get_admin_user)):
|
async def get_pipelines_list(request: Request, user=Depends(get_admin_user)):
|
||||||
responses = await get_all_models_responses(request, user)
|
responses = await get_all_models_responses(request, user)
|
||||||
log.debug(f'get_pipelines_list: get_openai_models_responses returned {responses}')
|
log.debug('get_pipelines_list: get_openai_models_responses returned %s', responses)
|
||||||
|
|
||||||
urlIdxs = [idx for idx, response in enumerate(responses) if response is not None and 'pipelines' in response]
|
urlIdxs = [idx for idx, response in enumerate(responses) if response is not None and 'pipelines' in response]
|
||||||
base_urls = await Config.get('openai.api_base_urls', [])
|
base_urls = await Config.get('openai.api_base_urls', [])
|
||||||
|
|||||||
@@ -230,7 +230,7 @@ def get_rf(
|
|||||||
eos = getattr(cfg, 'eos_token_id', None)
|
eos = getattr(cfg, 'eos_token_id', None)
|
||||||
if eos is not None:
|
if eos is not None:
|
||||||
cfg.pad_token_id = eos
|
cfg.pad_token_id = eos
|
||||||
log.debug(f'Missing pad_token_id detected; set to eos_token_id={eos}')
|
log.debug('Missing pad_token_id detected; set to eos_token_id=%s', eos)
|
||||||
else:
|
else:
|
||||||
log.warning('Neither pad_token_id nor eos_token_id present in model config')
|
log.warning('Neither pad_token_id nor eos_token_id present in model config')
|
||||||
except Exception as e2:
|
except Exception as e2:
|
||||||
@@ -1643,7 +1643,7 @@ def save_docs_to_vector_db(
|
|||||||
|
|
||||||
return ', '.join(docs_info)
|
return ', '.join(docs_info)
|
||||||
|
|
||||||
log.debug(f'save_docs_to_vector_db: document {_get_docs_info(docs)} {collection_name}')
|
log.debug('save_docs_to_vector_db: document %s %s', _get_docs_info(docs), collection_name)
|
||||||
|
|
||||||
# Check if entries with the same hash (metadata.hash) already exist
|
# Check if entries with the same hash (metadata.hash) already exist
|
||||||
if metadata and 'hash' in metadata:
|
if metadata and 'hash' in metadata:
|
||||||
@@ -2567,7 +2567,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen
|
|||||||
result_items = []
|
result_items = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logging.debug(f'trying to web search with {config.WEB_SEARCH_ENGINE, form_data.queries}')
|
logging.debug('trying to web search with %s', (config.WEB_SEARCH_ENGINE, form_data.queries))
|
||||||
|
|
||||||
# Use semaphore to limit concurrent requests based on WEB_SEARCH_CONCURRENT_REQUESTS
|
# Use semaphore to limit concurrent requests based on WEB_SEARCH_CONCURRENT_REQUESTS
|
||||||
# 0 or None = unlimited (previous behavior), positive number = limited concurrency
|
# 0 or None = unlimited (previous behavior), positive number = limited concurrency
|
||||||
@@ -2610,7 +2610,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen
|
|||||||
urls.append(item.link)
|
urls.append(item.link)
|
||||||
|
|
||||||
urls = list(dict.fromkeys(urls))
|
urls = list(dict.fromkeys(urls))
|
||||||
log.debug(f'urls: {urls}')
|
log.debug('urls: %s', urls)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.exception('Web search failed')
|
log.exception('Web search failed')
|
||||||
|
|||||||
@@ -268,7 +268,7 @@ def get_scim_auth(request: Request, authorization: Optional[str] = Header(None))
|
|||||||
|
|
||||||
# Verify the SCIM token
|
# Verify the SCIM token
|
||||||
scim_token = getattr(request.app.state, 'SCIM_TOKEN', None)
|
scim_token = getattr(request.app.state, 'SCIM_TOKEN', None)
|
||||||
log.debug(f'SCIM token configured: {bool(scim_token)}')
|
log.debug('SCIM token configured: %s', bool(scim_token))
|
||||||
if not scim_token or not hmac.compare_digest(token, scim_token):
|
if not scim_token or not hmac.compare_digest(token, scim_token):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
|||||||
@@ -144,7 +144,7 @@ async def generate_title(request: Request, form_data: dict, user=Depends(get_ver
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating chat title using model {task_model_id} for user {user.email} ')
|
log.debug('generating chat title using model %s for user %s ', task_model_id, user.email)
|
||||||
|
|
||||||
title_template = await Config.get('task.title.prompt_template')
|
title_template = await Config.get('task.title.prompt_template')
|
||||||
if title_template != '':
|
if title_template != '':
|
||||||
@@ -223,7 +223,7 @@ async def generate_follow_ups(request: Request, form_data: dict, user=Depends(ge
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating chat title using model {task_model_id} for user {user.email} ')
|
log.debug('generating chat title using model %s for user %s ', task_model_id, user.email)
|
||||||
|
|
||||||
follow_up_template = await Config.get('task.follow_up.prompt_template')
|
follow_up_template = await Config.get('task.follow_up.prompt_template')
|
||||||
if follow_up_template != '':
|
if follow_up_template != '':
|
||||||
@@ -293,7 +293,7 @@ async def generate_chat_tags(request: Request, form_data: dict, user=Depends(get
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating chat tags using model {task_model_id} for user {user.email} ')
|
log.debug('generating chat tags using model %s for user %s ', task_model_id, user.email)
|
||||||
|
|
||||||
tags_template = await Config.get('task.tags.prompt_template')
|
tags_template = await Config.get('task.tags.prompt_template')
|
||||||
if tags_template != '':
|
if tags_template != '':
|
||||||
@@ -357,7 +357,7 @@ async def generate_image_prompt(request: Request, form_data: dict, user=Depends(
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating image prompt using model {task_model_id} for user {user.email} ')
|
log.debug('generating image prompt using model %s for user %s ', task_model_id, user.email)
|
||||||
|
|
||||||
image_prompt_template = await Config.get('task.image.prompt_template')
|
image_prompt_template = await Config.get('task.image.prompt_template')
|
||||||
if image_prompt_template != '':
|
if image_prompt_template != '':
|
||||||
@@ -439,7 +439,7 @@ async def generate_queries(request: Request, form_data: dict, user=Depends(get_v
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating {type} queries using model {task_model_id} for user {user.email}')
|
log.debug('generating %s queries using model %s for user %s', type, task_model_id, user.email)
|
||||||
|
|
||||||
query_template = await Config.get('task.query.prompt_template')
|
query_template = await Config.get('task.query.prompt_template')
|
||||||
if query_template.strip() != '':
|
if query_template.strip() != '':
|
||||||
@@ -520,7 +520,7 @@ async def generate_autocompletion(request: Request, form_data: dict, user=Depend
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating autocompletion using model {task_model_id} for user {user.email}')
|
log.debug('generating autocompletion using model %s for user %s', task_model_id, user.email)
|
||||||
|
|
||||||
autocomplete_template = await Config.get('task.autocomplete.prompt_template')
|
autocomplete_template = await Config.get('task.autocomplete.prompt_template')
|
||||||
if autocomplete_template.strip() != '':
|
if autocomplete_template.strip() != '':
|
||||||
@@ -584,7 +584,7 @@ async def generate_emoji(request: Request, form_data: dict, user=Depends(get_ver
|
|||||||
models,
|
models,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.debug(f'generating emoji using model {task_model_id} for user {user.email} ')
|
log.debug('generating emoji using model %s for user %s ', task_model_id, user.email)
|
||||||
|
|
||||||
template = DEFAULT_EMOJI_GENERATION_PROMPT_TEMPLATE
|
template = DEFAULT_EMOJI_GENERATION_PROMPT_TEMPLATE
|
||||||
|
|
||||||
|
|||||||
@@ -210,7 +210,7 @@ async def periodic_usage_pool_cleanup():
|
|||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
if attempt < max_retries:
|
if attempt < max_retries:
|
||||||
log.debug(f'Cleanup lock already exists. Retry {attempt + 1} after {retry_delay}s...')
|
log.debug('Cleanup lock already exists. Retry %s after %ss...', attempt + 1, retry_delay)
|
||||||
await asyncio.sleep(retry_delay)
|
await asyncio.sleep(retry_delay)
|
||||||
else:
|
else:
|
||||||
log.warning('Failed to acquire cleanup lock after retries. Skipping cleanup.')
|
log.warning('Failed to acquire cleanup lock after retries. Skipping cleanup.')
|
||||||
@@ -237,7 +237,7 @@ async def periodic_usage_pool_cleanup():
|
|||||||
del connections[sid]
|
del connections[sid]
|
||||||
|
|
||||||
if not connections:
|
if not connections:
|
||||||
log.debug(f'Cleaning up model {model_id} from usage pool')
|
log.debug('Cleaning up model %s from usage pool', model_id)
|
||||||
del USAGE_POOL[model_id]
|
del USAGE_POOL[model_id]
|
||||||
else:
|
else:
|
||||||
USAGE_POOL[model_id] = connections
|
USAGE_POOL[model_id] = connections
|
||||||
@@ -299,7 +299,7 @@ async def emit_to_users(event: str, data: dict, user_ids: list[str]):
|
|||||||
for user_id in user_ids:
|
for user_id in user_ids:
|
||||||
await sio.emit(event, data, room=f'user:{user_id}')
|
await sio.emit(event, data, room=f'user:{user_id}')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to emit event {event} to users {user_ids}: {e}')
|
log.debug('Failed to emit event %s to users %s: %s', event, user_ids, e)
|
||||||
|
|
||||||
|
|
||||||
async def enter_room_for_users(room: str, user_ids: list[str]):
|
async def enter_room_for_users(room: str, user_ids: list[str]):
|
||||||
@@ -315,7 +315,7 @@ async def enter_room_for_users(room: str, user_ids: list[str]):
|
|||||||
for sid in session_ids:
|
for sid in session_ids:
|
||||||
await sio.enter_room(sid, room)
|
await sio.enter_room(sid, room)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to make users {user_ids} join room {room}: {e}')
|
log.debug('Failed to make users %s join room %s: %s', user_ids, room, e)
|
||||||
|
|
||||||
|
|
||||||
async def disconnect_user_sessions(user_id: str):
|
async def disconnect_user_sessions(user_id: str):
|
||||||
@@ -411,7 +411,7 @@ async def user_join(sid, data):
|
|||||||
# Join all the channels only if user has channels permission
|
# Join all the channels only if user has channels permission
|
||||||
if user.role == 'admin' or await has_permission(user.id, 'features.channels'):
|
if user.role == 'admin' or await has_permission(user.id, 'features.channels'):
|
||||||
channels = await Channels.get_channels_by_user_id(user.id)
|
channels = await Channels.get_channels_by_user_id(user.id)
|
||||||
log.debug(f'{channels=}')
|
log.debug('channels=%r', channels)
|
||||||
for channel in channels:
|
for channel in channels:
|
||||||
await sio.enter_room(sid, f'channel:{channel.id}')
|
await sio.enter_room(sid, f'channel:{channel.id}')
|
||||||
|
|
||||||
@@ -443,7 +443,7 @@ async def join_channel(sid, data):
|
|||||||
# Join all the channels only if user has channels permission
|
# Join all the channels only if user has channels permission
|
||||||
if user.role == 'admin' or await has_permission(user.id, 'features.channels'):
|
if user.role == 'admin' or await has_permission(user.id, 'features.channels'):
|
||||||
channels = await Channels.get_channels_by_user_id(user.id)
|
channels = await Channels.get_channels_by_user_id(user.id)
|
||||||
log.debug(f'{channels=}')
|
log.debug('channels=%r', channels)
|
||||||
for channel in channels:
|
for channel in channels:
|
||||||
await sio.enter_room(sid, f'channel:{channel.id}')
|
await sio.enter_room(sid, f'channel:{channel.id}')
|
||||||
|
|
||||||
@@ -480,7 +480,7 @@ async def join_note(sid, data):
|
|||||||
log.error(f'User {user.id} does not have access to note {data["note_id"]}')
|
log.error(f'User {user.id} does not have access to note {data["note_id"]}')
|
||||||
return
|
return
|
||||||
|
|
||||||
log.debug(f'Joining note {note.id} for user {user.id}')
|
log.debug('Joining note %s for user %s', note.id, user.id)
|
||||||
await sio.enter_room(sid, f'note:{note.id}')
|
await sio.enter_room(sid, f'note:{note.id}')
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ async def has_access_to_file(
|
|||||||
file.user_id == user.id separately before calling this.
|
file.user_id == user.id separately before calling this.
|
||||||
"""
|
"""
|
||||||
file = await Files.get_file_by_id(file_id, db=db)
|
file = await Files.get_file_by_id(file_id, db=db)
|
||||||
log.debug(f'Checking if user has {access_type} access to file')
|
log.debug('Checking if user has %s access to file', access_type)
|
||||||
if not file:
|
if not file:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|||||||
@@ -222,7 +222,7 @@ class AuditLoggingMiddleware:
|
|||||||
user = await get_current_user(request, None, None, get_http_authorization_cred(auth_header))
|
user = await get_current_user(request, None, None, get_http_authorization_cred(auth_header))
|
||||||
return user
|
return user
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f'Failed to get authenticated user: {str(e)}')
|
logger.debug('Failed to get authenticated user: {}', e)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -667,7 +667,7 @@ async def _check_calendar_alerts(app) -> None:
|
|||||||
CalendarEventUpdateForm(meta={'alerted_at': now_ns}),
|
CalendarEventUpdateForm(meta={'alerted_at': now_ns}),
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
log.debug(f'Failed to mark event {event.id} as alerted', exc_info=True)
|
log.debug('Failed to mark event %s as alerted', event.id, exc_info=True)
|
||||||
|
|
||||||
# Send target notification if user has one configured
|
# Send target notification if user has one configured
|
||||||
try:
|
try:
|
||||||
@@ -687,7 +687,7 @@ async def _check_calendar_alerts(app) -> None:
|
|||||||
message=event.title,
|
message=event.title,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
log.debug(f'Failed to send notification for calendar alert {event.id}', exc_info=True)
|
log.debug('Failed to send notification for calendar alert %s', event.id, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
async def _record_run(
|
async def _record_run(
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ async def generate_direct_chat_completion(
|
|||||||
else:
|
else:
|
||||||
yield f'data: {data}\n\n'
|
yield f'data: {data}\n\n'
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error in event generator: {e}')
|
log.debug('Error in event generator: %s', e)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Define a background task to run the event generator
|
# Define a background task to run the event generator
|
||||||
@@ -155,7 +155,7 @@ async def generate_chat_completion(
|
|||||||
bypass_filter: bool = False,
|
bypass_filter: bool = False,
|
||||||
bypass_system_prompt: bool = False,
|
bypass_system_prompt: bool = False,
|
||||||
):
|
):
|
||||||
log.debug(f'generate_chat_completion: {form_data}')
|
log.debug('generate_chat_completion: %s', form_data)
|
||||||
if BYPASS_MODEL_ACCESS_CONTROL:
|
if BYPASS_MODEL_ACCESS_CONTROL:
|
||||||
bypass_filter = True
|
bypass_filter = True
|
||||||
|
|
||||||
@@ -184,7 +184,7 @@ async def generate_chat_completion(
|
|||||||
**dict(request.app.state.MODELS.items()),
|
**dict(request.app.state.MODELS.items()),
|
||||||
request.state.model['id']: request.state.model,
|
request.state.model['id']: request.state.model,
|
||||||
}
|
}
|
||||||
log.debug(f'direct connection to model: {request.state.model["id"]}')
|
log.debug('direct connection to model: %s', request.state.model['id'])
|
||||||
else:
|
else:
|
||||||
models = request.app.state.MODELS
|
models = request.app.state.MODELS
|
||||||
|
|
||||||
|
|||||||
@@ -186,7 +186,7 @@ async def process_filter_function(
|
|||||||
|
|
||||||
form_data = await run_filter_handler(handler, params)
|
form_data = await run_filter_handler(handler, params)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error in {filter_type} handler {filter_id}: {e}')
|
log.debug('Error in %s handler %s: %s', filter_type, filter_id, e)
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
return form_data, valves_by_id, skip_files
|
return form_data, valves_by_id, skip_files
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ default_headers = {'User-Agent': 'Mozilla/5.0'}
|
|||||||
async def queue_prompt(prompt, client_id, base_url, api_key):
|
async def queue_prompt(prompt, client_id, base_url, api_key):
|
||||||
log.info('queue_prompt')
|
log.info('queue_prompt')
|
||||||
p = {'prompt': prompt, 'client_id': client_id}
|
p = {'prompt': prompt, 'client_id': client_id}
|
||||||
log.debug(f'queue_prompt data: {p}')
|
log.debug('queue_prompt data: %s', p)
|
||||||
try:
|
try:
|
||||||
session = await get_session()
|
session = await get_session()
|
||||||
async with session.post(
|
async with session.post(
|
||||||
@@ -202,7 +202,7 @@ async def comfyui_create_image(model: str, payload: ComfyUICreateImageForm, clie
|
|||||||
) as ws:
|
) as ws:
|
||||||
log.info('WebSocket connection established.')
|
log.info('WebSocket connection established.')
|
||||||
log.info('Sending workflow to WebSocket server.')
|
log.info('Sending workflow to WebSocket server.')
|
||||||
log.debug(f'Workflow: {workflow}')
|
log.debug('Workflow: %s', workflow)
|
||||||
images = await _ws_get_images(ws, workflow, client_id, base_url, api_key)
|
images = await _ws_get_images(ws, workflow, client_id, base_url, api_key)
|
||||||
except aiohttp.WSServerHandshakeError as e:
|
except aiohttp.WSServerHandshakeError as e:
|
||||||
log.exception(f'Failed to connect to WebSocket server: {e}')
|
log.exception(f'Failed to connect to WebSocket server: {e}')
|
||||||
@@ -243,7 +243,7 @@ async def comfyui_edit_image(model: str, payload: ComfyUIEditImageForm, client_i
|
|||||||
) as ws:
|
) as ws:
|
||||||
log.info('WebSocket connection established.')
|
log.info('WebSocket connection established.')
|
||||||
log.info('Sending workflow to WebSocket server.')
|
log.info('Sending workflow to WebSocket server.')
|
||||||
log.debug(f'Workflow: {workflow}')
|
log.debug('Workflow: %s', workflow)
|
||||||
images = await _ws_get_images(ws, workflow, client_id, base_url, api_key)
|
images = await _ws_get_images(ws, workflow, client_id, base_url, api_key)
|
||||||
except aiohttp.WSServerHandshakeError as e:
|
except aiohttp.WSServerHandshakeError as e:
|
||||||
log.exception(f'Failed to connect to WebSocket server: {e}')
|
log.exception(f'Failed to connect to WebSocket server: {e}')
|
||||||
|
|||||||
@@ -457,7 +457,7 @@ async def review_memory_after_turn(
|
|||||||
try:
|
try:
|
||||||
done_task.result()
|
done_task.result()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Memory review failed: {e}')
|
log.debug('Memory review failed: %s', e)
|
||||||
|
|
||||||
task.add_done_callback(log_failure)
|
task.add_done_callback(log_failure)
|
||||||
|
|
||||||
|
|||||||
@@ -1210,9 +1210,9 @@ async def chat_completion_tools_handler(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
response = await generate_chat_completion(request, form_data=payload, user=user)
|
response = await generate_chat_completion(request, form_data=payload, user=user)
|
||||||
log.debug(f'{response=}')
|
log.debug('response=%r', response)
|
||||||
content = await get_content_from_response(response)
|
content = await get_content_from_response(response)
|
||||||
log.debug(f'{content=}')
|
log.debug('content=%r', content)
|
||||||
|
|
||||||
if not content:
|
if not content:
|
||||||
return body, {}
|
return body, {}
|
||||||
@@ -1227,7 +1227,7 @@ async def chat_completion_tools_handler(
|
|||||||
async def tool_call_handler(tool_call):
|
async def tool_call_handler(tool_call):
|
||||||
nonlocal skip_files
|
nonlocal skip_files
|
||||||
|
|
||||||
log.debug(f'{tool_call=}')
|
log.debug('tool_call=%r', tool_call)
|
||||||
|
|
||||||
tool_function_name = tool_call.get('name', None)
|
tool_function_name = tool_call.get('name', None)
|
||||||
if tool_function_name not in tools:
|
if tool_function_name not in tools:
|
||||||
@@ -1341,13 +1341,13 @@ async def chat_completion_tools_handler(
|
|||||||
await tool_call_handler(result)
|
await tool_call_handler(result)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error: {e}')
|
log.debug('Error: %s', e)
|
||||||
content = None
|
content = None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error: {e}')
|
log.debug('Error: %s', e)
|
||||||
content = None
|
content = None
|
||||||
|
|
||||||
log.debug(f'tool_contexts: {sources}')
|
log.debug('tool_contexts: %s', sources)
|
||||||
|
|
||||||
if skip_files and 'files' in body.get('metadata', {}):
|
if skip_files and 'files' in body.get('metadata', {}):
|
||||||
del body['metadata']['files']
|
del body['metadata']['files']
|
||||||
@@ -1912,7 +1912,7 @@ async def chat_completion_files_handler(
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.exception(e)
|
log.exception(e)
|
||||||
|
|
||||||
log.debug(f'rag_contexts:sources: {sources}')
|
log.debug('rag_contexts:sources: %s', sources)
|
||||||
|
|
||||||
unique_ids = set()
|
unique_ids = set()
|
||||||
for source in sources or []:
|
for source in sources or []:
|
||||||
@@ -2028,7 +2028,7 @@ async def convert_url_images_to_base64(form_data, user=None):
|
|||||||
else:
|
else:
|
||||||
new_content.append(item)
|
new_content.append(item)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error converting image URL to base64: {e}')
|
log.debug('Error converting image URL to base64: %s', e)
|
||||||
new_content.append(item)
|
new_content.append(item)
|
||||||
|
|
||||||
message['content'] = new_content
|
message['content'] = new_content
|
||||||
@@ -2286,7 +2286,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||||||
model_system_prompt = (form_data.get('params') or {}).get('system')
|
model_system_prompt = (form_data.get('params') or {}).get('system')
|
||||||
|
|
||||||
form_data = apply_params_to_form_data(form_data, model)
|
form_data = apply_params_to_form_data(form_data, model)
|
||||||
log.debug(f'form_data: {form_data}')
|
log.debug('form_data: %s', form_data)
|
||||||
|
|
||||||
# Guided regeneration: extract before it reaches the LLM provider
|
# Guided regeneration: extract before it reaches the LLM provider
|
||||||
regeneration_prompt = form_data.pop('regeneration_prompt', None)
|
regeneration_prompt = form_data.pop('regeneration_prompt', None)
|
||||||
@@ -2725,8 +2725,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||||||
# Client side tools
|
# Client side tools
|
||||||
direct_tool_servers = metadata.get('tool_servers', None)
|
direct_tool_servers = metadata.get('tool_servers', None)
|
||||||
|
|
||||||
log.debug(f'{tool_ids=}')
|
log.debug('tool_ids=%r', tool_ids)
|
||||||
log.debug(f'{direct_tool_servers=}')
|
log.debug('direct_tool_servers=%r', direct_tool_servers)
|
||||||
|
|
||||||
tools_dict = {}
|
tools_dict = {}
|
||||||
|
|
||||||
@@ -3498,7 +3498,7 @@ async def outlet_filter_handler(ctx):
|
|||||||
try:
|
try:
|
||||||
outlet_data = await process_pipeline_outlet_filter(request, outlet_data, user, models)
|
outlet_data = await process_pipeline_outlet_filter(request, outlet_data, user, models)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Pipeline outlet filter error: {e}')
|
log.debug('Pipeline outlet filter error: %s', e)
|
||||||
|
|
||||||
# Function outlet filters
|
# Function outlet filters
|
||||||
extra_params = {
|
extra_params = {
|
||||||
@@ -3559,7 +3559,7 @@ async def outlet_filter_handler(ctx):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error running outlet filters: {e}')
|
log.debug('Error running outlet filters: %s', e)
|
||||||
|
|
||||||
|
|
||||||
async def non_streaming_chat_response_handler(response, ctx):
|
async def non_streaming_chat_response_handler(response, ctx):
|
||||||
@@ -3704,7 +3704,7 @@ async def non_streaming_chat_response_handler(response, ctx):
|
|||||||
|
|
||||||
response = build_response_object(response, merge_events_into_response(response_data, events))
|
response = build_response_object(response, merge_events_into_response(response_data, events))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Error occurred while processing request: {e}')
|
log.debug('Error occurred while processing request: %s', e)
|
||||||
chat_id = metadata.get('chat_id')
|
chat_id = metadata.get('chat_id')
|
||||||
if getattr(request.state, 'internal', False) is not True and chat_id and is_saved_chat_id(chat_id):
|
if getattr(request.state, 'internal', False) is not True and chat_id and is_saved_chat_id(chat_id):
|
||||||
webui_url = await Config.get('webui.url')
|
webui_url = await Config.get('webui.url')
|
||||||
@@ -4815,7 +4815,7 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
if done:
|
if done:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
log.debug(f'Error: {e}')
|
log.debug('Error: %s', e)
|
||||||
continue
|
continue
|
||||||
await flush_pending_delta_data()
|
await flush_pending_delta_data()
|
||||||
|
|
||||||
@@ -5336,7 +5336,7 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
)
|
)
|
||||||
|
|
||||||
retries += 1
|
retries += 1
|
||||||
log.debug(f'Attempt count: {retries}')
|
log.debug('Attempt count: %s', retries)
|
||||||
|
|
||||||
ci_item = output[-1]
|
ci_item = output[-1]
|
||||||
ci_output = ''
|
ci_output = ''
|
||||||
@@ -5398,7 +5398,7 @@ async def streaming_chat_response_handler(response, ctx):
|
|||||||
else:
|
else:
|
||||||
ci_output = {'stdout': 'Code interpreter engine not configured.'}
|
ci_output = {'stdout': 'Code interpreter engine not configured.'}
|
||||||
|
|
||||||
log.debug(f'Code interpreter output: {ci_output}')
|
log.debug('Code interpreter output: %s', ci_output)
|
||||||
|
|
||||||
# Handle error responses from event_caller
|
# Handle error responses from event_caller
|
||||||
# (e.g. session disconnected, timeout)
|
# (e.g. session disconnected, timeout)
|
||||||
|
|||||||
@@ -307,7 +307,7 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
|
|||||||
try:
|
try:
|
||||||
await get_function_module_from_cache(request, function_id, function=function)
|
await get_function_module_from_cache(request, function_id, function=function)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to load function module for {function_id}: {e}')
|
log.debug('Failed to load function module for %s: %s', function_id, e)
|
||||||
|
|
||||||
# Apply global model defaults to all models
|
# Apply global model defaults to all models
|
||||||
# Per-model overrides take precedence over global defaults
|
# Per-model overrides take precedence over global defaults
|
||||||
@@ -413,7 +413,7 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
|
|||||||
filter_items_by_id[filter_id] = items
|
filter_items_by_id[filter_id] = items
|
||||||
model['filters'].extend({**item} for item in items)
|
model['filters'].extend({**item} for item in items)
|
||||||
|
|
||||||
log.debug(f'get_all_models() returned {len(models)} models')
|
log.debug('get_all_models() returned %s models', len(models))
|
||||||
|
|
||||||
models_dict = {model['id']: model for model in models}
|
models_dict = {model['id']: model for model in models}
|
||||||
if isinstance(request.app.state.MODELS, RedisDict):
|
if isinstance(request.app.state.MODELS, RedisDict):
|
||||||
|
|||||||
@@ -241,7 +241,7 @@ def _normalize_token_expiry(token: dict) -> dict:
|
|||||||
if exp is not None:
|
if exp is not None:
|
||||||
expires_at = min(expires_at, int(exp))
|
expires_at = min(expires_at, int(exp))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Could not read exp from id_token: {e}')
|
log.debug('Could not read exp from id_token: %s', e)
|
||||||
|
|
||||||
token['expires_at'] = expires_at
|
token['expires_at'] = expires_at
|
||||||
return token
|
return token
|
||||||
@@ -411,7 +411,7 @@ async def get_protected_resource_metadata(server_url: str) -> ProtectedResourceM
|
|||||||
)
|
)
|
||||||
if match:
|
if match:
|
||||||
resource_metadata_urls = [match.group(1) or match.group(2)]
|
resource_metadata_urls = [match.group(1) or match.group(2)]
|
||||||
log.debug(f'Found resource_metadata URL: {resource_metadata_urls[0]}')
|
log.debug('Found resource_metadata URL: %s', resource_metadata_urls[0])
|
||||||
else:
|
else:
|
||||||
# Fall back to well-known resource metadata URIs (RFC 9728 §4.2)
|
# Fall back to well-known resource metadata URIs (RFC 9728 §4.2)
|
||||||
parsed, base_url = get_parsed_and_base_url(server_url)
|
parsed, base_url = get_parsed_and_base_url(server_url)
|
||||||
@@ -423,7 +423,7 @@ async def get_protected_resource_metadata(server_url: str) -> ProtectedResourceM
|
|||||||
resource_metadata_urls.append(
|
resource_metadata_urls.append(
|
||||||
urllib.parse.urljoin(base_url, '/.well-known/oauth-protected-resource')
|
urllib.parse.urljoin(base_url, '/.well-known/oauth-protected-resource')
|
||||||
)
|
)
|
||||||
log.debug(f'No resource_metadata in header, trying well-known URIs: {resource_metadata_urls}')
|
log.debug('No resource_metadata in header, trying well-known URIs: %s', resource_metadata_urls)
|
||||||
|
|
||||||
# Fetch Protected Resource metadata from candidate URLs
|
# Fetch Protected Resource metadata from candidate URLs
|
||||||
for resource_metadata_url in resource_metadata_urls:
|
for resource_metadata_url in resource_metadata_urls:
|
||||||
@@ -436,22 +436,22 @@ async def get_protected_resource_metadata(server_url: str) -> ProtectedResourceM
|
|||||||
|
|
||||||
resource = resource_metadata.get('resource') or None
|
resource = resource_metadata.get('resource') or None
|
||||||
if resource:
|
if resource:
|
||||||
log.debug(f'Discovered resource indicator: {resource}')
|
log.debug('Discovered resource indicator: %s', resource)
|
||||||
|
|
||||||
servers = resource_metadata.get('authorization_servers', [])
|
servers = resource_metadata.get('authorization_servers', [])
|
||||||
scopes = resource_metadata.get('scopes_supported', [])
|
scopes = resource_metadata.get('scopes_supported', [])
|
||||||
if scopes:
|
if scopes:
|
||||||
log.debug(f'Discovered resource scopes: {scopes}')
|
log.debug('Discovered resource scopes: %s', scopes)
|
||||||
|
|
||||||
if servers:
|
if servers:
|
||||||
authorization_servers = servers
|
authorization_servers = servers
|
||||||
log.debug(f'Discovered authorization servers: {servers}')
|
log.debug('Discovered authorization servers: %s', servers)
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to fetch resource metadata from {resource_metadata_url}: {e}')
|
log.debug('Failed to fetch resource metadata from %s: %s', resource_metadata_url, e)
|
||||||
continue
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'MCP Protected Resource discovery failed: {e}')
|
log.debug('MCP Protected Resource discovery failed: %s', e)
|
||||||
|
|
||||||
return ProtectedResourceMetadata(
|
return ProtectedResourceMetadata(
|
||||||
resource=resource, authorization_servers=authorization_servers, scopes_supported=scopes
|
resource=resource, authorization_servers=authorization_servers, scopes_supported=scopes
|
||||||
@@ -801,7 +801,7 @@ async def recover_static_oauth_client_metadata(connection: dict, oauth_client_in
|
|||||||
try:
|
try:
|
||||||
resource_metadata = await get_protected_resource_metadata(server_url)
|
resource_metadata = await get_protected_resource_metadata(server_url)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Unable to recover static OAuth metadata for {server_url}: {e}')
|
log.debug('Unable to recover static OAuth metadata for %s: %s', server_url, e)
|
||||||
return oauth_client_info
|
return oauth_client_info
|
||||||
|
|
||||||
recovered = {**oauth_client_info}
|
recovered = {**oauth_client_info}
|
||||||
@@ -946,9 +946,7 @@ class OAuthClientManager:
|
|||||||
if not authorization_url:
|
if not authorization_url:
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(
|
log.debug('Skipping OAuth preflight for client %s: %s', client_info.client_id, e)
|
||||||
f'Skipping OAuth preflight for client {client_info.client_id}: {e}',
|
|
||||||
)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -994,7 +992,7 @@ class OAuthClientManager:
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Skipping OAuth preflight network check for client {client_info.client_id}: {e}')
|
log.debug('Skipping OAuth preflight network check for client %s: %s', client_info.client_id, e)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -1043,7 +1041,7 @@ class OAuthClientManager:
|
|||||||
or session.expires_at is None
|
or session.expires_at is None
|
||||||
or datetime.now() + timedelta(minutes=5) >= datetime.fromtimestamp(session.expires_at)
|
or datetime.now() + timedelta(minutes=5) >= datetime.fromtimestamp(session.expires_at)
|
||||||
):
|
):
|
||||||
log.debug(f'Token refresh needed for user {user_id}, client_id {session.provider}')
|
log.debug('Token refresh needed for user %s, client_id %s', user_id, session.provider)
|
||||||
refreshed_token = await self._refresh_token(session)
|
refreshed_token = await self._refresh_token(session)
|
||||||
if refreshed_token:
|
if refreshed_token:
|
||||||
return refreshed_token
|
return refreshed_token
|
||||||
@@ -1160,7 +1158,7 @@ class OAuthClientManager:
|
|||||||
|
|
||||||
_normalize_token_expiry(new_token_data)
|
_normalize_token_expiry(new_token_data)
|
||||||
|
|
||||||
log.debug(f'Token refresh successful for client_id {client_id}')
|
log.debug('Token refresh successful for client_id %s', client_id)
|
||||||
return new_token_data
|
return new_token_data
|
||||||
else:
|
else:
|
||||||
error_text = await r.text()
|
error_text = await r.text()
|
||||||
@@ -1326,8 +1324,9 @@ class OAuthManager:
|
|||||||
# the session (#24618).
|
# the session (#24618).
|
||||||
if (session.provider or '').startswith('mcp:'):
|
if (session.provider or '').startswith('mcp:'):
|
||||||
log.debug(
|
log.debug(
|
||||||
f'Skipping MCP session {session.id} (provider={session.provider}) '
|
'Skipping MCP session %s (provider=%s) in SSO OAuthManager — handled by oauth_client_manager',
|
||||||
f'in SSO OAuthManager — handled by oauth_client_manager'
|
session.id,
|
||||||
|
session.provider,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -1336,7 +1335,7 @@ class OAuthManager:
|
|||||||
or session.expires_at is None
|
or session.expires_at is None
|
||||||
or datetime.now() + timedelta(minutes=5) >= datetime.fromtimestamp(session.expires_at)
|
or datetime.now() + timedelta(minutes=5) >= datetime.fromtimestamp(session.expires_at)
|
||||||
):
|
):
|
||||||
log.debug(f'Token refresh needed for user {user_id}, provider {session.provider}')
|
log.debug('Token refresh needed for user %s, provider %s', user_id, session.provider)
|
||||||
refreshed_token = await self._refresh_token(session)
|
refreshed_token = await self._refresh_token(session)
|
||||||
if refreshed_token:
|
if refreshed_token:
|
||||||
return refreshed_token
|
return refreshed_token
|
||||||
@@ -1452,7 +1451,7 @@ class OAuthManager:
|
|||||||
|
|
||||||
_normalize_token_expiry(new_token_data)
|
_normalize_token_expiry(new_token_data)
|
||||||
|
|
||||||
log.debug(f'Token refresh successful for provider {provider}')
|
log.debug('Token refresh successful for provider %s', provider)
|
||||||
return new_token_data
|
return new_token_data
|
||||||
else:
|
else:
|
||||||
error_text = await r.text()
|
error_text = await r.text()
|
||||||
@@ -1511,10 +1510,10 @@ class OAuthManager:
|
|||||||
elif isinstance(claim_data, int):
|
elif isinstance(claim_data, int):
|
||||||
oauth_roles = [str(claim_data)]
|
oauth_roles = [str(claim_data)]
|
||||||
|
|
||||||
log.debug(f'Oauth Roles claim: {oauth_claim}')
|
log.debug('Oauth Roles claim: %s', oauth_claim)
|
||||||
log.debug(f'User roles from oauth: {oauth_roles}')
|
log.debug('User roles from oauth: %s', oauth_roles)
|
||||||
log.debug(f'Accepted user roles: {oauth_allowed_roles}')
|
log.debug('Accepted user roles: %s', oauth_allowed_roles)
|
||||||
log.debug(f'Accepted admin roles: {oauth_admin_roles}')
|
log.debug('Accepted admin roles: %s', oauth_admin_roles)
|
||||||
|
|
||||||
# If roles are present in the token, they must match; otherwise deny access
|
# If roles are present in the token, they must match; otherwise deny access
|
||||||
if oauth_roles:
|
if oauth_roles:
|
||||||
@@ -1591,7 +1590,7 @@ class OAuthManager:
|
|||||||
# Determine creator ID: Prefer admin, fallback to current user if no admin exists
|
# Determine creator ID: Prefer admin, fallback to current user if no admin exists
|
||||||
admin_user = await Users.get_super_admin_user()
|
admin_user = await Users.get_super_admin_user()
|
||||||
creator_id = admin_user.id if admin_user else user.id
|
creator_id = admin_user.id if admin_user else user.id
|
||||||
log.debug(f'Using creator ID {creator_id} for potential group creation.')
|
log.debug('Using creator ID %s for potential group creation.', creator_id)
|
||||||
|
|
||||||
for group_name in user_oauth_groups:
|
for group_name in user_oauth_groups:
|
||||||
if group_name not in all_group_names:
|
if group_name not in all_group_names:
|
||||||
@@ -1622,10 +1621,10 @@ class OAuthManager:
|
|||||||
all_available_groups = await Groups.get_all_groups(db=db)
|
all_available_groups = await Groups.get_all_groups(db=db)
|
||||||
log.debug('Refreshed list of all available groups after creation.')
|
log.debug('Refreshed list of all available groups after creation.')
|
||||||
|
|
||||||
log.debug(f'Oauth Groups claim: {oauth_claim}')
|
log.debug('Oauth Groups claim: %s', oauth_claim)
|
||||||
log.debug(f'User oauth groups: {user_oauth_groups}')
|
log.debug('User oauth groups: %s', user_oauth_groups)
|
||||||
log.debug(f"User's current groups: {[g.name for g in user_current_groups]}")
|
log.debug("User's current groups: %s", [g.name for g in user_current_groups])
|
||||||
log.debug(f'All groups available in OpenWebUI: {[g.name for g in all_available_groups]}')
|
log.debug('All groups available in OpenWebUI: %s', [g.name for g in all_available_groups])
|
||||||
|
|
||||||
# Remove groups that user is no longer a part of
|
# Remove groups that user is no longer a part of
|
||||||
for group_model in user_current_groups:
|
for group_model in user_current_groups:
|
||||||
@@ -1635,7 +1634,7 @@ class OAuthManager:
|
|||||||
and not is_in_blocked_groups(group_model.name, blocked_groups)
|
and not is_in_blocked_groups(group_model.name, blocked_groups)
|
||||||
):
|
):
|
||||||
# Remove group from user
|
# Remove group from user
|
||||||
log.debug(f'Removing user from group {group_model.name} as it is no longer in their oauth groups')
|
log.debug('Removing user from group %s as it is no longer in their oauth groups', group_model.name)
|
||||||
await Groups.remove_users_from_group(group_model.id, [user.id], db=db)
|
await Groups.remove_users_from_group(group_model.id, [user.id], db=db)
|
||||||
|
|
||||||
# In case a group is created, but perms are never assigned to the group by hitting "save"
|
# In case a group is created, but perms are never assigned to the group by hitting "save"
|
||||||
@@ -1663,7 +1662,7 @@ class OAuthManager:
|
|||||||
and not is_in_blocked_groups(group_model.name, blocked_groups)
|
and not is_in_blocked_groups(group_model.name, blocked_groups)
|
||||||
):
|
):
|
||||||
# Add user to group
|
# Add user to group
|
||||||
log.debug(f'Adding user to group {group_model.name} as it was found in their oauth groups')
|
log.debug('Adding user to group %s as it was found in their oauth groups', group_model.name)
|
||||||
|
|
||||||
await Groups.add_users_to_group(group_model.id, [user.id], db=db)
|
await Groups.add_users_to_group(group_model.id, [user.id], db=db)
|
||||||
|
|
||||||
@@ -1916,7 +1915,7 @@ class OAuthManager:
|
|||||||
if new_name and new_name != user.name:
|
if new_name and new_name != user.name:
|
||||||
await Users.update_user_by_id(user.id, {'name': new_name}, db=db)
|
await Users.update_user_by_id(user.id, {'name': new_name}, db=db)
|
||||||
user.name = new_name
|
user.name = new_name
|
||||||
log.debug(f'Updated name for user {user.email}')
|
log.debug('Updated name for user %s', user.email)
|
||||||
|
|
||||||
if auth_config.OAUTH_UPDATE_EMAIL_ON_LOGIN:
|
if auth_config.OAUTH_UPDATE_EMAIL_ON_LOGIN:
|
||||||
email_claim = auth_config.OAUTH_EMAIL_CLAIM
|
email_claim = auth_config.OAUTH_EMAIL_CLAIM
|
||||||
@@ -1931,7 +1930,7 @@ class OAuthManager:
|
|||||||
else:
|
else:
|
||||||
await Auths.update_email_by_id(user.id, new_email.lower(), db=db)
|
await Auths.update_email_by_id(user.id, new_email.lower(), db=db)
|
||||||
user.email = new_email.lower()
|
user.email = new_email.lower()
|
||||||
log.debug(f'Updated email for user {user.id}')
|
log.debug('Updated email for user %s', user.id)
|
||||||
|
|
||||||
# Update profile picture if enabled and different from current
|
# Update profile picture if enabled and different from current
|
||||||
if auth_config.OAUTH_UPDATE_PICTURE_ON_LOGIN:
|
if auth_config.OAUTH_UPDATE_PICTURE_ON_LOGIN:
|
||||||
@@ -1946,7 +1945,7 @@ class OAuthManager:
|
|||||||
)
|
)
|
||||||
if processed_picture_url != user.profile_image_url:
|
if processed_picture_url != user.profile_image_url:
|
||||||
await Users.update_user_profile_image_url_by_id(user.id, processed_picture_url, db=db)
|
await Users.update_user_profile_image_url_by_id(user.id, processed_picture_url, db=db)
|
||||||
log.debug(f'Updated profile picture for user {user.email}')
|
log.debug('Updated profile picture for user %s', user.email)
|
||||||
else:
|
else:
|
||||||
# If the user does not exist, check if signups are enabled
|
# If the user does not exist, check if signups are enabled
|
||||||
if auth_config.ENABLE_OAUTH_SIGNUP:
|
if auth_config.ENABLE_OAUTH_SIGNUP:
|
||||||
@@ -2172,7 +2171,7 @@ class OAuthManager:
|
|||||||
matched_issuer = provider_issuer
|
matched_issuer = provider_issuer
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Back-channel logout: error checking provider {provider_name}: {e}')
|
log.debug('Back-channel logout: error checking provider %s: %s', provider_name, e)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not matched_provider or not matched_client_id or not matched_jwks_uri:
|
if not matched_provider or not matched_client_id or not matched_jwks_uri:
|
||||||
@@ -2249,10 +2248,12 @@ class OAuthManager:
|
|||||||
users_to_logout.append(user)
|
users_to_logout.append(user)
|
||||||
|
|
||||||
if not users_to_logout and sid:
|
if not users_to_logout and sid:
|
||||||
log.debug(f'Back-channel logout: no user found by sub, sid-based lookup not yet supported (sid={sid})')
|
log.debug('Back-channel logout: no user found by sub, sid-based lookup not yet supported (sid=%s)', sid)
|
||||||
|
|
||||||
if not users_to_logout:
|
if not users_to_logout:
|
||||||
log.debug(f'Back-channel logout: no matching user for provider={matched_provider}, sub={sub}, sid={sid}')
|
log.debug(
|
||||||
|
'Back-channel logout: no matching user for provider=%s, sub=%s, sid=%s', matched_provider, sub, sid
|
||||||
|
)
|
||||||
return JSONResponse(status_code=200, content={})
|
return JSONResponse(status_code=200, content={})
|
||||||
|
|
||||||
# 9. Revoke tokens and delete sessions
|
# 9. Revoke tokens and delete sessions
|
||||||
|
|||||||
@@ -1187,7 +1187,7 @@ async def get_terminal_cwd(
|
|||||||
data = await resp.json()
|
data = await resp.json()
|
||||||
return data.get('cwd')
|
return data.get('cwd')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to fetch terminal CWD: {e}')
|
log.debug('Failed to fetch terminal CWD: %s', e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -1226,7 +1226,7 @@ async def get_terminal_system_prompt(
|
|||||||
data = await resp.json()
|
data = await resp.json()
|
||||||
return data.get('prompt')
|
return data.get('prompt')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug(f'Failed to fetch terminal system prompt: {e}')
|
log.debug('Failed to fetch terminal system prompt: %s', e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -1447,7 +1447,7 @@ async def get_tool_server_data(url: str, headers: dict | None) -> dict[str, Any]
|
|||||||
error = str(err)
|
error = str(err)
|
||||||
raise Exception(error)
|
raise Exception(error)
|
||||||
|
|
||||||
log.debug(f'Fetched data: {res}')
|
log.debug('Fetched data: %s', res)
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ def _event_text(message: str, description: str | None = None, event_data: dict |
|
|||||||
|
|
||||||
async def post_webhook(name: str, url: str, message: str, event_data: dict, description: str | None = None) -> bool:
|
async def post_webhook(name: str, url: str, message: str, event_data: dict, description: str | None = None) -> bool:
|
||||||
try:
|
try:
|
||||||
log.debug(f'post_webhook: {url}, {message}, {event_data}')
|
log.debug('post_webhook: %s, %s, %s', url, message, event_data)
|
||||||
# Block private-IP / loopback / cloud-metadata targets — the URL is
|
# Block private-IP / loopback / cloud-metadata targets — the URL is
|
||||||
# caller-controlled (user notification settings under
|
# caller-controlled (user notification settings under
|
||||||
# ENABLE_USER_WEBHOOKS, automation notification triggers).
|
# ENABLE_USER_WEBHOOKS, automation notification triggers).
|
||||||
@@ -83,7 +83,7 @@ async def post_webhook(name: str, url: str, message: str, event_data: dict, desc
|
|||||||
else:
|
else:
|
||||||
payload = event_data
|
payload = event_data
|
||||||
|
|
||||||
log.debug(f'payload: {payload}')
|
log.debug('payload: %s', payload)
|
||||||
async with get_ssrf_safe_session() as session:
|
async with get_ssrf_safe_session() as session:
|
||||||
async with session.post(
|
async with session.post(
|
||||||
url,
|
url,
|
||||||
@@ -93,7 +93,7 @@ async def post_webhook(name: str, url: str, message: str, event_data: dict, desc
|
|||||||
) as r:
|
) as r:
|
||||||
r_text = await r.text()
|
r_text = await r.text()
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
log.debug(f'r.text: {r_text}')
|
log.debug('r.text: %s', r_text)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
Reference in new issue
Block a user