diff --git a/backend/open_webui/events.py b/backend/open_webui/events.py index 1e87d912bd..71f9f15f1b 100644 --- a/backend/open_webui/events.py +++ b/backend/open_webui/events.py @@ -442,6 +442,11 @@ class EventDefinitions(BaseModel): description='Model provider configuration was updated.', message='Model Provider Config updated', ) + MODEL_PROVIDER_REQUEST_FAILED: EventDefinition = EventDefinition( + name='model.provider_request.failed', + description='A model provider request failed.', + message='Model provider request failed', + ) MODEL_PROVIDER_MODEL_CREATED: EventDefinition = EventDefinition( name='model.provider_model.created', description='A provider model was created.', @@ -1108,3 +1113,60 @@ async def publish_event( await sink.handle_event(app, event_payload, request=request) except Exception: log.exception('Event sink failed for %s', event_payload.event) + + +async def publish_model_provider_request_failed( + request_or_app: Any, + *, + actor: Any | None, + provider: str, + base_url: str, + status: int, + requested_model: str | None = None, + api_key: str | None = None, + upstream_error: Any = None, +) -> None: + error = upstream_error.get('error') if isinstance(upstream_error, dict) else upstream_error + error_code = None + if isinstance(error, dict): + error_code = error.get('code') or error.get('type') or error.get('error_code') + error = error.get('message') or error.get('detail') or error + + error_text = str(error or '') + marker = f'{error_code or ""} {error_text}'.lower() + error_type = ( + 'model_not_found' + if status == 404 + and any(value in marker for value in ('model_not_found', 'model not found', 'does not exist', 'no such model')) + else 'authentication_failed' + if status in (401, 403) + else 'rate_limited' + if status == 429 + else 'server_failed' + if status >= 500 + else 'upstream_error' + ) + + data = { + 'error_type': error_type, + 'status': status, + 'provider': provider, + 'base_url': base_url, + } + if requested_model: + data['requested_model'] = requested_model + if api_key: + data['api_key_suffix'] = f'...{api_key[-4:]}' + if error_code: + data['upstream_error_code'] = error_code + if error: + data['upstream_message'] = error + + await publish_event( + request_or_app, + EVENTS.MODEL_PROVIDER_REQUEST_FAILED, + actor=actor, + subject_id=requested_model, + subject_type='model', + data=data, + ) diff --git a/backend/open_webui/routers/ollama.py b/backend/open_webui/routers/ollama.py index b755acf80f..956e310088 100644 --- a/backend/open_webui/routers/ollama.py +++ b/backend/open_webui/routers/ollama.py @@ -20,7 +20,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from open_webui.config import UPLOAD_DIR from open_webui.constants import ERROR_MESSAGES -from open_webui.events import EVENTS, publish_event +from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed from open_webui.env import ( AIOHTTP_CLIENT_SESSION_SSL, AIOHTTP_CLIENT_TIMEOUT, @@ -133,12 +133,27 @@ async def send_request( if not r.ok: try: res = await r.json() + await publish_model_provider_request_failed( + request, + actor=user, + provider='ollama', + base_url=url, + status=r.status, + upstream_error=res, + ) if 'error' in res: raise HTTPException(status_code=r.status, detail=res['error']) except HTTPException: raise except Exception as e: log.error(f'Failed to parse error response: {e}') + await publish_model_provider_request_failed( + request, + actor=user, + provider='ollama', + base_url=url, + status=r.status, + ) raise HTTPException( status_code=r.status, detail=ERROR_MESSAGES.SERVER_CONNECTION_ERROR, diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index f7019fbf3a..66e1932318 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -22,7 +22,7 @@ from open_webui.config import ( CACHE_DIR, ) from open_webui.constants import ERROR_MESSAGES -from open_webui.events import EVENTS, publish_event +from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed from open_webui.env import ( AIOHTTP_CLIENT_SESSION_SSL, AIOHTTP_CLIENT_TIMEOUT, @@ -1236,6 +1236,7 @@ async def generate_chat_completion( request_url = f'{url}/responses' else: request_url = f'{url}/chat/completions' + requested_model = payload.get('model') # For Chat Completions, strip image parts from multimodal tool messages # (Chat Completions doesn't support images in tool content). if not is_responses and 'messages' in payload: @@ -1278,8 +1279,28 @@ async def generate_chat_completion( ) try: error_json = json.loads(error_body) + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=url, + api_key=key, + status=r.status, + requested_model=requested_model, + upstream_error=error_json, + ) return JSONResponse(status_code=r.status, content=error_json) except json.JSONDecodeError: + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=url, + api_key=key, + status=r.status, + requested_model=requested_model, + upstream_error=error_body, + ) return JSONResponse( status_code=r.status, content={'error': {'message': error_body, 'code': r.status}}, @@ -1299,6 +1320,16 @@ async def generate_chat_completion( response = await r.text() if r.status >= 400: + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=url, + api_key=key, + status=r.status, + requested_model=requested_model, + upstream_error=response, + ) if isinstance(response, (dict, list)): return JSONResponse(status_code=r.status, content=response) else: @@ -1372,6 +1403,7 @@ async def embeddings(request: Request, form_data: dict, user): headers['api-version'] = api_version else: embeddings_url = f'{url}/embeddings' + requested_model = form_data.get('model') try: session = await get_session() @@ -1399,6 +1431,16 @@ async def embeddings(request: Request, form_data: dict, user): response_data = await r.text() if r.status >= 400: + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=url, + api_key=key, + status=r.status, + requested_model=requested_model, + upstream_error=response_data, + ) if isinstance(response_data, (dict, list)): return JSONResponse(status_code=r.status, content=response_data) else: @@ -1515,6 +1557,16 @@ async def responses( response_data = await r.text() if r.status >= 400: + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=url, + api_key=key, + status=r.status, + requested_model=payload.get('model'), + upstream_error=response_data, + ) if isinstance(response_data, (dict, list)): return JSONResponse(status_code=r.status, content=response_data) else: @@ -1569,6 +1621,7 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)): idx = models[model_id]['urlIdx'] url, key, api_config = await get_openai_connection(idx) + base_url = url r = None streaming = False @@ -1625,6 +1678,16 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)): response_data = await r.text() if r.status >= 400: + await publish_model_provider_request_failed( + request, + actor=user, + provider='openai-compatible', + base_url=base_url, + api_key=key, + status=r.status, + requested_model=model_id, + upstream_error=response_data, + ) if isinstance(response_data, (dict, list)): return JSONResponse(status_code=r.status, content=response_data) else: