refac

backend/open_webui/main.py

@@ -1524,6 +1524,106 @@ async def get_base_models(request: Request, user=Depends(get_admin_user)):
     return {'data': models}
 
 
+class ModelUnloadForm(BaseModel):
+    model: str
+
+
+@app.post('/api/models/unload')
+async def unload_model(request: Request, form_data: ModelUnloadForm, user=Depends(get_admin_user)):
+    """
+    Unified model unload endpoint.
+    Resolves the provider that owns the model and calls its native unload mechanism.
+    Supports: Ollama (keep_alive=0) and llama.cpp (/models/unload).
+    """
+    model_id = form_data.model
+
+    # --- Ollama provider ---
+    ollama_models = getattr(request.app.state, 'OLLAMA_MODELS', None) or {}
+    if model_id in ollama_models:
+        url_indices = ollama_models[model_id].get('urls', [])
+        errors = []
+        for idx in url_indices:
+            url = request.app.state.config.OLLAMA_BASE_URLS[idx]
+            api_config = request.app.state.config.OLLAMA_API_CONFIGS.get(
+                str(idx),
+                request.app.state.config.OLLAMA_API_CONFIGS.get(url, {}),
+            )
+            key = api_config.get('key', None)
+
+            prefix_id = api_config.get('prefix_id', None)
+            actual_model = model_id
+            if prefix_id and actual_model.startswith(f'{prefix_id}.'):
+                actual_model = actual_model[len(f'{prefix_id}.'):]
+
+            payload = json.dumps({'model': actual_model, 'keep_alive': 0, 'prompt': ''})
+
+            try:
+                timeout = aiohttp.ClientTimeout(total=30)
+                async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
+                    headers = {
+                        'Content-Type': 'application/json',
+                        **({"Authorization": f"Bearer {key}"} if key else {}),
+                    }
+                    async with session.post(
+                        f'{url}/api/generate',
+                        data=payload,
+                        headers=headers,
+                    ) as r:
+                        if not r.ok:
+                            errors.append({'url_idx': idx, 'error': await r.text()})
+            except Exception as e:
+                log.exception(f'Failed to unload model on Ollama node {idx}: {e}')
+                errors.append({'url_idx': idx, 'error': str(e)})
+
+        if errors:
+            raise HTTPException(
+                status_code=500,
+                detail=f'Failed to unload model on {len(errors)} node(s): {errors}',
+            )
+        return {'status': True}
+
+    # --- OpenAI-compatible providers ---
+    openai_models = getattr(request.app.state, 'OPENAI_MODELS', None) or {}
+    if model_id in openai_models:
+        model_info = openai_models[model_id]
+        idx = model_info.get('urlIdx')
+        api_config = request.app.state.config.OPENAI_API_CONFIGS.get(str(idx), {})
+        provider = api_config.get('provider', '')
+        base_url = request.app.state.config.OPENAI_API_BASE_URLS[idx]
+        key = request.app.state.config.OPENAI_API_KEYS[idx] if idx < len(request.app.state.config.OPENAI_API_KEYS) else ''
+
+        if provider == 'llama.cpp':
+            root_url = base_url.rstrip('/').removesuffix('/v1')
+            try:
+                timeout = aiohttp.ClientTimeout(total=30)
+                async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
+                    headers = {
+                        'Content-Type': 'application/json',
+                        **({"Authorization": f"Bearer {key}"} if key else {}),
+                    }
+                    async with session.post(
+                        f'{root_url}/models/unload',
+                        json={'model': model_id},
+                        headers=headers,
+                    ) as r:
+                        if not r.ok:
+                            detail = await r.text()
+                            raise HTTPException(status_code=r.status, detail=detail)
+                        return await r.json()
+            except HTTPException:
+                raise
+            except Exception as e:
+                log.exception(f'Failed to unload model via llama.cpp: {e}')
+                raise HTTPException(status_code=500, detail=str(e))
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=f'Provider "{provider or "default"}" does not support model unloading',
+            )
+
+    raise HTTPException(status_code=404, detail=f'Model "{model_id}" not found')
+
+
 ##################################
 # Embeddings
 ##################################

backend/open_webui/routers/openai.py

@@ -439,6 +439,7 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list:
             connection_type = api_config.get('connection_type', 'external')
             prefix_id = api_config.get('prefix_id', None)
             tags = api_config.get('tags', [])
+            provider = api_config.get('provider', '')
 
             model_list = response if isinstance(response, list) else response.get('data', [])
             if not isinstance(model_list, list):
@@ -459,6 +460,9 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list:
                 if connection_type:
                     model['connection_type'] = connection_type
 
+                if provider:
+                    model['provider'] = provider
+
     log.debug(f'get_all_models:responses() {responses}')
     return responses
 
@@ -488,6 +492,41 @@ async def get_filtered_models(models, user, db=None):
     return filtered_models
 
 
+async def get_openai_loaded_models(request: Request, models: dict, api_base_urls: list):
+    """
+    Fetch loaded-model state from providers that expose it and annotate
+    each model dict with a ``loaded`` boolean.
+
+    Currently supports:
+      - **llama.cpp** – queries ``GET /slots`` and matches slot model IDs.
+    """
+    api_configs = request.app.state.config.OPENAI_API_CONFIGS
+    api_keys = request.app.state.config.OPENAI_API_KEYS
+
+    for idx, url in enumerate(api_base_urls):
+        api_config = api_configs.get(
+            str(idx),
+            api_configs.get(url, {}),
+        )
+        provider = api_config.get('provider', '')
+
+        if provider == 'llama.cpp':
+            try:
+                root_url = url.rstrip('/').removesuffix('/v1')
+                key = api_keys[idx] if idx < len(api_keys) else None
+                slots = await send_get_request(url=f'{root_url}/slots', key=key)
+                loaded_model_ids = (
+                    {s.get('model') for s in slots if s.get('model')}
+                    if isinstance(slots, list)
+                    else set()
+                )
+                for model_id, model in models.items():
+                    if model.get('urlIdx') == idx:
+                        model['loaded'] = model_id in loaded_model_ids
+            except Exception as e:
+                log.debug(f'Failed to fetch llama.cpp slots for idx {idx}: {e}')
+
+
 @cached(
     ttl=MODELS_CACHE_TTL,
     key=lambda _, user: f'openai_all_models_{user.id}' if user else 'openai_all_models',
@@ -548,6 +587,7 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
                             'owned_by': 'openai',
                             'openai': model,
                             'connection_type': model.get('connection_type', 'external'),
+                            'provider': model.get('provider', ''),
                             'urlIdx': idx,
                         }
 
@@ -556,6 +596,9 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
     models = get_merged_models(map(extract_data, responses))
     log.debug(f'models: {models}')
 
+    # Fetch loaded state for providers that support it (e.g. llama.cpp /slots)
+    await get_openai_loaded_models(request, models, api_base_urls)
+
     request.app.state.OPENAI_MODELS = models
     return {'data': list(models.values())}
 

backend/open_webui/utils/models.py

@@ -47,6 +47,7 @@ async def fetch_ollama_models(request: Request, user: UserModel = None):
             'created': int(time.time()),
             'owned_by': 'ollama',
             'ollama': model,
+            'loaded': 'expires_at' in model,
             'connection_type': model.get('connection_type', 'local'),
             'tags': model.get('tags', []),
         }
@@ -199,6 +200,8 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
                 'connection_type': connection_type,
                 'preset': True,
                 **({'pipe': pipe} if pipe is not None else {}),
+                **({'provider': base_model.get('provider')} if base_model and base_model.get('provider') else {}),
+                **({'loaded': base_model.get('loaded')} if base_model and base_model.get('loaded') is not None else {}),
             }
 
             info = custom_model.model_dump()

src/lib/apis/index.ts

@@ -159,6 +159,39 @@ export const getModels = async (
 	return models;
 };
 
+export const unloadModel = async (token: string, model: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_BASE_URL}/api/models/unload`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			...(token && { authorization: `Bearer ${token}` })
+		},
+		body: JSON.stringify({ model })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.error(err);
+			if ('detail' in err) {
+				error = err.detail;
+			} else {
+				error = err;
+			}
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 type ChatCompletedForm = {
 	model: string;
 	messages: Record<string, unknown>[];

src/lib/components/AddConnectionModal.svelte

@@ -36,9 +36,10 @@
 	let auth_type = 'bearer';
 
 	let connectionType = 'external';
-	let azure = false;
+	let provider = '';
 	$: azure =
-		(url.includes('azure.') || url.includes('cognitive.microsoft.com')) && !direct ? true : false;
+		provider === 'azure' ||
+		((url.includes('azure.') || url.includes('cognitive.microsoft.com')) && !direct && provider === '');
 
 	let prefixId = '';
 	let enable = true;
@@ -98,7 +99,7 @@
 				key,
 				config: {
 					auth_type,
-					azure: azure,
+					...(provider ? { provider } : azure ? { azure: true } : {}),
 					api_version: apiVersion,
 					...(_headers ? { headers: _headers } : {})
 				}
@@ -186,7 +187,8 @@
 				connection_type: connectionType,
 				auth_type,
 				headers: headers ? JSON.parse(headers) : undefined,
-				...(!ollama && azure ? { azure: true, api_version: apiVersion } : {}),
+				...(provider ? { provider } : !ollama && azure ? { azure: true } : {}),
+				...(azure ? { api_version: apiVersion } : {}),
 				...(apiType ? { api_type: apiType } : {})
 			}
 		};
@@ -223,7 +225,7 @@
 				connectionType = connection.config?.connection_type ?? 'local';
 			} else {
 				connectionType = connection.config?.connection_type ?? 'external';
-				azure = connection.config?.azure ?? false;
+				provider = connection.config?.provider ?? (connection.config?.azure ? 'azure' : '');
 				apiVersion = connection.config?.api_version ?? '';
 				apiType = connection.config?.api_type ?? '';
 			}
@@ -491,22 +493,22 @@
 						{#if !ollama && !direct}
 							<div class="flex flex-row justify-between items-center w-full mt-2">
 								<label
-									for="prefix-id-input"
+									for="provider-select"
 									class={`mb-0.5 text-xs text-gray-500
 								${($settings?.highContrastMode ?? false) ? 'text-gray-800 dark:text-gray-100' : ''}`}
-									>{$i18n.t('Provider Type')}</label
+									>{$i18n.t('Provider')}</label
 								>
 
 								<div>
-									<button
+									<select
-										on:click={() => {
+										id="provider-select"
-											azure = !azure;
+										bind:value={provider}
-										}}
+										class="text-xs text-gray-700 dark:text-gray-300 bg-transparent outline-hidden"
-										type="button"
-										class=" text-xs text-gray-700 dark:text-gray-300"
 									>
-										{azure ? $i18n.t('Azure OpenAI') : $i18n.t('OpenAI')}
+										<option value="">{$i18n.t('Default')}</option>
-									</button>
+										<option value="azure">{$i18n.t('Azure OpenAI')}</option>
+										<option value="llama.cpp">{$i18n.t('llama.cpp')}</option>
+									</select>
 								</div>
 							</div>
 						{/if}

src/lib/components/chat/ModelSelector/ModelItem.svelte

@@ -120,25 +120,28 @@
 							</Tooltip>
 						</div>
 					{/if}
-					{#if item.model.ollama?.expires_at && new Date(item.model.ollama?.expires_at * 1000) > new Date()}
+				{/if}
-						<div class="flex items-center translate-y-[0.5px] px-0.5">
+
-							<Tooltip
+				{#if item.model.loaded}
-								content={`${$i18n.t('Unloads {{FROM_NOW}}', {
+					<div class="flex items-center translate-y-[0.5px] px-0.5">
+						<Tooltip
+							content={item.model.ollama?.expires_at && new Date(item.model.ollama?.expires_at * 1000) > new Date()
+								? `${$i18n.t('Unloads {{FROM_NOW}}', {
 									FROM_NOW: dayjs(item.model.ollama?.expires_at * 1000).fromNow()
-								})}`}
+								})}`
-								className="self-end"
+								: `${$i18n.t('Loaded')}`}
-							>
+							className="self-end"
-								<div class=" flex items-center">
+						>
-									<span class="relative flex size-2">
+							<div class=" flex items-center">
-										<span
+								<span class="relative flex size-2">
-											class="animate-ping absolute inline-flex h-full w-full rounded-full bg-green-400 opacity-75"
+									<span
-										/>
+										class="animate-ping absolute inline-flex h-full w-full rounded-full bg-green-400 opacity-75"
-										<span class="relative inline-flex rounded-full size-2 bg-green-500" />
+									/>
-									</span>
+									<span class="relative inline-flex rounded-full size-2 bg-green-500" />
-								</div>
+								</span>
-							</Tooltip>
+							</div>
-						</div>
+						</Tooltip>
-					{/if}
+					</div>
 				{/if}
 
 				<!-- {JSON.stringify(item.info)} -->
@@ -233,7 +236,7 @@
 	</div>
 
 	<div class="ml-auto pl-2 pr-1 flex items-center gap-1.5 shrink-0">
-		{#if $user?.role === 'admin' && item.model.owned_by === 'ollama' && item.model.ollama?.expires_at && new Date(item.model.ollama?.expires_at * 1000) > new Date()}
+	{#if $user?.role === 'admin' && item.model.loaded}
 			<Tooltip
 				content={`${$i18n.t('Eject')}`}
 				className="flex-shrink-0 group-hover/item:opacity-100 opacity-0 "

src/lib/components/chat/ModelSelector/Selector.svelte

@@ -13,7 +13,8 @@
 	import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
 	import { goto } from '$app/navigation';
 
-	import { deleteModel, getOllamaVersion, pullModel, unloadModel } from '$lib/apis/ollama';
+	import { deleteModel, getOllamaVersion, pullModel } from '$lib/apis/ollama';
+	import { unloadModel } from '$lib/apis';
 
 	import {
 		user,

src/routes/+layout.svelte

@@ -824,7 +824,8 @@
 				if (event.data.action === 'add') {
 					await addOpenAIConnection(token, {
 						url: event.data.url,
-						key: event.data.key
+						key: event.data.key,
+						config: event.data.config
 					});
 				} else if (event.data.action === 'remove') {
 					await removeOpenAIConnection(token, event.data.url);