refac

backend/open_webui/routers/audio.py

@@ -1166,6 +1166,11 @@ def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None
     else:
         if is_audio_conversion_required(file_path):
             file_path = convert_audio_to_mp3(file_path)
+            if not file_path:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail='Audio conversion failed. The audio file may be corrupted or empty.',
+                )
 
         try:
             file_path = compress_audio(file_path)

src/lib/components/chat/MessageInput/CallOverlay.svelte

@@ -32,6 +32,7 @@
 	let confirmed = false;
 	let interrupted = false;
 	let assistantSpeaking = false;
+	let muted = false;
 
 	let emoji = null;
 	let camera = false;
@@ -155,6 +156,10 @@
 
 	const transcribeHandler = async (audioBlob) => {
 		// Create a blob from the audio chunks
+		if (!audioBlob || audioBlob.size < 100) {
+			console.log('Audio blob too small or empty, skipping transcription');
+			return;
+		}
 
 		await tick();
 		const file = blobToFile(audioBlob, 'recording.wav');
@@ -236,6 +241,11 @@
 					}
 				});
 			}
+
+			if (audioStream) {
+				// hardware track muting disabled to prevent backend translation errors with malformed WebM files
+			}
+
 			mediaRecorder = new MediaRecorder(audioStream);
 
 			mediaRecorder.onstart = () => {
@@ -310,8 +320,8 @@
 					return;
 				}
 
-				if (assistantSpeaking && !($settings?.voiceInterruption ?? false)) {
+				if (muted || (assistantSpeaking && !($settings?.voiceInterruption ?? false))) {
-					// Mute the audio if the assistant is speaking
+					// Suppress mic input when muted or when assistant is speaking without interruption enabled
 					analyser.maxDecibels = 0;
 					analyser.minDecibels = -1;
 				} else {
@@ -325,6 +335,10 @@
 				// Calculate RMS level from time domain data
 				rmsLevel = calculateRMS(timeDomainData);
 
+				if (muted || (assistantSpeaking && !($settings?.voiceInterruption ?? false))) {
+					rmsLevel = 0;
+				}
+
 				// Check if initial speech/noise has started
 				const hasSound = domainData.some((value) => value > 0);
 				if (hasSound) {
@@ -627,6 +641,43 @@
 		chatStreaming = false;
 	};
 
+	const toggleMute = () => {
+		muted = !muted;
+		if (muted && hasStartedSpeaking) {
+			// Abort the ongoing recording so it doesn't accidentally send a partial sentence
+			hasStartedSpeaking = false;
+			confirmed = false;
+			audioChunks = [];
+			if (mediaRecorder && mediaRecorder.state === 'recording') {
+				mediaRecorder.stop();
+			}
+		}
+	};
+
+	let wasAssistantSpeaking = false;
+	$: {
+		if (assistantSpeaking && !wasAssistantSpeaking) {
+			wasAssistantSpeaking = true;
+		} else if (!assistantSpeaking && wasAssistantSpeaking) {
+			wasAssistantSpeaking = false;
+			// Auto unmute when AI finishes speaking
+			if (muted) {
+				muted = false;
+			}
+		}
+	}
+
+	const handleKeydown = (e: KeyboardEvent) => {
+		// Only handle M key when not typing in an input/textarea
+		if (e.key === 'm' || e.key === 'M') {
+			const target = e.target as HTMLElement;
+			if (target.tagName !== 'INPUT' && target.tagName !== 'TEXTAREA' && !target.isContentEditable) {
+				e.preventDefault();
+				toggleMute();
+			}
+		}
+	};
+
 	onMount(async () => {
 		const setWakeLock = async () => {
 			try {
@@ -664,6 +715,8 @@
 		eventTarget.addEventListener('chat', chatEventHandler);
 		eventTarget.addEventListener('chat:finish', chatFinishHandler);
 
+		document.addEventListener('keydown', handleKeydown);
+
 		return async () => {
 			await stopAllAudio();
 
@@ -673,6 +726,8 @@
 			eventTarget.removeEventListener('chat', chatEventHandler);
 			eventTarget.removeEventListener('chat:finish', chatFinishHandler);
 
+			document.removeEventListener('keydown', handleKeydown);
+
 			audioAbortController.abort();
 			await tick();
 
@@ -692,6 +747,9 @@
 		eventTarget.removeEventListener('chat:start', chatStartHandler);
 		eventTarget.removeEventListener('chat', chatEventHandler);
 		eventTarget.removeEventListener('chat:finish', chatFinishHandler);
+
+		document.removeEventListener('keydown', handleKeydown);
+
 		audioAbortController.abort();
 
 		await tick();
@@ -887,8 +945,30 @@
 			{/if}
 		</div>
 
-		<div class="flex justify-between items-center pb-2 w-full">
+		<div class="flex flex-col items-center gap-4 pb-4 w-full">
-			<div>
+			<button
+				type="button"
+				class="z-10"
+				on:click={() => {
+					if (assistantSpeaking) {
+						stopAllAudio();
+					}
+				}}
+			>
+				<div class="line-clamp-1 text-sm font-medium">
+					{#if loading}
+						{$i18n.t('Thinking...')}
+					{:else if muted}
+						{$i18n.t('Muted')}
+					{:else if assistantSpeaking}
+						{$i18n.t('Tap to interrupt')}
+					{:else}
+						{$i18n.t('Listening...')}
+					{/if}
+				</div>
+			</button>
+
+			<div class="flex items-center justify-center gap-4 z-10">
 				{#if camera}
 					<VideoInputMenu
 						devices={videoInputDevices}
@@ -900,7 +980,7 @@
 							await startVideoStream();
 						}}
 					>
-						<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
+						<button class="p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
 							<svg
 								xmlns="http://www.w3.org/2000/svg"
 								viewBox="0 0 20 20"
@@ -918,7 +998,7 @@
 				{:else}
 					<Tooltip content={$i18n.t('Camera')}>
 						<button
-							class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
+							class="p-3 rounded-full bg-gray-50 dark:bg-gray-900"
 							type="button"
 							on:click={async () => {
 								await navigator.mediaDevices.getUserMedia({ video: true });
@@ -947,32 +1027,53 @@
 						</button>
 					</Tooltip>
 				{/if}
-			</div>
 
-			<div>
+				<Tooltip content={muted ? $i18n.t('Unmute') + ' (M)' : $i18n.t('Mute') + ' (M)'}>
-				<button
+					<button
-					type="button"
+						class="p-3 rounded-full transition-colors duration-200 {muted ? 'bg-red-500 text-white' : 'bg-gray-50 dark:bg-gray-900'}"
-					on:click={() => {
+						type="button"
-						if (assistantSpeaking) {
+						aria-label={muted ? $i18n.t('Unmute') : $i18n.t('Mute')}
-							stopAllAudio();
+						on:click={toggleMute}
-						}
+					>
-					}}
+						{#if muted}
-				>
+							<!-- Mic Off icon -->
-					<div class=" line-clamp-1 text-sm font-medium">
+							<svg
-						{#if loading}
+								xmlns="http://www.w3.org/2000/svg"
-							{$i18n.t('Thinking...')}
+								fill="none"
-						{:else if assistantSpeaking}
+								viewBox="0 0 24 24"
-							{$i18n.t('Tap to interrupt')}
+								stroke-width="1.5"
+								stroke="currentColor"
+								class="size-5"
+							>
+								<path
+									stroke-linecap="round"
+									stroke-linejoin="round"
+									d="M12 18.75a6 6 0 0 0 6-6v-1.5m-6 7.5a6 6 0 0 1-6-6v-1.5m6 7.5v3.75m-3.75 0h7.5M12 15.75a3 3 0 0 1-3-3V4.5a3 3 0 1 1 6 0v8.25a3 3 0 0 1-3 3Z"
+								/>
+								<line x1="3" y1="3" x2="21" y2="21" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" />
+							</svg>
 						{:else}
-							{$i18n.t('Listening...')}
+							<!-- Mic On icon -->
+							<svg
+								xmlns="http://www.w3.org/2000/svg"
+								fill="none"
+								viewBox="0 0 24 24"
+								stroke-width="1.5"
+								stroke="currentColor"
+								class="size-5"
+							>
+								<path
+									stroke-linecap="round"
+									stroke-linejoin="round"
+									d="M12 18.75a6 6 0 0 0 6-6v-1.5m-6 7.5a6 6 0 0 1-6-6v-1.5m6 7.5v3.75m-3.75 0h7.5M12 15.75a3 3 0 0 1-3-3V4.5a3 3 0 1 1 6 0v8.25a3 3 0 0 1-3 3Z"
+								/>
+							</svg>
 						{/if}
-					</div>
+					</button>
-				</button>
+				</Tooltip>
-			</div>
 
-			<div>
 				<button
-					class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
+					class="p-3 rounded-full bg-gray-50 dark:bg-gray-900"
 					on:click={async () => {
 						await stopAudioStream();
 						await stopVideoStream();