Merge pull request #1144 from open-webui/dev

0.1.112

.dockerignore

@@ -7,7 +7,6 @@ node_modules
 /package
 .env
 .env.*
-!.env.example
 vite.config.js.timestamp-*
 vite.config.ts.timestamp-*
 __pycache__

.env.example

@@ -1,6 +1,6 @@
 # Ollama URL for the backend to connect
-# The path '/ollama/api' will be redirected to the specified backend URL
-OLLAMA_API_BASE_URL='http://localhost:11434/api'
+# The path '/ollama' will be redirected to the specified backend URL
+OLLAMA_BASE_URL='http://localhost:11434'
 
 OPENAI_API_BASE_URL=''
 OPENAI_API_KEY=''

.github/ISSUE_TEMPLATE/bug_report.md

@@ -32,7 +32,7 @@ assignees: ''
 **Confirmation:**
 
 - [ ] I have read and followed all the instructions provided in the README.md.
-- [ ] I have reviewed the troubleshooting.md document.
+- [ ] I am on the latest version of both Open WebUI and Ollama.
 - [ ] I have included the browser console logs.
 - [ ] I have included the Docker container logs.
 

.github/workflows/build-release.yml

@@ -19,24 +19,34 @@ jobs:
           echo "No changes to package.json"
           exit 1
         }
-
+    
     - name: Get version number from package.json
       id: get_version
       run: |
         VERSION=$(jq -r '.version' package.json)
         echo "::set-output name=version::$VERSION"
 
+    - name: Extract latest CHANGELOG entry
+      id: changelog
+      run: |
+        CHANGELOG_CONTENT=$(awk 'BEGIN {print_section=0;} /^## \[/ {if (print_section == 0) {print_section=1;} else {exit;}} print_section {print;}' CHANGELOG.md)
+        CHANGELOG_ESCAPED=$(echo "$CHANGELOG_CONTENT" | sed ':a;N;$!ba;s/\n/%0A/g')
+        echo "Extracted latest release notes from CHANGELOG.md:" 
+        echo -e "$CHANGELOG_CONTENT" 
+        echo "::set-output name=content::$CHANGELOG_ESCAPED"
+
     - name: Create GitHub release
       uses: actions/github-script@v5
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
+          const changelog = `${{ steps.changelog.outputs.content }}`;
           const release = await github.rest.repos.createRelease({
             owner: context.repo.owner,
             repo: context.repo.repo,
             tag_name: `v${{ steps.get_version.outputs.version }}`,
             name: `v${{ steps.get_version.outputs.version }}`,
-            body: 'Automatically created new release',
+            body: changelog,
           })
           console.log(`Created release ${release.data.html_url}`)
 

CHANGELOG.md

@@ -5,6 +5,64 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.112] - 2024-03-15
+
+### Fixed
+
+- 🗨️ Resolved chat malfunction after image generation.
+- 🎨 Fixed various RAG issues.
+- 🧪 Rectified experimental broken GGUF upload logic.
+
+## [0.1.111] - 2024-03-10
+
+### Added
+
+- 🛡️ **Model Whitelisting**: Admins now have the ability to whitelist models for users with the 'user' role.
+- 🔄 **Update All Models**: Added a convenient button to update all models at once.
+- 📄 **Toggle PDF OCR**: Users can now toggle PDF OCR option for improved parsing performance.
+- 🎨 **DALL-E Integration**: Introduced DALL-E integration for image generation alongside automatic1111.
+- 🛠️ **RAG API Refactoring**: Refactored RAG logic and exposed its API, with additional documentation to follow.
+
+### Fixed
+
+- 🔒 **Max Token Settings**: Added max token settings for anthropic/claude-3-sonnet-20240229 (Issue #1094).
+- 🔧 **Misalignment Issue**: Corrected misalignment of Edit and Delete Icons when Chat Title is Empty (Issue #1104).
+- 🔄 **Context Loss Fix**: Resolved RAG losing context on model response regeneration with Groq models via API key (Issue #1105).
+- 📁 **File Handling Bug**: Addressed File Not Found Notification when Dropping a Conversation Element (Issue #1098).
+- 🖱️ **Dragged File Styling**: Fixed dragged file layover styling issue.
+
+## [0.1.110] - 2024-03-06
+
+### Added
+
+- **🌐 Multiple OpenAI Servers Support**: Enjoy seamless integration with multiple OpenAI-compatible APIs, now supported natively.
+
+### Fixed
+
+- **🔍 OCR Issue**: Resolved PDF parsing issue caused by OCR malfunction.
+- **🚫 RAG Issue**: Fixed the RAG functionality, ensuring it operates smoothly.
+- **📄 "Add Docs" Model Button**: Addressed the non-functional behavior of the "Add Docs" model button.
+
+## [0.1.109] - 2024-03-06
+
+### Added
+
+- **🔄 Multiple Ollama Servers Support**: Enjoy enhanced scalability and performance with support for multiple Ollama servers in a single WebUI. Load balancing features are now available, providing improved efficiency (#788, #278).
+- **🔧 Support for Claude 3 and Gemini**: Responding to user requests, we've expanded our toolset to include Claude 3 and Gemini, offering a wider range of functionalities within our platform (#1064).
+- **🔍 OCR Functionality for PDF Loader**: We've augmented our PDF loader with Optical Character Recognition (OCR) capabilities. Now, extract text from scanned documents and images within PDFs, broadening the scope of content processing (#1050).
+
+### Fixed
+
+- **🛠️ RAG Collection**: Implemented a dynamic mechanism to recreate RAG collections, ensuring users have up-to-date and accurate data (#1031).
+- **📝 User Agent Headers**: Fixed issue of RAG web requests being sent with empty user_agent headers, reducing rejections from certain websites. Realistic headers are now utilized for these requests (#1024).
+- **⏹️ Playground Cancel Functionality**: Introducing a new "Cancel" option for stopping Ollama generation in the Playground, enhancing user control and usability (#1006).
+- **🔤 Typographical Error in 'ASSISTANT' Field**: Corrected a typographical error in the 'ASSISTANT' field within the GGUF model upload template for accuracy and consistency (#1061).
+
+### Changed
+
+- **🔄 Refactored Message Deletion Logic**: Streamlined message deletion process for improved efficiency and user experience, simplifying interactions within the platform (#1004).
+- **⚠️ Deprecation of `OLLAMA_API_BASE_URL`**: Deprecated `OLLAMA_API_BASE_URL` environment variable; recommend using `OLLAMA_BASE_URL` instead. Refer to our documentation for further details.
+
 ## [0.1.108] - 2024-03-02
 
 ### Added

Dockerfile

@@ -20,7 +20,7 @@ FROM python:3.11-slim-bookworm as base
 ENV ENV=prod
 ENV PORT ""
 
-ENV OLLAMA_API_BASE_URL "/ollama/api"
+ENV OLLAMA_BASE_URL "/ollama"
 
 ENV OPENAI_API_BASE_URL ""
 ENV OPENAI_API_KEY ""
@@ -41,7 +41,7 @@ ENV WHISPER_MODEL_DIR="/app/backend/data/cache/whisper/models"
 # for better persormance and multilangauge support use "intfloat/multilingual-e5-large" (~2.5GB) or "intfloat/multilingual-e5-base" (~1.5GB)
 # IMPORTANT: If you change the default model (all-MiniLM-L6-v2) and vice versa, you aren't able to use RAG Chat with your previous documents loaded in the WebUI! You need to re-embed them.
 ENV RAG_EMBEDDING_MODEL="all-MiniLM-L6-v2"
-# device type for whisper tts and ebbeding models - "cpu" (default), "cuda" (nvidia gpu and CUDA required) or "mps" (apple silicon) - choosing this right can lead to better performance
+# device type for whisper tts and embbeding models - "cpu" (default), "cuda" (nvidia gpu and CUDA required) or "mps" (apple silicon) - choosing this right can lead to better performance
 ENV RAG_EMBEDDING_MODEL_DEVICE_TYPE="cpu"
 ENV RAG_EMBEDDING_MODEL_DIR="/app/backend/data/cache/embedding/models"
 ENV SENTENCE_TRANSFORMERS_HOME $RAG_EMBEDDING_MODEL_DIR
@@ -53,6 +53,8 @@ WORKDIR /app/backend
 # install python dependencies
 COPY ./backend/requirements.txt ./requirements.txt
 
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+
 RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir
 RUN pip3 install -r requirements.txt --no-cache-dir
 
@@ -79,4 +81,4 @@ COPY --from=build /app/package.json /app/package.json
 # copy backend files
 COPY ./backend .
 
-CMD [ "bash", "start.sh"]
+CMD [ "bash", "start.sh"]

README.md

@@ -53,8 +53,6 @@ User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-co
 
 - 💬 **Collaborative Chat**: Harness the collective intelligence of multiple models by seamlessly orchestrating group conversations. Use the `@` command to specify the model, enabling dynamic and diverse dialogues within your chat interface. Immerse yourself in the collective intelligence woven into your chat environment.
 
-- 🤝 **OpenAI API Integration**: Effortlessly integrate OpenAI-compatible API for versatile conversations alongside Ollama models. Customize the API Base URL to link with **LMStudio, Mistral, OpenRouter, and more**.
-
 - 🔄 **Regeneration History Access**: Easily revisit and explore your entire regeneration history.
 
 - 📜 **Chat History**: Effortlessly access and manage your conversation history.
@@ -65,8 +63,18 @@ User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-co
 
 - ⚙️ **Fine-Tuned Control with Advanced Parameters**: Gain a deeper level of control by adjusting parameters such as temperature and defining your system prompts to tailor the conversation to your specific preferences and needs.
 
+- 🎨🤖 **Image Generation Integration**: Seamlessly incorporate image generation capabilities using AUTOMATIC1111 API (local) and DALL-E, enriching your chat experience with dynamic visual content.
+
+- 🤝 **OpenAI API Integration**: Effortlessly integrate OpenAI-compatible API for versatile conversations alongside Ollama models. Customize the API Base URL to link with **LMStudio, Mistral, OpenRouter, and more**.
+
+- ✨ **Multiple OpenAI-Compatible API Support**: Seamlessly integrate and customize various OpenAI-compatible APIs, enhancing the versatility of your chat interactions.
+
 - 🔗 **External Ollama Server Connection**: Seamlessly link to an external Ollama server hosted on a different address by configuring the environment variable.
 
+- 🔀 **Multiple Ollama Instance Load Balancing**: Effortlessly distribute chat requests across multiple Ollama instances for enhanced performance and reliability.
+
+- 👥 **Multi-User Management**: Easily oversee and administer users via our intuitive admin panel, streamlining user management processes.
+
 - 🔐 **Role-Based Access Control (RBAC)**: Ensure secure access with restricted permissions; only authorized individuals can access your Ollama, and exclusive model creation/pulling rights are reserved for administrators.
 
 - 🔒 **Backend Reverse Proxy Support**: Bolster security through direct communication between Open WebUI backend and Ollama. This key feature eliminates the need to expose Ollama over LAN. Requests made to the '/ollama/api' route from the web UI are seamlessly redirected to Ollama from the backend, enhancing overall system security.
@@ -95,10 +103,10 @@ Don't forget to explore our sibling project, [Open WebUI Community](https://open
 
 - **If Ollama is on a Different Server**, use this command:
 
-- To connect to Ollama on another server, change the `OLLAMA_API_BASE_URL` to the server's URL:
+- To connect to Ollama on another server, change the `OLLAMA_BASE_URL` to the server's URL:
 
   ```bash
-  docker run -d -p 3000:8080 -e OLLAMA_API_BASE_URL=https://example.com/api -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main
+  docker run -d -p 3000:8080 -e OLLAMA_BASE_URL=https://example.com -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main
   ```
 
 - After installation, you can access Open WebUI at [http://localhost:3000](http://localhost:3000). Enjoy! 😄
@@ -110,7 +118,7 @@ If you're experiencing connection issues, it’s often due to the WebUI docker c
 **Example Docker Command**:
 
 ```bash
-docker run -d --network=host -v open-webui:/app/backend/data -e OLLAMA_API_BASE_URL=http://127.0.0.1:11434/api --name open-webui --restart always ghcr.io/open-webui/open-webui:main
+docker run -d --network=host -v open-webui:/app/backend/data -e OLLAMA_BASE_URL=http://127.0.0.1:11434 --name open-webui --restart always ghcr.io/open-webui/open-webui:main
 ```
 
 ### Other Installation Methods

TROUBLESHOOTING.md

@@ -4,7 +4,7 @@
 
 The Open WebUI system is designed to streamline interactions between the client (your browser) and the Ollama API. At the heart of this design is a backend reverse proxy, enhancing security and resolving CORS issues.
 
-- **How it Works**: The Open WebUI is designed to interact with the Ollama API through a specific route. When a request is made from the WebUI to Ollama, it is not directly sent to the Ollama API. Initially, the request is sent to the Open WebUI backend via `/ollama/api` route. From there, the backend is responsible for forwarding the request to the Ollama API. This forwarding is accomplished by using the route specified in the `OLLAMA_API_BASE_URL` environment variable. Therefore, a request made to `/ollama/api` in the WebUI is effectively the same as making a request to `OLLAMA_API_BASE_URL` in the backend. For instance, a request to `/ollama/api/tags` in the WebUI is equivalent to `OLLAMA_API_BASE_URL/tags` in the backend.
+- **How it Works**: The Open WebUI is designed to interact with the Ollama API through a specific route. When a request is made from the WebUI to Ollama, it is not directly sent to the Ollama API. Initially, the request is sent to the Open WebUI backend via `/ollama` route. From there, the backend is responsible for forwarding the request to the Ollama API. This forwarding is accomplished by using the route specified in the `OLLAMA_BASE_URL` environment variable. Therefore, a request made to `/ollama` in the WebUI is effectively the same as making a request to `OLLAMA_BASE_URL` in the backend. For instance, a request to `/ollama/api/tags` in the WebUI is equivalent to `OLLAMA_BASE_URL/api/tags` in the backend.
 
 - **Security Benefits**: This design prevents direct exposure of the Ollama API to the frontend, safeguarding against potential CORS (Cross-Origin Resource Sharing) issues and unauthorized access. Requiring authentication to access the Ollama API further enhances this security layer.
 
@@ -15,7 +15,7 @@ If you're experiencing connection issues, it’s often due to the WebUI docker c
 **Example Docker Command**:
 
 ```bash
-docker run -d --network=host -v open-webui:/app/backend/data -e OLLAMA_API_BASE_URL=http://127.0.0.1:11434/api --name open-webui --restart always ghcr.io/open-webui/open-webui:main
+docker run -d --network=host -v open-webui:/app/backend/data -e OLLAMA_BASE_URL=http://127.0.0.1:11434 --name open-webui --restart always ghcr.io/open-webui/open-webui:main
 ```
 
 ### General Connection Errors
@@ -25,8 +25,8 @@ docker run -d --network=host -v open-webui:/app/backend/data -e OLLAMA_API_BASE_
 **Troubleshooting Steps**:
 
 1. **Verify Ollama URL Format**:
-   - When running the Web UI container, ensure the `OLLAMA_API_BASE_URL` is correctly set, including the `/api` suffix. (e.g., `http://192.168.1.1:11434/api` for different host setups).
+   - When running the Web UI container, ensure the `OLLAMA_BASE_URL` is correctly set. (e.g., `http://192.168.1.1:11434` for different host setups).
    - In the Open WebUI, navigate to "Settings" > "General".
-   - Confirm that the Ollama Server URL is correctly set to `[OLLAMA URL]/api` (e.g., `http://localhost:11434/api`), including the `/api` suffix.
+   - Confirm that the Ollama Server URL is correctly set to `[OLLAMA URL]` (e.g., `http://localhost:11434`).
 
 By following these enhanced troubleshooting steps, connection issues should be effectively resolved. For further assistance or queries, feel free to reach out to us on our community Discord.

backend/apps/images/main.py

@@ -21,7 +21,16 @@ from utils.utils import (
 from utils.misc import calculate_sha256
 from typing import Optional
 from pydantic import BaseModel
-from config import AUTOMATIC1111_BASE_URL
+from pathlib import Path
+import uuid
+import base64
+import json
+
+from config import CACHE_DIR, AUTOMATIC1111_BASE_URL
+
+
+IMAGE_CACHE_DIR = Path(CACHE_DIR).joinpath("./image/generations/")
+IMAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
 
 app = FastAPI()
 app.add_middleware(
@@ -32,25 +41,34 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.state.ENGINE = ""
+app.state.ENABLED = False
+
+app.state.OPENAI_API_KEY = ""
+app.state.MODEL = ""
+
+
 app.state.AUTOMATIC1111_BASE_URL = AUTOMATIC1111_BASE_URL
-app.state.ENABLED = app.state.AUTOMATIC1111_BASE_URL != ""
+
 app.state.IMAGE_SIZE = "512x512"
 app.state.IMAGE_STEPS = 50
 
 
-@app.get("/enabled", response_model=bool)
-async def get_enable_status(request: Request, user=Depends(get_admin_user)):
-    return app.state.ENABLED
+@app.get("/config")
+async def get_config(request: Request, user=Depends(get_admin_user)):
+    return {"engine": app.state.ENGINE, "enabled": app.state.ENABLED}
 
 
-@app.get("/enabled/toggle", response_model=bool)
-async def toggle_enabled(request: Request, user=Depends(get_admin_user)):
-    try:
-        r = requests.head(app.state.AUTOMATIC1111_BASE_URL)
-        app.state.ENABLED = not app.state.ENABLED
-        return app.state.ENABLED
-    except Exception as e:
-        raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e))
+class ConfigUpdateForm(BaseModel):
+    engine: str
+    enabled: bool
+
+
+@app.post("/config/update")
+async def update_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)):
+    app.state.ENGINE = form_data.engine
+    app.state.ENABLED = form_data.enabled
+    return {"engine": app.state.ENGINE, "enabled": app.state.ENABLED}
 
 
 class UrlUpdateForm(BaseModel):
@@ -58,17 +76,24 @@ class UrlUpdateForm(BaseModel):
 
 
 @app.get("/url")
-async def get_openai_url(user=Depends(get_admin_user)):
+async def get_automatic1111_url(user=Depends(get_admin_user)):
     return {"AUTOMATIC1111_BASE_URL": app.state.AUTOMATIC1111_BASE_URL}
 
 
 @app.post("/url/update")
-async def update_openai_url(form_data: UrlUpdateForm, user=Depends(get_admin_user)):
+async def update_automatic1111_url(
+    form_data: UrlUpdateForm, user=Depends(get_admin_user)
+):
 
     if form_data.url == "":
         app.state.AUTOMATIC1111_BASE_URL = AUTOMATIC1111_BASE_URL
     else:
-        app.state.AUTOMATIC1111_BASE_URL = form_data.url.strip("/")
+        url = form_data.url.strip("/")
+        try:
+            r = requests.head(url)
+            app.state.AUTOMATIC1111_BASE_URL = url
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e))
 
     return {
         "AUTOMATIC1111_BASE_URL": app.state.AUTOMATIC1111_BASE_URL,
@@ -76,6 +101,30 @@ async def update_openai_url(form_data: UrlUpdateForm, user=Depends(get_admin_use
     }
 
 
+class OpenAIKeyUpdateForm(BaseModel):
+    key: str
+
+
+@app.get("/key")
+async def get_openai_key(user=Depends(get_admin_user)):
+    return {"OPENAI_API_KEY": app.state.OPENAI_API_KEY}
+
+
+@app.post("/key/update")
+async def update_openai_key(
+    form_data: OpenAIKeyUpdateForm, user=Depends(get_admin_user)
+):
+
+    if form_data.key == "":
+        raise HTTPException(status_code=400, detail=ERROR_MESSAGES.API_KEY_NOT_FOUND)
+
+    app.state.OPENAI_API_KEY = form_data.key
+    return {
+        "OPENAI_API_KEY": app.state.OPENAI_API_KEY,
+        "status": True,
+    }
+
+
 class ImageSizeUpdateForm(BaseModel):
     size: str
 
@@ -132,9 +181,22 @@ async def update_image_size(
 @app.get("/models")
 def get_models(user=Depends(get_current_user)):
     try:
-        r = requests.get(url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/sd-models")
-        models = r.json()
-        return models
+        if app.state.ENGINE == "openai":
+            return [
+                {"id": "dall-e-2", "name": "DALL·E 2"},
+                {"id": "dall-e-3", "name": "DALL·E 3"},
+            ]
+        else:
+            r = requests.get(
+                url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/sd-models"
+            )
+            models = r.json()
+            return list(
+                map(
+                    lambda model: {"id": model["title"], "name": model["model_name"]},
+                    models,
+                )
+            )
     except Exception as e:
         app.state.ENABLED = False
         raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e))
@@ -143,10 +205,12 @@ def get_models(user=Depends(get_current_user)):
 @app.get("/models/default")
 async def get_default_model(user=Depends(get_admin_user)):
     try:
-        r = requests.get(url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options")
-        options = r.json()
-
-        return {"model": options["sd_model_checkpoint"]}
+        if app.state.ENGINE == "openai":
+            return {"model": app.state.MODEL if app.state.MODEL else "dall-e-2"}
+        else:
+            r = requests.get(url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options")
+            options = r.json()
+            return {"model": options["sd_model_checkpoint"]}
     except Exception as e:
         app.state.ENABLED = False
         raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e))
@@ -157,16 +221,21 @@ class UpdateModelForm(BaseModel):
 
 
 def set_model_handler(model: str):
-    r = requests.get(url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options")
-    options = r.json()
 
-    if model != options["sd_model_checkpoint"]:
-        options["sd_model_checkpoint"] = model
-        r = requests.post(
-            url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options", json=options
-        )
+    if app.state.ENGINE == "openai":
+        app.state.MODEL = model
+        return app.state.MODEL
+    else:
+        r = requests.get(url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options")
+        options = r.json()
 
-    return options
+        if model != options["sd_model_checkpoint"]:
+            options["sd_model_checkpoint"] = model
+            r = requests.post(
+                url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/options", json=options
+            )
+
+        return options
 
 
 @app.post("/models/default/update")
@@ -181,45 +250,116 @@ class GenerateImageForm(BaseModel):
     model: Optional[str] = None
     prompt: str
     n: int = 1
-    size: str = "512x512"
+    size: Optional[str] = None
     negative_prompt: Optional[str] = None
 
 
+def save_b64_image(b64_str):
+    image_id = str(uuid.uuid4())
+    file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.png")
+
+    try:
+        # Split the base64 string to get the actual image data
+        img_data = base64.b64decode(b64_str)
+
+        # Write the image data to a file
+        with open(file_path, "wb") as f:
+            f.write(img_data)
+
+        return image_id
+    except Exception as e:
+        print(f"Error saving image: {e}")
+        return None
+
+
 @app.post("/generations")
 def generate_image(
     form_data: GenerateImageForm,
     user=Depends(get_current_user),
 ):
 
-    print(form_data)
-
+    r = None
     try:
-        if form_data.model:
-            set_model_handler(form_data.model)
+        if app.state.ENGINE == "openai":
 
-        width, height = tuple(map(int, app.state.IMAGE_SIZE.split("x")))
+            headers = {}
+            headers["Authorization"] = f"Bearer {app.state.OPENAI_API_KEY}"
+            headers["Content-Type"] = "application/json"
 
-        data = {
-            "prompt": form_data.prompt,
-            "batch_size": form_data.n,
-            "width": width,
-            "height": height,
-        }
+            data = {
+                "model": app.state.MODEL if app.state.MODEL != "" else "dall-e-2",
+                "prompt": form_data.prompt,
+                "n": form_data.n,
+                "size": form_data.size if form_data.size else app.state.IMAGE_SIZE,
+                "response_format": "b64_json",
+            }
 
-        if app.state.IMAGE_STEPS != None:
-            data["steps"] = app.state.IMAGE_STEPS
+            r = requests.post(
+                url=f"https://api.openai.com/v1/images/generations",
+                json=data,
+                headers=headers,
+            )
 
-        if form_data.negative_prompt != None:
-            data["negative_prompt"] = form_data.negative_prompt
+            r.raise_for_status()
+            res = r.json()
 
-        print(data)
+            images = []
 
-        r = requests.post(
-            url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/txt2img",
-            json=data,
-        )
+            for image in res["data"]:
+                image_id = save_b64_image(image["b64_json"])
+                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
+
+                with open(file_body_path, "w") as f:
+                    json.dump(data, f)
+
+            return images
+
+        else:
+            if form_data.model:
+                set_model_handler(form_data.model)
+
+            width, height = tuple(map(int, app.state.IMAGE_SIZE.split("x")))
+
+            data = {
+                "prompt": form_data.prompt,
+                "batch_size": form_data.n,
+                "width": width,
+                "height": height,
+            }
+
+            if app.state.IMAGE_STEPS != None:
+                data["steps"] = app.state.IMAGE_STEPS
+
+            if form_data.negative_prompt != None:
+                data["negative_prompt"] = form_data.negative_prompt
+
+            r = requests.post(
+                url=f"{app.state.AUTOMATIC1111_BASE_URL}/sdapi/v1/txt2img",
+                json=data,
+            )
+
+            res = r.json()
+
+            print(res)
+
+            images = []
+
+            for image in res["images"]:
+                image_id = save_b64_image(image)
+                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
+
+                with open(file_body_path, "w") as f:
+                    json.dump({**data, "info": res["info"]}, f)
+
+            return images
 
-        return r.json()
     except Exception as e:
-        print(e)
-        raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e))
+        error = e
+
+        if r != None:
+            data = r.json()
+            if "error" in data:
+                error = data["error"]["message"]
+        raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(error))

backend/apps/litellm/main.py

@@ -0,0 +1,41 @@
+from litellm.proxy.proxy_server import ProxyConfig, initialize
+from litellm.proxy.proxy_server import app
+
+from fastapi import FastAPI, Request, Depends, status
+from fastapi.responses import JSONResponse
+from utils.utils import get_http_authorization_cred, get_current_user
+from config import ENV
+
+proxy_config = ProxyConfig()
+
+
+async def config():
+    router, model_list, general_settings = await proxy_config.load_config(
+        router=None, config_file_path="./data/litellm/config.yaml"
+    )
+
+    await initialize(config="./data/litellm/config.yaml", telemetry=False)
+
+
+async def startup():
+    await config()
+
+
+@app.on_event("startup")
+async def on_startup():
+    await startup()
+
+
+@app.middleware("http")
+async def auth_middleware(request: Request, call_next):
+    auth_header = request.headers.get("Authorization", "")
+
+    if ENV != "dev":
+        try:
+            user = get_current_user(get_http_authorization_cred(auth_header))
+            print(user)
+        except Exception as e:
+            return JSONResponse(status_code=400, content={"detail": str(e)})
+
+    response = await call_next(request)
+    return response

backend/apps/ollama/main.py

@@ -3,15 +3,22 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
 from fastapi.concurrency import run_in_threadpool
 
+from pydantic import BaseModel, ConfigDict
+
+import random
 import requests
 import json
 import uuid
-from pydantic import BaseModel
+import aiohttp
+import asyncio
 
 from apps.web.models.users import Users
 from constants import ERROR_MESSAGES
 from utils.utils import decode_token, get_current_user, get_admin_user
-from config import OLLAMA_BASE_URL, WEBUI_AUTH
+from config import OLLAMA_BASE_URLS, MODEL_FILTER_ENABLED, MODEL_FILTER_LIST
+
+from typing import Optional, List, Union
+
 
 app = FastAPI()
 app.add_middleware(
@@ -22,27 +29,48 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-app.state.OLLAMA_BASE_URL = OLLAMA_BASE_URL
 
-# TARGET_SERVER_URL = OLLAMA_API_BASE_URL
+app.state.MODEL_FILTER_ENABLED = MODEL_FILTER_ENABLED
+app.state.MODEL_FILTER_LIST = MODEL_FILTER_LIST
+
+app.state.OLLAMA_BASE_URLS = OLLAMA_BASE_URLS
+app.state.MODELS = {}
 
 
 REQUEST_POOL = []
 
 
-@app.get("/url")
-async def get_ollama_api_url(user=Depends(get_admin_user)):
-    return {"OLLAMA_BASE_URL": app.state.OLLAMA_BASE_URL}
+# TODO: Implement a more intelligent load balancing mechanism for distributing requests among multiple backend instances.
+# Current implementation uses a simple round-robin approach (random.choice). Consider incorporating algorithms like weighted round-robin,
+# least connections, or least response time for better resource utilization and performance optimization.
+
+
+@app.middleware("http")
+async def check_url(request: Request, call_next):
+    if len(app.state.MODELS) == 0:
+        await get_all_models()
+    else:
+        pass
+
+    response = await call_next(request)
+    return response
+
+
+@app.get("/urls")
+async def get_ollama_api_urls(user=Depends(get_admin_user)):
+    return {"OLLAMA_BASE_URLS": app.state.OLLAMA_BASE_URLS}
 
 
 class UrlUpdateForm(BaseModel):
-    url: str
+    urls: List[str]
 
 
-@app.post("/url/update")
+@app.post("/urls/update")
 async def update_ollama_api_url(form_data: UrlUpdateForm, user=Depends(get_admin_user)):
-    app.state.OLLAMA_BASE_URL = form_data.url
-    return {"OLLAMA_BASE_URL": app.state.OLLAMA_BASE_URL}
+    app.state.OLLAMA_BASE_URLS = form_data.urls
+
+    print(app.state.OLLAMA_BASE_URLS)
+    return {"OLLAMA_BASE_URLS": app.state.OLLAMA_BASE_URLS}
 
 
 @app.get("/cancel/{request_id}")
@@ -55,9 +83,824 @@ async def cancel_ollama_request(request_id: str, user=Depends(get_current_user))
         raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
 
 
+async def fetch_url(url):
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                return await response.json()
+    except Exception as e:
+        # Handle connection error here
+        print(f"Connection error: {e}")
+        return None
+
+
+def merge_models_lists(model_lists):
+    merged_models = {}
+
+    for idx, model_list in enumerate(model_lists):
+        for model in model_list:
+            digest = model["digest"]
+            if digest not in merged_models:
+                model["urls"] = [idx]
+                merged_models[digest] = model
+            else:
+                merged_models[digest]["urls"].append(idx)
+
+    return list(merged_models.values())
+
+
+# user=Depends(get_current_user)
+
+
+async def get_all_models():
+    print("get_all_models")
+    tasks = [fetch_url(f"{url}/api/tags") for url in app.state.OLLAMA_BASE_URLS]
+    responses = await asyncio.gather(*tasks)
+    responses = list(filter(lambda x: x is not None, responses))
+
+    models = {
+        "models": merge_models_lists(
+            map(lambda response: response["models"], responses)
+        )
+    }
+
+    app.state.MODELS = {model["model"]: model for model in models["models"]}
+
+    return models
+
+
+@app.get("/api/tags")
+@app.get("/api/tags/{url_idx}")
+async def get_ollama_tags(
+    url_idx: Optional[int] = None, user=Depends(get_current_user)
+):
+    if url_idx == None:
+        models = await get_all_models()
+
+        if app.state.MODEL_FILTER_ENABLED:
+            if user.role == "user":
+                models["models"] = list(
+                    filter(
+                        lambda model: model["name"] in app.state.MODEL_FILTER_LIST,
+                        models["models"],
+                    )
+                )
+                return models
+        return models
+    else:
+        url = app.state.OLLAMA_BASE_URLS[url_idx]
+        try:
+            r = requests.request(method="GET", url=f"{url}/api/tags")
+            r.raise_for_status()
+
+            return r.json()
+        except Exception as e:
+            print(e)
+            error_detail = "Open WebUI: Server Connection Error"
+            if r is not None:
+                try:
+                    res = r.json()
+                    if "error" in res:
+                        error_detail = f"Ollama: {res['error']}"
+                except:
+                    error_detail = f"Ollama: {e}"
+
+            raise HTTPException(
+                status_code=r.status_code if r else 500,
+                detail=error_detail,
+            )
+
+
+@app.get("/api/version")
+@app.get("/api/version/{url_idx}")
+async def get_ollama_versions(url_idx: Optional[int] = None):
+
+    if url_idx == None:
+
+        # returns lowest version
+        tasks = [fetch_url(f"{url}/api/version") for url in app.state.OLLAMA_BASE_URLS]
+        responses = await asyncio.gather(*tasks)
+        responses = list(filter(lambda x: x is not None, responses))
+
+        if len(responses) > 0:
+            lowest_version = min(
+                responses, key=lambda x: tuple(map(int, x["version"].split(".")))
+            )
+
+            return {"version": lowest_version["version"]}
+        else:
+            raise HTTPException(
+                status_code=500,
+                detail=ERROR_MESSAGES.OLLAMA_NOT_FOUND,
+            )
+    else:
+        url = app.state.OLLAMA_BASE_URLS[url_idx]
+        try:
+            r = requests.request(method="GET", url=f"{url}/api/version")
+            r.raise_for_status()
+
+            return r.json()
+        except Exception as e:
+            print(e)
+            error_detail = "Open WebUI: Server Connection Error"
+            if r is not None:
+                try:
+                    res = r.json()
+                    if "error" in res:
+                        error_detail = f"Ollama: {res['error']}"
+                except:
+                    error_detail = f"Ollama: {e}"
+
+            raise HTTPException(
+                status_code=r.status_code if r else 500,
+                detail=error_detail,
+            )
+
+
+class ModelNameForm(BaseModel):
+    name: str
+
+
+@app.post("/api/pull")
+@app.post("/api/pull/{url_idx}")
+async def pull_model(
+    form_data: ModelNameForm, url_idx: int = 0, user=Depends(get_admin_user)
+):
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    def get_request():
+        nonlocal url
+        nonlocal r
+        try:
+
+            def stream_content():
+                for chunk in r.iter_content(chunk_size=8192):
+                    yield chunk
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/api/pull",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class PushModelForm(BaseModel):
+    name: str
+    insecure: Optional[bool] = None
+    stream: Optional[bool] = None
+
+
+@app.delete("/api/push")
+@app.delete("/api/push/{url_idx}")
+async def push_model(
+    form_data: PushModelForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_admin_user),
+):
+    if url_idx == None:
+        if form_data.name in app.state.MODELS:
+            url_idx = app.state.MODELS[form_data.name]["urls"][0]
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.name),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    def get_request():
+        nonlocal url
+        nonlocal r
+        try:
+
+            def stream_content():
+                for chunk in r.iter_content(chunk_size=8192):
+                    yield chunk
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/api/push",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+            )
+
+            r.raise_for_status()
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class CreateModelForm(BaseModel):
+    name: str
+    modelfile: Optional[str] = None
+    stream: Optional[bool] = None
+    path: Optional[str] = None
+
+
+@app.post("/api/create")
+@app.post("/api/create/{url_idx}")
+async def create_model(
+    form_data: CreateModelForm, url_idx: int = 0, user=Depends(get_admin_user)
+):
+    print(form_data)
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    def get_request():
+        nonlocal url
+        nonlocal r
+        try:
+
+            def stream_content():
+                for chunk in r.iter_content(chunk_size=8192):
+                    yield chunk
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/api/create",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            print(r)
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class CopyModelForm(BaseModel):
+    source: str
+    destination: str
+
+
+@app.post("/api/copy")
+@app.post("/api/copy/{url_idx}")
+async def copy_model(
+    form_data: CopyModelForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_admin_user),
+):
+    if url_idx == None:
+        if form_data.source in app.state.MODELS:
+            url_idx = app.state.MODELS[form_data.source]["urls"][0]
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.source),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    try:
+        r = requests.request(
+            method="POST",
+            url=f"{url}/api/copy",
+            data=form_data.model_dump_json(exclude_none=True).encode(),
+        )
+        r.raise_for_status()
+
+        print(r.text)
+
+        return True
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+@app.delete("/api/delete")
+@app.delete("/api/delete/{url_idx}")
+async def delete_model(
+    form_data: ModelNameForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_admin_user),
+):
+    if url_idx == None:
+        if form_data.name in app.state.MODELS:
+            url_idx = app.state.MODELS[form_data.name]["urls"][0]
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.name),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    try:
+        r = requests.request(
+            method="DELETE",
+            url=f"{url}/api/delete",
+            data=form_data.model_dump_json(exclude_none=True).encode(),
+        )
+        r.raise_for_status()
+
+        print(r.text)
+
+        return True
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+@app.post("/api/show")
+async def show_model_info(form_data: ModelNameForm, user=Depends(get_current_user)):
+    if form_data.name not in app.state.MODELS:
+        raise HTTPException(
+            status_code=400,
+            detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.name),
+        )
+
+    url_idx = random.choice(app.state.MODELS[form_data.name]["urls"])
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    try:
+        r = requests.request(
+            method="POST",
+            url=f"{url}/api/show",
+            data=form_data.model_dump_json(exclude_none=True).encode(),
+        )
+        r.raise_for_status()
+
+        return r.json()
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class GenerateEmbeddingsForm(BaseModel):
+    model: str
+    prompt: str
+    options: Optional[dict] = None
+    keep_alive: Optional[Union[int, str]] = None
+
+
+@app.post("/api/embeddings")
+@app.post("/api/embeddings/{url_idx}")
+async def generate_embeddings(
+    form_data: GenerateEmbeddingsForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_current_user),
+):
+    if url_idx == None:
+        if form_data.model in app.state.MODELS:
+            url_idx = random.choice(app.state.MODELS[form_data.model]["urls"])
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    try:
+        r = requests.request(
+            method="POST",
+            url=f"{url}/api/embeddings",
+            data=form_data.model_dump_json(exclude_none=True).encode(),
+        )
+        r.raise_for_status()
+
+        return r.json()
+    except Exception as e:
+        print(e)
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class GenerateCompletionForm(BaseModel):
+    model: str
+    prompt: str
+    images: Optional[List[str]] = None
+    format: Optional[str] = None
+    options: Optional[dict] = None
+    system: Optional[str] = None
+    template: Optional[str] = None
+    context: Optional[str] = None
+    stream: Optional[bool] = True
+    raw: Optional[bool] = None
+    keep_alive: Optional[Union[int, str]] = None
+
+
+@app.post("/api/generate")
+@app.post("/api/generate/{url_idx}")
+async def generate_completion(
+    form_data: GenerateCompletionForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_current_user),
+):
+
+    if url_idx == None:
+        if form_data.model in app.state.MODELS:
+            url_idx = random.choice(app.state.MODELS[form_data.model]["urls"])
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail="error_detail",
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    def get_request():
+        nonlocal form_data
+        nonlocal r
+
+        request_id = str(uuid.uuid4())
+        try:
+            REQUEST_POOL.append(request_id)
+
+            def stream_content():
+                try:
+                    if form_data.stream:
+                        yield json.dumps({"id": request_id, "done": False}) + "\n"
+
+                    for chunk in r.iter_content(chunk_size=8192):
+                        if request_id in REQUEST_POOL:
+                            yield chunk
+                        else:
+                            print("User: canceled request")
+                            break
+                finally:
+                    if hasattr(r, "close"):
+                        r.close()
+                        if request_id in REQUEST_POOL:
+                            REQUEST_POOL.remove(request_id)
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/api/generate",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+    images: Optional[List[str]] = None
+
+
+class GenerateChatCompletionForm(BaseModel):
+    model: str
+    messages: List[ChatMessage]
+    format: Optional[str] = None
+    options: Optional[dict] = None
+    template: Optional[str] = None
+    stream: Optional[bool] = True
+    keep_alive: Optional[Union[int, str]] = None
+
+
+@app.post("/api/chat")
+@app.post("/api/chat/{url_idx}")
+async def generate_chat_completion(
+    form_data: GenerateChatCompletionForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_current_user),
+):
+
+    if url_idx == None:
+        if form_data.model in app.state.MODELS:
+            url_idx = random.choice(app.state.MODELS[form_data.model]["urls"])
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    print(form_data.model_dump_json(exclude_none=True).encode())
+
+    def get_request():
+        nonlocal form_data
+        nonlocal r
+
+        request_id = str(uuid.uuid4())
+        try:
+            REQUEST_POOL.append(request_id)
+
+            def stream_content():
+                try:
+                    if form_data.stream:
+                        yield json.dumps({"id": request_id, "done": False}) + "\n"
+
+                    for chunk in r.iter_content(chunk_size=8192):
+                        if request_id in REQUEST_POOL:
+                            yield chunk
+                        else:
+                            print("User: canceled request")
+                            break
+                finally:
+                    if hasattr(r, "close"):
+                        r.close()
+                        if request_id in REQUEST_POOL:
+                            REQUEST_POOL.remove(request_id)
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/api/chat",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            print(e)
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
+# TODO: we should update this part once Ollama supports other types
+class OpenAIChatMessage(BaseModel):
+    role: str
+    content: str
+
+    model_config = ConfigDict(extra="allow")
+
+
+class OpenAIChatCompletionForm(BaseModel):
+    model: str
+    messages: List[OpenAIChatMessage]
+
+    model_config = ConfigDict(extra="allow")
+
+
+@app.post("/v1/chat/completions")
+@app.post("/v1/chat/completions/{url_idx}")
+async def generate_openai_chat_completion(
+    form_data: OpenAIChatCompletionForm,
+    url_idx: Optional[int] = None,
+    user=Depends(get_current_user),
+):
+
+    if url_idx == None:
+        if form_data.model in app.state.MODELS:
+            url_idx = random.choice(app.state.MODELS[form_data.model]["urls"])
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model),
+            )
+
+    url = app.state.OLLAMA_BASE_URLS[url_idx]
+    print(url)
+
+    r = None
+
+    def get_request():
+        nonlocal form_data
+        nonlocal r
+
+        request_id = str(uuid.uuid4())
+        try:
+            REQUEST_POOL.append(request_id)
+
+            def stream_content():
+                try:
+                    if form_data.stream:
+                        yield json.dumps(
+                            {"request_id": request_id, "done": False}
+                        ) + "\n"
+
+                    for chunk in r.iter_content(chunk_size=8192):
+                        if request_id in REQUEST_POOL:
+                            yield chunk
+                        else:
+                            print("User: canceled request")
+                            break
+                finally:
+                    if hasattr(r, "close"):
+                        r.close()
+                        if request_id in REQUEST_POOL:
+                            REQUEST_POOL.remove(request_id)
+
+            r = requests.request(
+                method="POST",
+                url=f"{url}/v1/chat/completions",
+                data=form_data.model_dump_json(exclude_none=True).encode(),
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            return StreamingResponse(
+                stream_content(),
+                status_code=r.status_code,
+                headers=dict(r.headers),
+            )
+        except Exception as e:
+            raise e
+
+    try:
+        return await run_in_threadpool(get_request)
+    except Exception as e:
+        error_detail = "Open WebUI: Server Connection Error"
+        if r is not None:
+            try:
+                res = r.json()
+                if "error" in res:
+                    error_detail = f"Ollama: {res['error']}"
+            except:
+                error_detail = f"Ollama: {e}"
+
+        raise HTTPException(
+            status_code=r.status_code if r else 500,
+            detail=error_detail,
+        )
+
+
 @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
-async def proxy(path: str, request: Request, user=Depends(get_current_user)):
-    target_url = f"{app.state.OLLAMA_BASE_URL}/{path}"
+async def deprecated_proxy(path: str, request: Request, user=Depends(get_current_user)):
+    url = app.state.OLLAMA_BASE_URLS[0]
+    target_url = f"{url}/{path}"
 
     body = await request.body()
     headers = dict(request.headers)

backend/apps/ollama/old_main.py

@@ -1,127 +0,0 @@
-from fastapi import FastAPI, Request, Response, HTTPException, Depends
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import StreamingResponse
-
-import requests
-import json
-from pydantic import BaseModel
-
-from apps.web.models.users import Users
-from constants import ERROR_MESSAGES
-from utils.utils import decode_token, get_current_user
-from config import OLLAMA_API_BASE_URL, WEBUI_AUTH
-
-import aiohttp
-
-app = FastAPI()
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-app.state.OLLAMA_API_BASE_URL = OLLAMA_API_BASE_URL
-
-# TARGET_SERVER_URL = OLLAMA_API_BASE_URL
-
-
-@app.get("/url")
-async def get_ollama_api_url(user=Depends(get_current_user)):
-    if user and user.role == "admin":
-        return {"OLLAMA_API_BASE_URL": app.state.OLLAMA_API_BASE_URL}
-    else:
-        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
-
-
-class UrlUpdateForm(BaseModel):
-    url: str
-
-
-@app.post("/url/update")
-async def update_ollama_api_url(
-    form_data: UrlUpdateForm, user=Depends(get_current_user)
-):
-    if user and user.role == "admin":
-        app.state.OLLAMA_API_BASE_URL = form_data.url
-        return {"OLLAMA_API_BASE_URL": app.state.OLLAMA_API_BASE_URL}
-    else:
-        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
-
-
-# async def fetch_sse(method, target_url, body, headers):
-#     async with aiohttp.ClientSession() as session:
-#         try:
-#             async with session.request(
-#                 method, target_url, data=body, headers=headers
-#             ) as response:
-#                 print(response.status)
-#                 async for line in response.content:
-#                     yield line
-#         except Exception as e:
-#             print(e)
-#             error_detail = "Open WebUI: Server Connection Error"
-#             yield json.dumps({"error": error_detail, "message": str(e)}).encode()
-
-
-@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
-async def proxy(path: str, request: Request, user=Depends(get_current_user)):
-    target_url = f"{app.state.OLLAMA_API_BASE_URL}/{path}"
-    print(target_url)
-
-    body = await request.body()
-    headers = dict(request.headers)
-
-    if user.role in ["user", "admin"]:
-        if path in ["pull", "delete", "push", "copy", "create"]:
-            if user.role != "admin":
-                raise HTTPException(
-                    status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED
-                )
-    else:
-        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
-
-    headers.pop("Host", None)
-    headers.pop("Authorization", None)
-    headers.pop("Origin", None)
-    headers.pop("Referer", None)
-
-    session = aiohttp.ClientSession()
-    response = None
-    try:
-        response = await session.request(
-            request.method, target_url, data=body, headers=headers
-        )
-
-        print(response)
-        if not response.ok:
-            data = await response.json()
-            print(data)
-            response.raise_for_status()
-
-        async def generate():
-            async for line in response.content:
-                print(line)
-                yield line
-            await session.close()
-
-        return StreamingResponse(generate(), response.status)
-
-    except Exception as e:
-        print(e)
-        error_detail = "Open WebUI: Server Connection Error"
-
-        if response is not None:
-            try:
-                res = await response.json()
-                if "error" in res:
-                    error_detail = f"Ollama: {res['error']}"
-            except:
-                error_detail = f"Ollama: {e}"
-
-        await session.close()
-        raise HTTPException(
-            status_code=response.status if response else 500,
-            detail=error_detail,
-        )

backend/apps/openai/main.py

@@ -3,7 +3,10 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
 
 import requests
+import aiohttp
+import asyncio
 import json
+
 from pydantic import BaseModel
 
 
@@ -15,7 +18,15 @@ from utils.utils import (
     get_verified_user,
     get_admin_user,
 )
-from config import OPENAI_API_BASE_URL, OPENAI_API_KEY, CACHE_DIR
+from config import (
+    OPENAI_API_BASE_URLS,
+    OPENAI_API_KEYS,
+    CACHE_DIR,
+    MODEL_FILTER_ENABLED,
+    MODEL_FILTER_LIST,
+)
+from typing import List, Optional
+
 
 import hashlib
 from pathlib import Path
@@ -29,116 +40,225 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-app.state.OPENAI_API_BASE_URL = OPENAI_API_BASE_URL
-app.state.OPENAI_API_KEY = OPENAI_API_KEY
+app.state.MODEL_FILTER_ENABLED = MODEL_FILTER_ENABLED
+app.state.MODEL_FILTER_LIST = MODEL_FILTER_LIST
+
+app.state.OPENAI_API_BASE_URLS = OPENAI_API_BASE_URLS
+app.state.OPENAI_API_KEYS = OPENAI_API_KEYS
+
+app.state.MODELS = {}
 
 
-class UrlUpdateForm(BaseModel):
-    url: str
+@app.middleware("http")
+async def check_url(request: Request, call_next):
+    if len(app.state.MODELS) == 0:
+        await get_all_models()
+    else:
+        pass
+
+    response = await call_next(request)
+    return response
 
 
-class KeyUpdateForm(BaseModel):
-    key: str
+class UrlsUpdateForm(BaseModel):
+    urls: List[str]
 
 
-@app.get("/url")
-async def get_openai_url(user=Depends(get_admin_user)):
-    return {"OPENAI_API_BASE_URL": app.state.OPENAI_API_BASE_URL}
+class KeysUpdateForm(BaseModel):
+    keys: List[str]
 
 
-@app.post("/url/update")
-async def update_openai_url(form_data: UrlUpdateForm, user=Depends(get_admin_user)):
-    app.state.OPENAI_API_BASE_URL = form_data.url
-    return {"OPENAI_API_BASE_URL": app.state.OPENAI_API_BASE_URL}
+@app.get("/urls")
+async def get_openai_urls(user=Depends(get_admin_user)):
+    return {"OPENAI_API_BASE_URLS": app.state.OPENAI_API_BASE_URLS}
 
 
-@app.get("/key")
-async def get_openai_key(user=Depends(get_admin_user)):
-    return {"OPENAI_API_KEY": app.state.OPENAI_API_KEY}
+@app.post("/urls/update")
+async def update_openai_urls(form_data: UrlsUpdateForm, user=Depends(get_admin_user)):
+    app.state.OPENAI_API_BASE_URLS = form_data.urls
+    return {"OPENAI_API_BASE_URLS": app.state.OPENAI_API_BASE_URLS}
 
 
-@app.post("/key/update")
-async def update_openai_key(form_data: KeyUpdateForm, user=Depends(get_admin_user)):
-    app.state.OPENAI_API_KEY = form_data.key
-    return {"OPENAI_API_KEY": app.state.OPENAI_API_KEY}
+@app.get("/keys")
+async def get_openai_keys(user=Depends(get_admin_user)):
+    return {"OPENAI_API_KEYS": app.state.OPENAI_API_KEYS}
+
+
+@app.post("/keys/update")
+async def update_openai_key(form_data: KeysUpdateForm, user=Depends(get_admin_user)):
+    app.state.OPENAI_API_KEYS = form_data.keys
+    return {"OPENAI_API_KEYS": app.state.OPENAI_API_KEYS}
 
 
 @app.post("/audio/speech")
 async def speech(request: Request, user=Depends(get_verified_user)):
-    target_url = f"{app.state.OPENAI_API_BASE_URL}/audio/speech"
-
-    if app.state.OPENAI_API_KEY == "":
-        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.API_KEY_NOT_FOUND)
-
-    body = await request.body()
-
-    name = hashlib.sha256(body).hexdigest()
-
-    SPEECH_CACHE_DIR = Path(CACHE_DIR).joinpath("./audio/speech/")
-    SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
-    file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
-    file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
-
-    # Check if the file already exists in the cache
-    if file_path.is_file():
-        return FileResponse(file_path)
-
-    headers = {}
-    headers["Authorization"] = f"Bearer {app.state.OPENAI_API_KEY}"
-    headers["Content-Type"] = "application/json"
-
+    idx = None
     try:
-        print("openai")
-        r = requests.post(
-            url=target_url,
-            data=body,
-            headers=headers,
-            stream=True,
+        idx = app.state.OPENAI_API_BASE_URLS.index("https://api.openai.com/v1")
+        body = await request.body()
+        name = hashlib.sha256(body).hexdigest()
+
+        SPEECH_CACHE_DIR = Path(CACHE_DIR).joinpath("./audio/speech/")
+        SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
+        file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
+
+        # Check if the file already exists in the cache
+        if file_path.is_file():
+            return FileResponse(file_path)
+
+        headers = {}
+        headers["Authorization"] = f"Bearer {app.state.OPENAI_API_KEYS[idx]}"
+        headers["Content-Type"] = "application/json"
+
+        try:
+            r = requests.post(
+                url=f"{app.state.OPENAI_API_BASE_URLS[idx]}/audio/speech",
+                data=body,
+                headers=headers,
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            # Save the streaming content to a file
+            with open(file_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+
+            with open(file_body_path, "w") as f:
+                json.dump(json.loads(body.decode("utf-8")), f)
+
+            # Return the saved file
+            return FileResponse(file_path)
+
+        except Exception as e:
+            print(e)
+            error_detail = "Open WebUI: Server Connection Error"
+            if r is not None:
+                try:
+                    res = r.json()
+                    if "error" in res:
+                        error_detail = f"External: {res['error']}"
+                except:
+                    error_detail = f"External: {e}"
+
+            raise HTTPException(status_code=r.status_code, detail=error_detail)
+
+    except ValueError:
+        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.OPENAI_NOT_FOUND)
+
+
+async def fetch_url(url, key):
+    try:
+        headers = {"Authorization": f"Bearer {key}"}
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers) as response:
+                return await response.json()
+    except Exception as e:
+        # Handle connection error here
+        print(f"Connection error: {e}")
+        return None
+
+
+def merge_models_lists(model_lists):
+    merged_list = []
+
+    for idx, models in enumerate(model_lists):
+        merged_list.extend(
+            [
+                {**model, "urlIdx": idx}
+                for model in models
+                if "api.openai.com" not in app.state.OPENAI_API_BASE_URLS[idx]
+                or "gpt" in model["id"]
+            ]
         )
 
-        r.raise_for_status()
+    return merged_list
 
-        # Save the streaming content to a file
-        with open(file_path, "wb") as f:
-            for chunk in r.iter_content(chunk_size=8192):
-                f.write(chunk)
 
-        with open(file_body_path, "w") as f:
-            json.dump(json.loads(body.decode("utf-8")), f)
+async def get_all_models():
+    print("get_all_models")
 
-        # Return the saved file
-        return FileResponse(file_path)
+    if len(app.state.OPENAI_API_KEYS) == 1 and app.state.OPENAI_API_KEYS[0] == "":
+        models = {"data": []}
+    else:
+        tasks = [
+            fetch_url(f"{url}/models", app.state.OPENAI_API_KEYS[idx])
+            for idx, url in enumerate(app.state.OPENAI_API_BASE_URLS)
+        ]
+        responses = await asyncio.gather(*tasks)
+        responses = list(
+            filter(lambda x: x is not None and "error" not in x, responses)
+        )
+        models = {
+            "data": merge_models_lists(
+                list(map(lambda response: response["data"], responses))
+            )
+        }
+        app.state.MODELS = {model["id"]: model for model in models["data"]}
 
-    except Exception as e:
-        print(e)
-        error_detail = "Open WebUI: Server Connection Error"
-        if r is not None:
-            try:
-                res = r.json()
-                if "error" in res:
-                    error_detail = f"External: {res['error']}"
-            except:
-                error_detail = f"External: {e}"
+        return models
 
-        raise HTTPException(status_code=r.status_code, detail=error_detail)
+
+@app.get("/models")
+@app.get("/models/{url_idx}")
+async def get_models(url_idx: Optional[int] = None, user=Depends(get_current_user)):
+    if url_idx == None:
+        models = await get_all_models()
+        if app.state.MODEL_FILTER_ENABLED:
+            if user.role == "user":
+                models["data"] = list(
+                    filter(
+                        lambda model: model["id"] in app.state.MODEL_FILTER_LIST,
+                        models["data"],
+                    )
+                )
+                return models
+        return models
+    else:
+        url = app.state.OPENAI_API_BASE_URLS[url_idx]
+        try:
+            r = requests.request(method="GET", url=f"{url}/models")
+            r.raise_for_status()
+
+            response_data = r.json()
+            if "api.openai.com" in url:
+                response_data["data"] = list(
+                    filter(lambda model: "gpt" in model["id"], response_data["data"])
+                )
+
+            return response_data
+        except Exception as e:
+            print(e)
+            error_detail = "Open WebUI: Server Connection Error"
+            if r is not None:
+                try:
+                    res = r.json()
+                    if "error" in res:
+                        error_detail = f"External: {res['error']}"
+                except:
+                    error_detail = f"External: {e}"
+
+            raise HTTPException(
+                status_code=r.status_code if r else 500,
+                detail=error_detail,
+            )
 
 
 @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
 async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
-    target_url = f"{app.state.OPENAI_API_BASE_URL}/{path}"
-    print(target_url, app.state.OPENAI_API_KEY)
-
-    if app.state.OPENAI_API_KEY == "":
-        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.API_KEY_NOT_FOUND)
+    idx = 0
 
     body = await request.body()
-
     # TODO: Remove below after gpt-4-vision fix from Open AI
     # Try to decode the body of the request from bytes to a UTF-8 string (Require add max_token to fix gpt-4-vision)
     try:
         body = body.decode("utf-8")
         body = json.loads(body)
 
+        idx = app.state.MODELS[body.get("model")]["urlIdx"]
+
         # Check if the model is "gpt-4-vision-preview" and set "max_tokens" to 4000
         # This is a workaround until OpenAI fixes the issue with this model
         if body.get("model") == "gpt-4-vision-preview":
@@ -158,8 +278,16 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
     except json.JSONDecodeError as e:
         print("Error loading request body into a dictionary:", e)
 
+    url = app.state.OPENAI_API_BASE_URLS[idx]
+    key = app.state.OPENAI_API_KEYS[idx]
+
+    target_url = f"{url}/{path}"
+
+    if key == "":
+        raise HTTPException(status_code=401, detail=ERROR_MESSAGES.API_KEY_NOT_FOUND)
+
     headers = {}
-    headers["Authorization"] = f"Bearer {app.state.OPENAI_API_KEY}"
+    headers["Authorization"] = f"Bearer {key}"
     headers["Content-Type"] = "application/json"
 
     try:
@@ -181,21 +309,7 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
                 headers=dict(r.headers),
             )
         else:
-            # For non-SSE, read the response and return it
-            # response_data = (
-            #     r.json()
-            #     if r.headers.get("Content-Type", "")
-            #     == "application/json"
-            #     else r.text
-            # )
-
             response_data = r.json()
-
-            if "api.openai.com" in app.state.OPENAI_API_BASE_URL and path == "models":
-                response_data["data"] = list(
-                    filter(lambda model: "gpt" in model["id"], response_data["data"])
-                )
-
             return response_data
     except Exception as e:
         print(e)

backend/apps/rag/main.py

@@ -44,6 +44,8 @@ from apps.web.models.documents import (
     DocumentResponse,
 )
 
+from apps.rag.utils import query_doc, query_collection
+
 from utils.misc import (
     calculate_sha256,
     calculate_sha256_string,
@@ -75,6 +77,7 @@ from constants import ERROR_MESSAGES
 
 app = FastAPI()
 
+app.state.PDF_EXTRACT_IMAGES = False
 app.state.CHUNK_SIZE = CHUNK_SIZE
 app.state.CHUNK_OVERLAP = CHUNK_OVERLAP
 app.state.RAG_TEMPLATE = RAG_TEMPLATE
@@ -108,7 +111,7 @@ class StoreWebForm(CollectionNameForm):
     url: str
 
 
-def store_data_in_vector_db(data, collection_name) -> bool:
+def store_data_in_vector_db(data, collection_name, overwrite: bool = False) -> bool:
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=app.state.CHUNK_SIZE, chunk_overlap=app.state.CHUNK_OVERLAP
     )
@@ -118,6 +121,12 @@ def store_data_in_vector_db(data, collection_name) -> bool:
     metadatas = [doc.metadata for doc in docs]
 
     try:
+        if overwrite:
+            for collection in CHROMA_CLIENT.list_collections():
+                if collection_name == collection.name:
+                    print(f"deleting existing collection {collection_name}")
+                    CHROMA_CLIENT.delete_collection(name=collection_name)
+
         collection = CHROMA_CLIENT.create_collection(
             name=collection_name,
             embedding_function=app.state.sentence_transformer_ef,
@@ -176,12 +185,15 @@ async def update_embedding_model(
     }
 
 
-@app.get("/chunk")
-async def get_chunk_params(user=Depends(get_admin_user)):
+@app.get("/config")
+async def get_rag_config(user=Depends(get_admin_user)):
     return {
         "status": True,
-        "chunk_size": app.state.CHUNK_SIZE,
-        "chunk_overlap": app.state.CHUNK_OVERLAP,
+        "pdf_extract_images": app.state.PDF_EXTRACT_IMAGES,
+        "chunk": {
+            "chunk_size": app.state.CHUNK_SIZE,
+            "chunk_overlap": app.state.CHUNK_OVERLAP,
+        },
     }
 
 
@@ -190,17 +202,24 @@ class ChunkParamUpdateForm(BaseModel):
     chunk_overlap: int
 
 
-@app.post("/chunk/update")
-async def update_chunk_params(
-    form_data: ChunkParamUpdateForm, user=Depends(get_admin_user)
-):
-    app.state.CHUNK_SIZE = form_data.chunk_size
-    app.state.CHUNK_OVERLAP = form_data.chunk_overlap
+class ConfigUpdateForm(BaseModel):
+    pdf_extract_images: bool
+    chunk: ChunkParamUpdateForm
+
+
+@app.post("/config/update")
+async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)):
+    app.state.PDF_EXTRACT_IMAGES = form_data.pdf_extract_images
+    app.state.CHUNK_SIZE = form_data.chunk.chunk_size
+    app.state.CHUNK_OVERLAP = form_data.chunk.chunk_overlap
 
     return {
         "status": True,
-        "chunk_size": app.state.CHUNK_SIZE,
-        "chunk_overlap": app.state.CHUNK_OVERLAP,
+        "pdf_extract_images": app.state.PDF_EXTRACT_IMAGES,
+        "chunk": {
+            "chunk_size": app.state.CHUNK_SIZE,
+            "chunk_overlap": app.state.CHUNK_OVERLAP,
+        },
     }
 
 
@@ -242,21 +261,18 @@ class QueryDocForm(BaseModel):
 
 
 @app.post("/query/doc")
-def query_doc(
+def query_doc_handler(
     form_data: QueryDocForm,
     user=Depends(get_current_user),
 ):
+
     try:
-        # if you use docker use the model from the environment variable
-        collection = CHROMA_CLIENT.get_collection(
-            name=form_data.collection_name,
+        return query_doc(
+            collection_name=form_data.collection_name,
+            query=form_data.query,
+            k=form_data.k if form_data.k else app.state.TOP_K,
             embedding_function=app.state.sentence_transformer_ef,
         )
-        result = collection.query(
-            query_texts=[form_data.query],
-            n_results=form_data.k if form_data.k else app.state.TOP_K,
-        )
-        return result
     except Exception as e:
         print(e)
         raise HTTPException(
@@ -271,76 +287,16 @@ class QueryCollectionsForm(BaseModel):
     k: Optional[int] = None
 
 
-def merge_and_sort_query_results(query_results, k):
-    # Initialize lists to store combined data
-    combined_ids = []
-    combined_distances = []
-    combined_metadatas = []
-    combined_documents = []
-
-    # Combine data from each dictionary
-    for data in query_results:
-        combined_ids.extend(data["ids"][0])
-        combined_distances.extend(data["distances"][0])
-        combined_metadatas.extend(data["metadatas"][0])
-        combined_documents.extend(data["documents"][0])
-
-    # Create a list of tuples (distance, id, metadata, document)
-    combined = list(
-        zip(combined_distances, combined_ids, combined_metadatas, combined_documents)
-    )
-
-    # Sort the list based on distances
-    combined.sort(key=lambda x: x[0])
-
-    # Unzip the sorted list
-    sorted_distances, sorted_ids, sorted_metadatas, sorted_documents = zip(*combined)
-
-    # Slicing the lists to include only k elements
-    sorted_distances = list(sorted_distances)[:k]
-    sorted_ids = list(sorted_ids)[:k]
-    sorted_metadatas = list(sorted_metadatas)[:k]
-    sorted_documents = list(sorted_documents)[:k]
-
-    # Create the output dictionary
-    merged_query_results = {
-        "ids": [sorted_ids],
-        "distances": [sorted_distances],
-        "metadatas": [sorted_metadatas],
-        "documents": [sorted_documents],
-        "embeddings": None,
-        "uris": None,
-        "data": None,
-    }
-
-    return merged_query_results
-
-
 @app.post("/query/collection")
-def query_collection(
+def query_collection_handler(
     form_data: QueryCollectionsForm,
     user=Depends(get_current_user),
 ):
-    results = []
-
-    for collection_name in form_data.collection_names:
-        try:
-            # if you use docker use the model from the environment variable
-            collection = CHROMA_CLIENT.get_collection(
-                name=collection_name,
-                embedding_function=app.state.sentence_transformer_ef,
-            )
-
-            result = collection.query(
-                query_texts=[form_data.query],
-                n_results=form_data.k if form_data.k else app.state.TOP_K,
-            )
-            results.append(result)
-        except:
-            pass
-
-    return merge_and_sort_query_results(
-        results, form_data.k if form_data.k else app.state.TOP_K
+    return query_collection(
+        collection_names=form_data.collection_names,
+        query=form_data.query,
+        k=form_data.k if form_data.k else app.state.TOP_K,
+        embedding_function=app.state.sentence_transformer_ef,
     )
 
 
@@ -355,7 +311,7 @@ def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
         if collection_name == "":
             collection_name = calculate_sha256_string(form_data.url)[:63]
 
-        store_data_in_vector_db(data, collection_name)
+        store_data_in_vector_db(data, collection_name, overwrite=True)
         return {
             "status": True,
             "collection_name": collection_name,
@@ -419,7 +375,7 @@ def get_loader(filename: str, file_content_type: str, file_path: str):
     ]
 
     if file_ext == "pdf":
-        loader = PyPDFLoader(file_path)
+        loader = PyPDFLoader(file_path, extract_images=app.state.PDF_EXTRACT_IMAGES)
     elif file_ext == "csv":
         loader = CSVLoader(file_path)
     elif file_ext == "rst":

backend/apps/rag/utils.py

@@ -0,0 +1,182 @@
+import re
+from typing import List
+
+from config import CHROMA_CLIENT
+
+
+def query_doc(collection_name: str, query: str, k: int, embedding_function):
+    try:
+        # if you use docker use the model from the environment variable
+        collection = CHROMA_CLIENT.get_collection(
+            name=collection_name,
+            embedding_function=embedding_function,
+        )
+        result = collection.query(
+            query_texts=[query],
+            n_results=k,
+        )
+        return result
+    except Exception as e:
+        raise e
+
+
+def merge_and_sort_query_results(query_results, k):
+    # Initialize lists to store combined data
+    combined_ids = []
+    combined_distances = []
+    combined_metadatas = []
+    combined_documents = []
+
+    # Combine data from each dictionary
+    for data in query_results:
+        combined_ids.extend(data["ids"][0])
+        combined_distances.extend(data["distances"][0])
+        combined_metadatas.extend(data["metadatas"][0])
+        combined_documents.extend(data["documents"][0])
+
+    # Create a list of tuples (distance, id, metadata, document)
+    combined = list(
+        zip(combined_distances, combined_ids, combined_metadatas, combined_documents)
+    )
+
+    # Sort the list based on distances
+    combined.sort(key=lambda x: x[0])
+
+    # Unzip the sorted list
+    sorted_distances, sorted_ids, sorted_metadatas, sorted_documents = zip(*combined)
+
+    # Slicing the lists to include only k elements
+    sorted_distances = list(sorted_distances)[:k]
+    sorted_ids = list(sorted_ids)[:k]
+    sorted_metadatas = list(sorted_metadatas)[:k]
+    sorted_documents = list(sorted_documents)[:k]
+
+    # Create the output dictionary
+    merged_query_results = {
+        "ids": [sorted_ids],
+        "distances": [sorted_distances],
+        "metadatas": [sorted_metadatas],
+        "documents": [sorted_documents],
+        "embeddings": None,
+        "uris": None,
+        "data": None,
+    }
+
+    return merged_query_results
+
+
+def query_collection(
+    collection_names: List[str], query: str, k: int, embedding_function
+):
+
+    results = []
+
+    for collection_name in collection_names:
+        try:
+            # if you use docker use the model from the environment variable
+            collection = CHROMA_CLIENT.get_collection(
+                name=collection_name,
+                embedding_function=embedding_function,
+            )
+
+            result = collection.query(
+                query_texts=[query],
+                n_results=k,
+            )
+            results.append(result)
+        except:
+            pass
+
+    return merge_and_sort_query_results(results, k)
+
+
+def rag_template(template: str, context: str, query: str):
+    template = template.replace("[context]", context)
+    template = template.replace("[query]", query)
+    return template
+
+
+def rag_messages(docs, messages, template, k, embedding_function):
+    print(docs)
+
+    last_user_message_idx = None
+    for i in range(len(messages) - 1, -1, -1):
+        if messages[i]["role"] == "user":
+            last_user_message_idx = i
+            break
+
+    user_message = messages[last_user_message_idx]
+
+    if isinstance(user_message["content"], list):
+        # Handle list content input
+        content_type = "list"
+        query = ""
+        for content_item in user_message["content"]:
+            if content_item["type"] == "text":
+                query = content_item["text"]
+                break
+    elif isinstance(user_message["content"], str):
+        # Handle text content input
+        content_type = "text"
+        query = user_message["content"]
+    else:
+        # Fallback in case the input does not match expected types
+        content_type = None
+        query = ""
+
+    relevant_contexts = []
+
+    for doc in docs:
+        context = None
+
+        try:
+            if doc["type"] == "collection":
+                context = query_collection(
+                    collection_names=doc["collection_names"],
+                    query=query,
+                    k=k,
+                    embedding_function=embedding_function,
+                )
+            else:
+                context = query_doc(
+                    collection_name=doc["collection_name"],
+                    query=query,
+                    k=k,
+                    embedding_function=embedding_function,
+                )
+        except Exception as e:
+            print(e)
+            context = None
+
+        relevant_contexts.append(context)
+
+    context_string = ""
+    for context in relevant_contexts:
+        if context:
+            context_string += " ".join(context["documents"][0]) + "\n"
+
+    ra_content = rag_template(
+        template=template,
+        context=context_string,
+        query=query,
+    )
+
+    if content_type == "list":
+        new_content = []
+        for content_item in user_message["content"]:
+            if content_item["type"] == "text":
+                # Update the text item's content with ra_content
+                new_content.append({"type": "text", "text": ra_content})
+            else:
+                # Keep other types of content as they are
+                new_content.append(content_item)
+        new_user_message = {**user_message, "content": new_content}
+    else:
+        new_user_message = {
+            **user_message,
+            "content": ra_content,
+        }
+
+    messages[last_user_message_idx] = new_user_message
+
+    return messages

backend/apps/web/routers/utils.py

@@ -14,7 +14,7 @@ import json
 from utils.utils import get_admin_user
 from utils.misc import calculate_sha256, get_gravatar_url
 
-from config import OLLAMA_API_BASE_URL, DATA_DIR, UPLOAD_DIR
+from config import OLLAMA_BASE_URLS, DATA_DIR, UPLOAD_DIR
 from constants import ERROR_MESSAGES
 
 
@@ -75,7 +75,7 @@ async def download_file_stream(url, file_path, file_name, chunk_size=1024 * 1024
                     hashed = calculate_sha256(file)
                     file.seek(0)
 
-                    url = f"{OLLAMA_API_BASE_URL}/blobs/sha256:{hashed}"
+                    url = f"{OLLAMA_BASE_URLS[0]}/api/blobs/sha256:{hashed}"
                     response = requests.post(url, data=file)
 
                     if response.ok:
@@ -147,7 +147,7 @@ def upload(file: UploadFile = File(...)):
                     hashed = calculate_sha256(f)
                     f.seek(0)
 
-                    url = f"{OLLAMA_API_BASE_URL}/blobs/sha256:{hashed}"
+                    url = f"{OLLAMA_BASE_URLS[0]}/blobs/sha256:{hashed}"
                     response = requests.post(url, data=f)
 
                     if response.ok:

backend/config.py

@@ -200,27 +200,33 @@ if not os.path.exists(LITELLM_CONFIG_PATH):
 
 
 ####################################
-# OLLAMA_API_BASE_URL
+# OLLAMA_BASE_URL
 ####################################
 
 OLLAMA_API_BASE_URL = os.environ.get(
     "OLLAMA_API_BASE_URL", "http://localhost:11434/api"
 )
 
-if ENV == "prod":
-    if OLLAMA_API_BASE_URL == "/ollama/api":
-        OLLAMA_API_BASE_URL = "http://host.docker.internal:11434/api"
-
-
 OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "")
 
-if OLLAMA_BASE_URL == "":
+
+if OLLAMA_BASE_URL == "" and OLLAMA_API_BASE_URL != "":
     OLLAMA_BASE_URL = (
         OLLAMA_API_BASE_URL[:-4]
         if OLLAMA_API_BASE_URL.endswith("/api")
         else OLLAMA_API_BASE_URL
     )
 
+if ENV == "prod":
+    if OLLAMA_BASE_URL == "/ollama":
+        OLLAMA_BASE_URL = "http://host.docker.internal:11434"
+
+
+OLLAMA_BASE_URLS = os.environ.get("OLLAMA_BASE_URLS", "")
+OLLAMA_BASE_URLS = OLLAMA_BASE_URLS if OLLAMA_BASE_URLS != "" else OLLAMA_BASE_URL
+
+OLLAMA_BASE_URLS = [url.strip() for url in OLLAMA_BASE_URLS.split(";")]
+
 
 ####################################
 # OPENAI_API
@@ -229,9 +235,23 @@ if OLLAMA_BASE_URL == "":
 OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "")
 OPENAI_API_BASE_URL = os.environ.get("OPENAI_API_BASE_URL", "")
 
+
 if OPENAI_API_BASE_URL == "":
     OPENAI_API_BASE_URL = "https://api.openai.com/v1"
 
+OPENAI_API_KEYS = os.environ.get("OPENAI_API_KEYS", "")
+OPENAI_API_KEYS = OPENAI_API_KEYS if OPENAI_API_KEYS != "" else OPENAI_API_KEY
+
+OPENAI_API_KEYS = [url.strip() for url in OPENAI_API_KEYS.split(";")]
+
+
+OPENAI_API_BASE_URLS = os.environ.get("OPENAI_API_BASE_URLS", "")
+OPENAI_API_BASE_URLS = (
+    OPENAI_API_BASE_URLS if OPENAI_API_BASE_URLS != "" else OPENAI_API_BASE_URL
+)
+
+OPENAI_API_BASE_URLS = [url.strip() for url in OPENAI_API_BASE_URLS.split(";")]
+
 
 ####################################
 # WEBUI
@@ -271,6 +291,11 @@ DEFAULT_USER_ROLE = os.getenv("DEFAULT_USER_ROLE", "pending")
 USER_PERMISSIONS = {"chat": {"deletion": True}}
 
 
+MODEL_FILTER_ENABLED = os.environ.get("MODEL_FILTER_ENABLED", False)
+MODEL_FILTER_LIST = os.environ.get("MODEL_FILTER_LIST", "")
+MODEL_FILTER_LIST = [model.strip() for model in MODEL_FILTER_LIST.split(";")]
+
+
 ####################################
 # WEBUI_VERSION
 ####################################

backend/constants.py

@@ -41,6 +41,7 @@ class ERROR_MESSAGES(str, Enum):
     NOT_FOUND = "We could not find what you're looking for :/"
     USER_NOT_FOUND = "We could not find what you're looking for :/"
     API_KEY_NOT_FOUND = "Oops! It looks like there's a hiccup. The API key is missing. Please make sure to provide a valid API key to access this feature."
+
     MALICIOUS = "Unusual activities detected, please try again in a few minutes."
 
     PANDOC_NOT_INSTALLED = "Pandoc is not installed on the server. Please contact your administrator for assistance."
@@ -48,3 +49,7 @@ class ERROR_MESSAGES(str, Enum):
         lambda err="": f"Invalid format. Please use the correct format{err if err else ''}"
     )
     RATE_LIMIT_EXCEEDED = "API rate limit exceeded"
+
+    MODEL_NOT_FOUND = lambda name="": f"Model '{name}' was not found"
+    OPENAI_NOT_FOUND = lambda name="": f"OpenAI API was not found"
+    OLLAMA_NOT_FOUND = "WebUI could not connect to Ollama"

backend/data/config.json

@@ -1,4 +1,5 @@
 {
+    "version": "0.0.1",
     "ui": {
         "prompt_suggestions": [
             {

backend/main.py

@@ -9,28 +9,38 @@ import requests
 from fastapi import FastAPI, Request, Depends, status
 from fastapi.staticfiles import StaticFiles
 from fastapi import HTTPException
-from fastapi.responses import JSONResponse
 from fastapi.middleware.wsgi import WSGIMiddleware
 from fastapi.middleware.cors import CORSMiddleware
 from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.middleware.base import BaseHTTPMiddleware
 
 
-from litellm.proxy.proxy_server import ProxyConfig, initialize
-from litellm.proxy.proxy_server import app as litellm_app
-
 from apps.ollama.main import app as ollama_app
 from apps.openai.main import app as openai_app
+from apps.litellm.main import app as litellm_app, startup as litellm_app_startup
 from apps.audio.main import app as audio_app
 from apps.images.main import app as images_app
 from apps.rag.main import app as rag_app
 from apps.web.main import app as webui_app
 
+from pydantic import BaseModel
+from typing import List
 
-from config import WEBUI_NAME, ENV, VERSION, CHANGELOG, FRONTEND_BUILD_DIR
+
+from utils.utils import get_admin_user
+from apps.rag.utils import rag_messages
+
+from config import (
+    WEBUI_NAME,
+    ENV,
+    VERSION,
+    CHANGELOG,
+    FRONTEND_BUILD_DIR,
+    MODEL_FILTER_ENABLED,
+    MODEL_FILTER_LIST,
+)
 from constants import ERROR_MESSAGES
 
-from utils.utils import get_http_authorization_cred, get_current_user
-
 
 class SPAStaticFiles(StaticFiles):
     async def get_response(self, path: str, scope):
@@ -43,25 +53,69 @@ class SPAStaticFiles(StaticFiles):
                 raise ex
 
 
-proxy_config = ProxyConfig()
-
-
-async def config():
-    router, model_list, general_settings = await proxy_config.load_config(
-        router=None, config_file_path="./data/litellm/config.yaml"
-    )
-
-    await initialize(config="./data/litellm/config.yaml", telemetry=False)
-
-
-async def startup():
-    await config()
-
-
 app = FastAPI(docs_url="/docs" if ENV == "dev" else None, redoc_url=None)
 
+app.state.MODEL_FILTER_ENABLED = MODEL_FILTER_ENABLED
+app.state.MODEL_FILTER_LIST = MODEL_FILTER_LIST
+
 origins = ["*"]
 
+
+class RAGMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        if request.method == "POST" and (
+            "/api/chat" in request.url.path or "/chat/completions" in request.url.path
+        ):
+            print(request.url.path)
+
+            # Read the original request body
+            body = await request.body()
+            # Decode body to string
+            body_str = body.decode("utf-8")
+            # Parse string to JSON
+            data = json.loads(body_str) if body_str else {}
+
+            # Example: Add a new key-value pair or modify existing ones
+            # data["modified"] = True  # Example modification
+            if "docs" in data:
+
+                data = {**data}
+                data["messages"] = rag_messages(
+                    data["docs"],
+                    data["messages"],
+                    rag_app.state.RAG_TEMPLATE,
+                    rag_app.state.TOP_K,
+                    rag_app.state.sentence_transformer_ef,
+                )
+                del data["docs"]
+
+                print(data["messages"])
+
+            modified_body_bytes = json.dumps(data).encode("utf-8")
+
+            # Replace the request body with the modified one
+            request._body = modified_body_bytes
+
+            # Set custom header to ensure content-length matches new body length
+            request.headers.__dict__["_list"] = [
+                (b"content-length", str(len(modified_body_bytes)).encode("utf-8")),
+                *[
+                    (k, v)
+                    for k, v in request.headers.raw
+                    if k.lower() != b"content-length"
+                ],
+            ]
+
+        response = await call_next(request)
+        return response
+
+    async def _receive(self, body: bytes):
+        return {"type": "http.request", "body": body, "more_body": False}
+
+
+app.add_middleware(RAGMiddleware)
+
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=origins,
@@ -71,11 +125,6 @@ app.add_middleware(
 )
 
 
-@app.on_event("startup")
-async def on_startup():
-    await startup()
-
-
 @app.middleware("http")
 async def check_url(request: Request, call_next):
     start_time = int(time.time())
@@ -86,25 +135,15 @@ async def check_url(request: Request, call_next):
     return response
 
 
-@litellm_app.middleware("http")
-async def auth_middleware(request: Request, call_next):
-    auth_header = request.headers.get("Authorization", "")
-
-    if ENV != "dev":
-        try:
-            user = get_current_user(get_http_authorization_cred(auth_header))
-            print(user)
-        except Exception as e:
-            return JSONResponse(status_code=400, content={"detail": str(e)})
-
-    response = await call_next(request)
-    return response
+@app.on_event("startup")
+async def on_startup():
+    await litellm_app_startup()
 
 
 app.mount("/api/v1", webui_app)
 app.mount("/litellm/api", litellm_app)
 
-app.mount("/ollama/api", ollama_app)
+app.mount("/ollama", ollama_app)
 app.mount("/openai/api", openai_app)
 
 app.mount("/images/api/v1", images_app)
@@ -125,6 +164,47 @@ async def get_app_config():
     }
 
 
+@app.get("/api/config/model/filter")
+async def get_model_filter_config(user=Depends(get_admin_user)):
+    return {
+        "enabled": app.state.MODEL_FILTER_ENABLED,
+        "models": app.state.MODEL_FILTER_LIST,
+    }
+
+
+class ModelFilterConfigForm(BaseModel):
+    enabled: bool
+    models: List[str]
+
+
+@app.post("/api/config/model/filter")
+async def get_model_filter_config(
+    form_data: ModelFilterConfigForm, user=Depends(get_admin_user)
+):
+
+    app.state.MODEL_FILTER_ENABLED = form_data.enabled
+    app.state.MODEL_FILTER_LIST = form_data.models
+
+    ollama_app.state.MODEL_FILTER_ENABLED = app.state.MODEL_FILTER_ENABLED
+    ollama_app.state.MODEL_FILTER_LIST = app.state.MODEL_FILTER_LIST
+
+    openai_app.state.MODEL_FILTER_ENABLED = app.state.MODEL_FILTER_ENABLED
+    openai_app.state.MODEL_FILTER_LIST = app.state.MODEL_FILTER_LIST
+
+    return {
+        "enabled": app.state.MODEL_FILTER_ENABLED,
+        "models": app.state.MODEL_FILTER_LIST,
+    }
+
+
+@app.get("/api/version")
+async def get_app_config():
+
+    return {
+        "version": VERSION,
+    }
+
+
 @app.get("/api/changelog")
 async def get_app_changelog():
     return CHANGELOG
@@ -148,6 +228,7 @@ async def get_app_latest_release_version():
 
 
 app.mount("/static", StaticFiles(directory="static"), name="static")
+app.mount("/cache", StaticFiles(directory="data/cache"), name="cache")
 
 
 app.mount(

backend/requirements.txt

@@ -16,12 +16,14 @@ aiohttp
 peewee
 bcrypt
 
-litellm
+litellm==1.30.7
+argon2-cffi
 apscheduler
 google-generativeai
 
 langchain
 langchain-community
+fake_useragent
 chromadb
 sentence_transformers
 pypdf
@@ -34,6 +36,9 @@ openpyxl
 pyxlsb
 xlrd
 
+opencv-python-headless
+rapidocr-onnxruntime
+
 faster-whisper
 
 PyJWT

docker-compose.yaml

@@ -14,7 +14,7 @@ services:
     build:
       context: .
       args:
-        OLLAMA_API_BASE_URL: '/ollama/api'
+        OLLAMA_BASE_URL: '/ollama'
       dockerfile: Dockerfile
     image: ghcr.io/open-webui/open-webui:main
     container_name: open-webui
@@ -25,7 +25,7 @@ services:
     ports:
       - ${OPEN_WEBUI_PORT-3000}:8080
     environment:
-      - 'OLLAMA_API_BASE_URL=http://ollama:11434/api'
+      - 'OLLAMA_BASE_URL=http://ollama:11434'
       - 'WEBUI_SECRET_KEY='
     extra_hosts:
       - host.docker.internal:host-gateway

kubernetes/helm/templates/webui-deployment.yaml

@@ -40,7 +40,7 @@ spec:
         - name: data
           mountPath: /app/backend/data
         env:
-        - name: OLLAMA_API_BASE_URL
+        - name: OLLAMA_BASE_URL
           value: {{ include "ollama.url" . | quote }}
         tty: true
       {{- with .Values.webui.nodeSelector }}

kubernetes/manifest/base/webui-deployment.yaml

@@ -26,8 +26,8 @@ spec:
             cpu: "1000m"
             memory: "1Gi"
         env:
-        - name: OLLAMA_API_BASE_URL
-          value: "http://ollama-service.open-webui.svc.cluster.local:11434/api"
+        - name: OLLAMA_BASE_URL
+          value: "http://ollama-service.open-webui.svc.cluster.local:11434"
         tty: true
         volumeMounts:
         - name: webui-volume

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "open-webui",
-	"version": "0.1.108",
+	"version": "0.1.112",
 	"private": true,
 	"scripts": {
 		"dev": "vite dev --host",

src/app.css

@@ -43,6 +43,10 @@ ol > li {
 	font-weight: 400;
 }
 
+li p {
+	display: inline;
+}
+
 ::-webkit-scrollbar-thumb {
 	--tw-border-opacity: 1;
 	background-color: rgba(217, 217, 227, 0.8);

src/lib/apis/images/index.ts

@@ -1,9 +1,9 @@
 import { IMAGES_API_BASE_URL } from '$lib/constants';
 
-export const getImageGenerationEnabledStatus = async (token: string = '') => {
+export const getImageGenerationConfig = async (token: string = '') => {
 	let error = null;
 
-	const res = await fetch(`${IMAGES_API_BASE_URL}/enabled`, {
+	const res = await fetch(`${IMAGES_API_BASE_URL}/config`, {
 		method: 'GET',
 		headers: {
 			Accept: 'application/json',
@@ -32,10 +32,50 @@ export const getImageGenerationEnabledStatus = async (token: string = '') => {
 	return res;
 };
 
-export const toggleImageGenerationEnabledStatus = async (token: string = '') => {
+export const updateImageGenerationConfig = async (
+	token: string = '',
+	engine: string,
+	enabled: boolean
+) => {
 	let error = null;
 
-	const res = await fetch(`${IMAGES_API_BASE_URL}/enabled/toggle`, {
+	const res = await fetch(`${IMAGES_API_BASE_URL}/config/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			...(token && { authorization: `Bearer ${token}` })
+		},
+		body: JSON.stringify({
+			engine,
+			enabled
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			if ('detail' in err) {
+				error = err.detail;
+			} else {
+				error = 'Server connection failed';
+			}
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getOpenAIKey = async (token: string = '') => {
+	let error = null;
+
+	const res = await fetch(`${IMAGES_API_BASE_URL}/key`, {
 		method: 'GET',
 		headers: {
 			Accept: 'application/json',
@@ -61,7 +101,42 @@ export const toggleImageGenerationEnabledStatus = async (token: string = '') =>
 		throw error;
 	}
 
-	return res;
+	return res.OPENAI_API_KEY;
+};
+
+export const updateOpenAIKey = async (token: string = '', key: string) => {
+	let error = null;
+
+	const res = await fetch(`${IMAGES_API_BASE_URL}/key/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			...(token && { authorization: `Bearer ${token}` })
+		},
+		body: JSON.stringify({
+			key: key
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			if ('detail' in err) {
+				error = err.detail;
+			} else {
+				error = 'Server connection failed';
+			}
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res.OPENAI_API_KEY;
 };
 
 export const getAUTOMATIC1111Url = async (token: string = '') => {
@@ -263,7 +338,7 @@ export const updateImageSteps = async (token: string = '', steps: number) => {
 	return res.IMAGE_STEPS;
 };
 
-export const getDiffusionModels = async (token: string = '') => {
+export const getImageGenerationModels = async (token: string = '') => {
 	let error = null;
 
 	const res = await fetch(`${IMAGES_API_BASE_URL}/models`, {
@@ -295,7 +370,7 @@ export const getDiffusionModels = async (token: string = '') => {
 	return res;
 };
 
-export const getDefaultDiffusionModel = async (token: string = '') => {
+export const getDefaultImageGenerationModel = async (token: string = '') => {
 	let error = null;
 
 	const res = await fetch(`${IMAGES_API_BASE_URL}/models/default`, {
@@ -327,7 +402,7 @@ export const getDefaultDiffusionModel = async (token: string = '') => {
 	return res.model;
 };
 
-export const updateDefaultDiffusionModel = async (token: string = '', model: string) => {
+export const updateDefaultImageGenerationModel = async (token: string = '', model: string) => {
 	let error = null;
 
 	const res = await fetch(`${IMAGES_API_BASE_URL}/models/default/update`, {

src/lib/apis/index.ts

@@ -77,3 +77,65 @@ export const getVersionUpdates = async () => {
 
 	return res;
 };
+
+export const getModelFilterConfig = async (token: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_BASE_URL}/api/config/model/filter`, {
+		method: 'GET',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			error = err;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateModelFilterConfig = async (
+	token: string,
+	enabled: boolean,
+	models: string[]
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_BASE_URL}/api/config/model/filter`, {
+		method: 'POST',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			enabled: enabled,
+			models: models
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			error = err;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};

src/lib/apis/litellm/index.ts

@@ -77,6 +77,7 @@ type AddLiteLLMModelForm = {
 	api_base: string;
 	api_key: string;
 	rpm: string;
+	max_tokens: string;
 };
 
 export const addLiteLLMModel = async (token: string = '', payload: AddLiteLLMModelForm) => {
@@ -95,7 +96,8 @@ export const addLiteLLMModel = async (token: string = '', payload: AddLiteLLMMod
 				model: payload.model,
 				...(payload.api_base === '' ? {} : { api_base: payload.api_base }),
 				...(payload.api_key === '' ? {} : { api_key: payload.api_key }),
-				...(isNaN(parseInt(payload.rpm)) ? {} : { rpm: parseInt(payload.rpm) })
+				...(isNaN(parseInt(payload.rpm)) ? {} : { rpm: parseInt(payload.rpm) }),
+				...(payload.max_tokens === '' ? {} : { max_tokens: payload.max_tokens })
 			}
 		})
 	})

src/lib/apis/ollama/index.ts

@@ -1,9 +1,9 @@
 import { OLLAMA_API_BASE_URL } from '$lib/constants';
 
-export const getOllamaAPIUrl = async (token: string = '') => {
+export const getOllamaUrls = async (token: string = '') => {
 	let error = null;
 
-	const res = await fetch(`${OLLAMA_API_BASE_URL}/url`, {
+	const res = await fetch(`${OLLAMA_API_BASE_URL}/urls`, {
 		method: 'GET',
 		headers: {
 			Accept: 'application/json',
@@ -29,13 +29,13 @@ export const getOllamaAPIUrl = async (token: string = '') => {
 		throw error;
 	}
 
-	return res.OLLAMA_BASE_URL;
+	return res.OLLAMA_BASE_URLS;
 };
 
-export const updateOllamaAPIUrl = async (token: string = '', url: string) => {
+export const updateOllamaUrls = async (token: string = '', urls: string[]) => {
 	let error = null;
 
-	const res = await fetch(`${OLLAMA_API_BASE_URL}/url/update`, {
+	const res = await fetch(`${OLLAMA_API_BASE_URL}/urls/update`, {
 		method: 'POST',
 		headers: {
 			Accept: 'application/json',
@@ -43,7 +43,7 @@ export const updateOllamaAPIUrl = async (token: string = '', url: string) => {
 			...(token && { authorization: `Bearer ${token}` })
 		},
 		body: JSON.stringify({
-			url: url
+			urls: urls
 		})
 	})
 		.then(async (res) => {
@@ -64,7 +64,7 @@ export const updateOllamaAPIUrl = async (token: string = '', url: string) => {
 		throw error;
 	}
 
-	return res.OLLAMA_BASE_URL;
+	return res.OLLAMA_BASE_URLS;
 };
 
 export const getOllamaVersion = async (token: string = '') => {
@@ -151,7 +151,8 @@ export const generateTitle = async (
 	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/generate`, {
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
@@ -189,7 +190,8 @@ export const generatePrompt = async (token: string = '', model: string, conversa
 	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/generate`, {
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
@@ -223,7 +225,8 @@ export const generateTextCompletion = async (token: string = '', model: string,
 	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/generate`, {
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
@@ -251,7 +254,8 @@ export const generateChatCompletion = async (token: string = '', body: object) =
 		signal: controller.signal,
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify(body)
@@ -294,7 +298,8 @@ export const createModel = async (token: string, tagName: string, content: strin
 	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/create`, {
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
@@ -313,19 +318,23 @@ export const createModel = async (token: string, tagName: string, content: strin
 	return res;
 };
 
-export const deleteModel = async (token: string, tagName: string) => {
+export const deleteModel = async (token: string, tagName: string, urlIdx: string | null = null) => {
 	let error = null;
 
-	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/delete`, {
-		method: 'DELETE',
-		headers: {
-			'Content-Type': 'text/event-stream',
-			Authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			name: tagName
-		})
-	})
+	const res = await fetch(
+		`${OLLAMA_API_BASE_URL}/api/delete${urlIdx !== null ? `/${urlIdx}` : ''}`,
+		{
+			method: 'DELETE',
+			headers: {
+				Accept: 'application/json',
+				'Content-Type': 'application/json',
+				Authorization: `Bearer ${token}`
+			},
+			body: JSON.stringify({
+				name: tagName
+			})
+		}
+	)
 		.then(async (res) => {
 			if (!res.ok) throw await res.json();
 			return res.json();
@@ -336,7 +345,12 @@ export const deleteModel = async (token: string, tagName: string) => {
 		})
 		.catch((err) => {
 			console.log(err);
-			error = err.error;
+			error = err;
+
+			if ('detail' in err) {
+				error = err.detail;
+			}
+
 			return null;
 		});
 
@@ -347,13 +361,14 @@ export const deleteModel = async (token: string, tagName: string) => {
 	return res;
 };
 
-export const pullModel = async (token: string, tagName: string) => {
+export const pullModel = async (token: string, tagName: string, urlIdx: string | null = null) => {
 	let error = null;
 
-	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/pull`, {
+	const res = await fetch(`${OLLAMA_API_BASE_URL}/api/pull${urlIdx !== null ? `/${urlIdx}` : ''}`, {
 		method: 'POST',
 		headers: {
-			'Content-Type': 'text/event-stream',
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({

src/lib/apis/openai/index.ts

@@ -1,9 +1,9 @@
 import { OPENAI_API_BASE_URL } from '$lib/constants';
 
-export const getOpenAIUrl = async (token: string = '') => {
+export const getOpenAIUrls = async (token: string = '') => {
 	let error = null;
 
-	const res = await fetch(`${OPENAI_API_BASE_URL}/url`, {
+	const res = await fetch(`${OPENAI_API_BASE_URL}/urls`, {
 		method: 'GET',
 		headers: {
 			Accept: 'application/json',
@@ -29,13 +29,13 @@ export const getOpenAIUrl = async (token: string = '') => {
 		throw error;
 	}
 
-	return res.OPENAI_API_BASE_URL;
+	return res.OPENAI_API_BASE_URLS;
 };
 
-export const updateOpenAIUrl = async (token: string = '', url: string) => {
+export const updateOpenAIUrls = async (token: string = '', urls: string[]) => {
 	let error = null;
 
-	const res = await fetch(`${OPENAI_API_BASE_URL}/url/update`, {
+	const res = await fetch(`${OPENAI_API_BASE_URL}/urls/update`, {
 		method: 'POST',
 		headers: {
 			Accept: 'application/json',
@@ -43,7 +43,7 @@ export const updateOpenAIUrl = async (token: string = '', url: string) => {
 			...(token && { authorization: `Bearer ${token}` })
 		},
 		body: JSON.stringify({
-			url: url
+			urls: urls
 		})
 	})
 		.then(async (res) => {
@@ -64,13 +64,13 @@ export const updateOpenAIUrl = async (token: string = '', url: string) => {
 		throw error;
 	}
 
-	return res.OPENAI_API_BASE_URL;
+	return res.OPENAI_API_BASE_URLS;
 };
 
-export const getOpenAIKey = async (token: string = '') => {
+export const getOpenAIKeys = async (token: string = '') => {
 	let error = null;
 
-	const res = await fetch(`${OPENAI_API_BASE_URL}/key`, {
+	const res = await fetch(`${OPENAI_API_BASE_URL}/keys`, {
 		method: 'GET',
 		headers: {
 			Accept: 'application/json',
@@ -96,13 +96,13 @@ export const getOpenAIKey = async (token: string = '') => {
 		throw error;
 	}
 
-	return res.OPENAI_API_KEY;
+	return res.OPENAI_API_KEYS;
 };
 
-export const updateOpenAIKey = async (token: string = '', key: string) => {
+export const updateOpenAIKeys = async (token: string = '', keys: string[]) => {
 	let error = null;
 
-	const res = await fetch(`${OPENAI_API_BASE_URL}/key/update`, {
+	const res = await fetch(`${OPENAI_API_BASE_URL}/keys/update`, {
 		method: 'POST',
 		headers: {
 			Accept: 'application/json',
@@ -110,7 +110,7 @@ export const updateOpenAIKey = async (token: string = '', key: string) => {
 			...(token && { authorization: `Bearer ${token}` })
 		},
 		body: JSON.stringify({
-			key: key
+			keys: keys
 		})
 	})
 		.then(async (res) => {
@@ -131,7 +131,7 @@ export const updateOpenAIKey = async (token: string = '', key: string) => {
 		throw error;
 	}
 
-	return res.OPENAI_API_KEY;
+	return res.OPENAI_API_KEYS;
 };
 
 export const getOpenAIModels = async (token: string = '') => {

src/lib/apis/rag/index.ts

@@ -1,9 +1,9 @@
 import { RAG_API_BASE_URL } from '$lib/constants';
 
-export const getChunkParams = async (token: string) => {
+export const getRAGConfig = async (token: string) => {
 	let error = null;
 
-	const res = await fetch(`${RAG_API_BASE_URL}/chunk`, {
+	const res = await fetch(`${RAG_API_BASE_URL}/config`, {
 		method: 'GET',
 		headers: {
 			'Content-Type': 'application/json',
@@ -27,18 +27,27 @@ export const getChunkParams = async (token: string) => {
 	return res;
 };
 
-export const updateChunkParams = async (token: string, size: number, overlap: number) => {
+type ChunkConfigForm = {
+	chunk_size: number;
+	chunk_overlap: number;
+};
+
+type RAGConfigForm = {
+	pdf_extract_images: boolean;
+	chunk: ChunkConfigForm;
+};
+
+export const updateRAGConfig = async (token: string, payload: RAGConfigForm) => {
 	let error = null;
 
-	const res = await fetch(`${RAG_API_BASE_URL}/chunk/update`, {
+	const res = await fetch(`${RAG_API_BASE_URL}/config/update`, {
 		method: 'POST',
 		headers: {
 			'Content-Type': 'application/json',
 			Authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
-			chunk_size: size,
-			chunk_overlap: overlap
+			...payload
 		})
 	})
 		.then(async (res) => {
@@ -252,7 +261,7 @@ export const queryCollection = async (
 	token: string,
 	collection_names: string,
 	query: string,
-	k: number
+	k: number | null = null
 ) => {
 	let error = null;
 

src/lib/components/admin/Settings/Users.svelte

@@ -1,10 +1,14 @@
 <script lang="ts">
+	import { getModelFilterConfig, updateModelFilterConfig } from '$lib/apis';
 	import { getSignUpEnabledStatus, toggleSignUpEnabledStatus } from '$lib/apis/auths';
 	import { getUserPermissions, updateUserPermissions } from '$lib/apis/users';
+	import { models } from '$lib/stores';
 	import { onMount } from 'svelte';
 
 	export let saveHandler: Function;
 
+	let whitelistEnabled = false;
+	let whitelistModels = [''];
 	let permissions = {
 		chat: {
 			deletion: true
@@ -13,6 +17,13 @@
 
 	onMount(async () => {
 		permissions = await getUserPermissions(localStorage.token);
+
+		const res = await getModelFilterConfig(localStorage.token);
+		if (res) {
+			whitelistEnabled = res.enabled;
+
+			whitelistModels = res.models.length > 0 ? res.models : [''];
+		}
 	});
 </script>
 
@@ -21,6 +32,8 @@
 	on:submit|preventDefault={async () => {
 		// console.log('submit');
 		await updateUserPermissions(localStorage.token, permissions);
+
+		await updateModelFilterConfig(localStorage.token, whitelistEnabled, whitelistModels);
 		saveHandler();
 	}}
 >
@@ -69,6 +82,106 @@
 				</button>
 			</div>
 		</div>
+
+		<hr class=" dark:border-gray-700 my-2" />
+
+		<div class="mt-2 space-y-3 pr-1.5">
+			<div>
+				<div class="mb-2">
+					<div class="flex justify-between items-center text-xs">
+						<div class=" text-sm font-medium">Manage Models</div>
+					</div>
+				</div>
+
+				<div class=" space-y-3">
+					<div>
+						<div class="flex justify-between items-center text-xs">
+							<div class=" text-xs font-medium">Model Whitelisting</div>
+
+							<button
+								class=" text-xs font-medium text-gray-500"
+								type="button"
+								on:click={() => {
+									whitelistEnabled = !whitelistEnabled;
+								}}>{whitelistEnabled ? 'On' : 'Off'}</button
+							>
+						</div>
+					</div>
+
+					{#if whitelistEnabled}
+						<div>
+							<div class=" space-y-1.5">
+								{#each whitelistModels as modelId, modelIdx}
+									<div class="flex w-full">
+										<div class="flex-1 mr-2">
+											<select
+												class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+												bind:value={modelId}
+												placeholder="Select a model"
+											>
+												<option value="" disabled selected>Select a model</option>
+												{#each $models.filter((model) => model.id) as model}
+													<option value={model.id} class="bg-gray-100 dark:bg-gray-700"
+														>{model.name}</option
+													>
+												{/each}
+											</select>
+										</div>
+
+										{#if modelIdx === 0}
+											<button
+												class="px-2.5 bg-gray-100 hover:bg-gray-200 text-gray-800 dark:bg-gray-900 dark:text-white rounded-lg transition"
+												type="button"
+												on:click={() => {
+													if (whitelistModels.at(-1) !== '') {
+														whitelistModels = [...whitelistModels, ''];
+													}
+												}}
+											>
+												<svg
+													xmlns="http://www.w3.org/2000/svg"
+													viewBox="0 0 16 16"
+													fill="currentColor"
+													class="w-4 h-4"
+												>
+													<path
+														d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+													/>
+												</svg>
+											</button>
+										{:else}
+											<button
+												class="px-2.5 bg-gray-100 hover:bg-gray-200 text-gray-800 dark:bg-gray-900 dark:text-white rounded-lg transition"
+												type="button"
+												on:click={() => {
+													whitelistModels.splice(modelIdx, 1);
+													whitelistModels = whitelistModels;
+												}}
+											>
+												<svg
+													xmlns="http://www.w3.org/2000/svg"
+													viewBox="0 0 16 16"
+													fill="currentColor"
+													class="w-4 h-4"
+												>
+													<path d="M3.75 7.25a.75.75 0 0 0 0 1.5h8.5a.75.75 0 0 0 0-1.5h-8.5Z" />
+												</svg>
+											</button>
+										{/if}
+									</div>
+								{/each}
+							</div>
+
+							<div class="flex justify-end items-center text-xs mt-1.5 text-right">
+								<div class=" text-xs font-medium">
+									{whitelistModels.length} Model(s) Whitelisted
+								</div>
+							</div>
+						</div>
+					{/if}
+				</div>
+			</div>
+		</div>
 	</div>
 
 	<div class="flex justify-end pt-3 text-sm font-medium">

src/lib/components/chat/MessageInput.svelte

@@ -19,7 +19,7 @@
 
 	export let suggestionPrompts = [];
 	export let autoScroll = true;
-
+	let chatTextAreaElement: HTMLTextAreaElement;
 	let filesInputElement;
 
 	let promptsElement;
@@ -43,11 +43,9 @@
 	let speechRecognition;
 
 	$: if (prompt) {
-		const chatInput = document.getElementById('chat-textarea');
-
-		if (chatInput) {
-			chatInput.style.height = '';
-			chatInput.style.height = Math.min(chatInput.scrollHeight, 200) + 'px';
+		if (chatTextAreaElement) {
+			chatTextAreaElement.style.height = '';
+			chatTextAreaElement.style.height = Math.min(chatTextAreaElement.scrollHeight, 200) + 'px';
 		}
 	}
 
@@ -86,9 +84,7 @@
 			if (res) {
 				prompt = res.text;
 				await tick();
-
-				const inputElement = document.getElementById('chat-textarea');
-				inputElement?.focus();
+				chatTextAreaElement?.focus();
 
 				if (prompt !== '' && $settings?.speechAutoSend === true) {
 					submitPrompt(prompt, user);
@@ -191,8 +187,7 @@
 						prompt = `${prompt}${transcript}`;
 
 						await tick();
-						const inputElement = document.getElementById('chat-textarea');
-						inputElement?.focus();
+						chatTextAreaElement?.focus();
 
 						// Restart the inactivity timeout
 						timeoutId = setTimeout(() => {
@@ -294,8 +289,7 @@
 	};
 
 	onMount(() => {
-		const chatInput = document.getElementById('chat-textarea');
-		window.setTimeout(() => chatInput?.focus(), 0);
+		window.setTimeout(() => chatTextAreaElement?.focus(), 0);
 
 		const dropZone = document.querySelector('body');
 
@@ -365,12 +359,12 @@
 
 {#if dragged}
 	<div
-		class="fixed w-full h-full flex z-50 touch-none pointer-events-none"
+		class="fixed lg:w-[calc(100%-260px)] w-full h-full flex z-50 touch-none pointer-events-none"
 		id="dropzone"
 		role="region"
 		aria-label="Drag and Drop Container"
 	>
-		<div class="absolute rounded-xl w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
+		<div class="absolute w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
 			<div class="m-auto pt-64 flex flex-col justify-center">
 				<div class="max-w-md">
 					<AddFilesPlaceholder />
@@ -663,6 +657,7 @@
 
 						<textarea
 							id="chat-textarea"
+							bind:this={chatTextAreaElement}
 							class=" dark:bg-gray-900 dark:text-gray-100 outline-none w-full py-3 px-3 {fileUploadEnabled
 								? ''
 								: ' pl-4'} rounded-xl resize-none h-[48px]"

src/lib/components/chat/Messages.svelte

@@ -223,33 +223,80 @@
 		}, 100);
 	};
 
-	// TODO: change delete behaviour
-	// const deleteMessageAndDescendants = async (messageId: string) => {
-	// 	if (history.messages[messageId]) {
-	// 		history.messages[messageId].deleted = true;
-
-	// 		for (const childId of history.messages[messageId].childrenIds) {
-	// 			await deleteMessageAndDescendants(childId);
-	// 		}
-	// 	}
-	// };
-
-	// const triggerDeleteMessageRecursive = async (messageId: string) => {
-	// 	await deleteMessageAndDescendants(messageId);
-	// 	await updateChatById(localStorage.token, chatId, { history });
-	// 	await chats.set(await getChatList(localStorage.token));
-	// };
-
 	const messageDeleteHandler = async (messageId) => {
-		if (history.messages[messageId]) {
-			history.messages[messageId].deleted = true;
-
-			for (const childId of history.messages[messageId].childrenIds) {
-				history.messages[childId].deleted = true;
+		const messageToDelete = history.messages[messageId];
+		const messageParentId = messageToDelete.parentId;
+		const messageChildrenIds = messageToDelete.childrenIds ?? [];
+		const hasSibling = messageChildrenIds.some(
+			(childId) => history.messages[childId]?.childrenIds?.length > 0
+		);
+		messageChildrenIds.forEach((childId) => {
+			const child = history.messages[childId];
+			if (child && child.childrenIds) {
+				if (child.childrenIds.length === 0 && !hasSibling) {
+					// if last prompt/response pair
+					history.messages[messageParentId].childrenIds = [];
+					history.currentId = messageParentId;
+				} else {
+					child.childrenIds.forEach((grandChildId) => {
+						if (history.messages[grandChildId]) {
+							history.messages[grandChildId].parentId = messageParentId;
+							history.messages[messageParentId].childrenIds.push(grandChildId);
+						}
+					});
+				}
 			}
-		}
-		await updateChatById(localStorage.token, chatId, { history });
+			// remove response
+			history.messages[messageParentId].childrenIds = history.messages[
+				messageParentId
+			].childrenIds.filter((id) => id !== childId);
+		});
+		// remove prompt
+		history.messages[messageParentId].childrenIds = history.messages[
+			messageParentId
+		].childrenIds.filter((id) => id !== messageId);
+		await updateChatById(localStorage.token, chatId, {
+			messages: messages,
+			history: history
+		});
 	};
+
+	// const messageDeleteHandler = async (messageId) => {
+	// 	const message = history.messages[messageId];
+	// 	const parentId = message.parentId;
+	// 	const childrenIds = message.childrenIds ?? [];
+	// 	const grandchildrenIds = [];
+
+	// 	// Iterate through childrenIds to find grandchildrenIds
+	// 	for (const childId of childrenIds) {
+	// 		const childMessage = history.messages[childId];
+	// 		const grandChildrenIds = childMessage.childrenIds ?? [];
+
+	// 		for (const grandchildId of grandchildrenIds) {
+	// 			const childMessage = history.messages[grandchildId];
+	// 			childMessage.parentId = parentId;
+	// 		}
+	// 		grandchildrenIds.push(...grandChildrenIds);
+	// 	}
+
+	// 	history.messages[parentId].childrenIds.push(...grandchildrenIds);
+	// 	history.messages[parentId].childrenIds = history.messages[parentId].childrenIds.filter(
+	// 		(id) => id !== messageId
+	// 	);
+
+	// 	// Select latest message
+	// 	let currentMessageId = grandchildrenIds.at(-1);
+	// 	if (currentMessageId) {
+	// 		let messageChildrenIds = history.messages[currentMessageId].childrenIds;
+	// 		while (messageChildrenIds.length !== 0) {
+	// 			currentMessageId = messageChildrenIds.at(-1);
+	// 			messageChildrenIds = history.messages[currentMessageId].childrenIds;
+	// 		}
+	// 		history.currentId = currentMessageId;
+	// 	}
+
+	// 	await updateChatById(localStorage.token, chatId, { messages, history });
+	// };
 </script>
 
 {#if messages.length == 0}
@@ -258,57 +305,55 @@
 	<div class=" pb-10">
 		{#key chatId}
 			{#each messages as message, messageIdx}
-				{#if !message.deleted}
-					<div class=" w-full">
-						<div
-							class="flex flex-col justify-between px-5 mb-3 {$settings?.fullScreenMode ?? null
-								? 'max-w-full'
-								: 'max-w-3xl'} mx-auto rounded-lg group"
-						>
-							{#if message.role === 'user'}
-								<UserMessage
-									on:delete={() => messageDeleteHandler(message.id)}
-									user={$user}
-									{message}
-									isFirstMessage={messageIdx === 0}
-									siblings={message.parentId !== null
-										? history.messages[message.parentId]?.childrenIds ?? []
-										: Object.values(history.messages)
-												.filter((message) => message.parentId === null)
-												.map((message) => message.id) ?? []}
-									{confirmEditMessage}
-									{showPreviousMessage}
-									{showNextMessage}
-									{copyToClipboard}
-								/>
-							{:else}
-								<ResponseMessage
-									{message}
-									modelfiles={selectedModelfiles}
-									siblings={history.messages[message.parentId]?.childrenIds ?? []}
-									isLastMessage={messageIdx + 1 === messages.length}
-									{confirmEditResponseMessage}
-									{showPreviousMessage}
-									{showNextMessage}
-									{rateMessage}
-									{copyToClipboard}
-									{continueGeneration}
-									{regenerateResponse}
-									on:save={async (e) => {
-										console.log('save', e);
+				<div class=" w-full">
+					<div
+						class="flex flex-col justify-between px-5 mb-3 {$settings?.fullScreenMode ?? null
+							? 'max-w-full'
+							: 'max-w-3xl'} mx-auto rounded-lg group"
+					>
+						{#if message.role === 'user'}
+							<UserMessage
+								on:delete={() => messageDeleteHandler(message.id)}
+								user={$user}
+								{message}
+								isFirstMessage={messageIdx === 0}
+								siblings={message.parentId !== null
+									? history.messages[message.parentId]?.childrenIds ?? []
+									: Object.values(history.messages)
+											.filter((message) => message.parentId === null)
+											.map((message) => message.id) ?? []}
+								{confirmEditMessage}
+								{showPreviousMessage}
+								{showNextMessage}
+								{copyToClipboard}
+							/>
+						{:else}
+							<ResponseMessage
+								{message}
+								modelfiles={selectedModelfiles}
+								siblings={history.messages[message.parentId]?.childrenIds ?? []}
+								isLastMessage={messageIdx + 1 === messages.length}
+								{confirmEditResponseMessage}
+								{showPreviousMessage}
+								{showNextMessage}
+								{rateMessage}
+								{copyToClipboard}
+								{continueGeneration}
+								{regenerateResponse}
+								on:save={async (e) => {
+									console.log('save', e);
 
-										const message = e.detail;
-										history.messages[message.id] = message;
-										await updateChatById(localStorage.token, chatId, {
-											messages: messages,
-											history: history
-										});
-									}}
-								/>
-							{/if}
-						</div>
+									const message = e.detail;
+									history.messages[message.id] = message;
+									await updateChatById(localStorage.token, chatId, {
+										messages: messages,
+										history: history
+									});
+								}}
+							/>
+						{/if}
 					</div>
-				{/if}
+				</div>
 			{/each}
 
 			{#if bottomPadding}

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -6,6 +6,7 @@
 	import auto_render from 'katex/dist/contrib/auto-render.mjs';
 	import 'katex/dist/katex.min.css';
 
+	import { fade } from 'svelte/transition';
 	import { createEventDispatcher } from 'svelte';
 	import { onMount, tick } from 'svelte';
 
@@ -22,6 +23,7 @@
 	import CodeBlock from './CodeBlock.svelte';
 	import Image from '$lib/components/common/Image.svelte';
 	import { WEBUI_BASE_URL } from '$lib/constants';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
 	export let modelfiles = [];
 	export let message;
@@ -40,7 +42,7 @@
 
 	let edit = false;
 	let editedContent = '';
-
+	let editTextAreaElement: HTMLTextAreaElement;
 	let tooltipInstance = null;
 
 	let sentencesAudio = {};
@@ -247,10 +249,9 @@
 		editedContent = message.content;
 
 		await tick();
-		const editElement = document.getElementById(`message-edit-${message.id}`);
 
-		editElement.style.height = '';
-		editElement.style.height = `${editElement.scrollHeight}px`;
+		editTextAreaElement.style.height = '';
+		editTextAreaElement.style.height = `${editTextAreaElement.scrollHeight}px`;
 	};
 
 	const editMessageConfirmHandler = async () => {
@@ -276,13 +277,15 @@
 
 	const generateImage = async (message) => {
 		generatingImage = true;
-		const res = await imageGenerations(localStorage.token, message.content);
+		const res = await imageGenerations(localStorage.token, message.content).catch((error) => {
+			toast.error(error);
+		});
 		console.log(res);
 
 		if (res) {
-			message.files = res.images.map((image) => ({
+			message.files = res.map((image) => ({
 				type: 'image',
-				url: `data:image/png;base64,${image}`
+				url: `${image.url}`
 			}));
 
 			dispatch('save', message);
@@ -341,9 +344,11 @@
 							<div class=" w-full">
 								<textarea
 									id="message-edit-{message.id}"
+									bind:this={editTextAreaElement}
 									class=" bg-transparent outline-none w-full resize-none"
 									bind:value={editedContent}
 									on:input={(e) => {
+										e.target.style.height = '';
 										e.target.style.height = `${e.target.scrollHeight}px`;
 									}}
 								/>
@@ -462,189 +467,125 @@
 											</div>
 										{/if}
 
-										<button
-											class="{isLastMessage
-												? 'visible'
-												: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition"
-											on:click={() => {
-												editMessageHandler();
-											}}
-										>
-											<svg
-												xmlns="http://www.w3.org/2000/svg"
-												fill="none"
-												viewBox="0 0 24 24"
-												stroke-width="1.5"
-												stroke="currentColor"
-												class="w-4 h-4"
-											>
-												<path
-													stroke-linecap="round"
-													stroke-linejoin="round"
-													d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
-												/>
-											</svg>
-										</button>
-
-										<button
-											class="{isLastMessage
-												? 'visible'
-												: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition copy-response-button"
-											on:click={() => {
-												copyToClipboard(message.content);
-											}}
-										>
-											<svg
-												xmlns="http://www.w3.org/2000/svg"
-												fill="none"
-												viewBox="0 0 24 24"
-												stroke-width="1.5"
-												stroke="currentColor"
-												class="w-4 h-4"
-											>
-												<path
-													stroke-linecap="round"
-													stroke-linejoin="round"
-													d="M15.666 3.888A2.25 2.25 0 0013.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 01-.75.75H9a.75.75 0 01-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 01-2.25 2.25H6.75A2.25 2.25 0 014.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 011.927-.184"
-												/>
-											</svg>
-										</button>
-
-										<button
-											class="{isLastMessage
-												? 'visible'
-												: 'invisible group-hover:visible'} p-1 rounded {message.rating === 1
-												? 'bg-gray-100 dark:bg-gray-800'
-												: ''} dark:hover:text-white hover:text-black transition"
-											on:click={() => {
-												rateMessage(message.id, 1);
-											}}
-										>
-											<svg
-												stroke="currentColor"
-												fill="none"
-												stroke-width="2"
-												viewBox="0 0 24 24"
-												stroke-linecap="round"
-												stroke-linejoin="round"
-												class="w-4 h-4"
-												xmlns="http://www.w3.org/2000/svg"
-												><path
-													d="M14 9V5a3 3 0 0 0-3-3l-4 9v11h11.28a2 2 0 0 0 2-1.7l1.38-9a2 2 0 0 0-2-2.3zM7 22H4a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2h3"
-												/></svg
-											>
-										</button>
-										<button
-											class="{isLastMessage
-												? 'visible'
-												: 'invisible group-hover:visible'} p-1 rounded {message.rating === -1
-												? 'bg-gray-100 dark:bg-gray-800'
-												: ''} dark:hover:text-white hover:text-black transition"
-											on:click={() => {
-												rateMessage(message.id, -1);
-											}}
-										>
-											<svg
-												stroke="currentColor"
-												fill="none"
-												stroke-width="2"
-												viewBox="0 0 24 24"
-												stroke-linecap="round"
-												stroke-linejoin="round"
-												class="w-4 h-4"
-												xmlns="http://www.w3.org/2000/svg"
-												><path
-													d="M10 15v4a3 3 0 0 0 3 3l4-9V2H5.72a2 2 0 0 0-2 1.7l-1.38 9a2 2 0 0 0 2 2.3zm7-13h2.67A2.31 2.31 0 0 1 22 4v7a2.31 2.31 0 0 1-2.33 2H17"
-												/></svg
-											>
-										</button>
-
-										<button
-											id="speak-button-{message.id}"
-											class="{isLastMessage
-												? 'visible'
-												: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition"
-											on:click={() => {
-												if (!loadingSpeech) {
-													toggleSpeakMessage(message);
-												}
-											}}
-										>
-											{#if loadingSpeech}
-												<svg
-													class=" w-4 h-4"
-													fill="currentColor"
-													viewBox="0 0 24 24"
-													xmlns="http://www.w3.org/2000/svg"
-													><style>
-														.spinner_S1WN {
-															animation: spinner_MGfb 0.8s linear infinite;
-															animation-delay: -0.8s;
-														}
-														.spinner_Km9P {
-															animation-delay: -0.65s;
-														}
-														.spinner_JApP {
-															animation-delay: -0.5s;
-														}
-														@keyframes spinner_MGfb {
-															93.75%,
-															100% {
-																opacity: 0.2;
-															}
-														}
-													</style><circle class="spinner_S1WN" cx="4" cy="12" r="3" /><circle
-														class="spinner_S1WN spinner_Km9P"
-														cx="12"
-														cy="12"
-														r="3"
-													/><circle class="spinner_S1WN spinner_JApP" cx="20" cy="12" r="3" /></svg
-												>
-											{:else if speaking}
-												<svg
-													xmlns="http://www.w3.org/2000/svg"
-													fill="none"
-													viewBox="0 0 24 24"
-													stroke-width="1.5"
-													stroke="currentColor"
-													class="w-4 h-4"
-												>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M17.25 9.75 19.5 12m0 0 2.25 2.25M19.5 12l2.25-2.25M19.5 12l-2.25 2.25m-10.5-6 4.72-4.72a.75.75 0 0 1 1.28.53v15.88a.75.75 0 0 1-1.28.53l-4.72-4.72H4.51c-.88 0-1.704-.507-1.938-1.354A9.009 9.009 0 0 1 2.25 12c0-.83.112-1.633.322-2.396C2.806 8.756 3.63 8.25 4.51 8.25H6.75Z"
-													/>
-												</svg>
-											{:else}
-												<svg
-													xmlns="http://www.w3.org/2000/svg"
-													fill="none"
-													viewBox="0 0 24 24"
-													stroke-width="1.5"
-													stroke="currentColor"
-													class="w-4 h-4"
-												>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M19.114 5.636a9 9 0 010 12.728M16.463 8.288a5.25 5.25 0 010 7.424M6.75 8.25l4.72-4.72a.75.75 0 011.28.53v15.88a.75.75 0 01-1.28.53l-4.72-4.72H4.51c-.88 0-1.704-.507-1.938-1.354A9.01 9.01 0 012.25 12c0-.83.112-1.633.322-2.396C2.806 8.756 3.63 8.25 4.51 8.25H6.75z"
-													/>
-												</svg>
-											{/if}
-										</button>
-
-										{#if $config.images}
+										<Tooltip content="Edit" placement="bottom">
 											<button
 												class="{isLastMessage
 													? 'visible'
 													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition"
 												on:click={() => {
-													if (!generatingImage) {
-														generateImage(message);
+													editMessageHandler();
+												}}
+											>
+												<svg
+													xmlns="http://www.w3.org/2000/svg"
+													fill="none"
+													viewBox="0 0 24 24"
+													stroke-width="2"
+													stroke="currentColor"
+													class="w-4 h-4"
+												>
+													<path
+														stroke-linecap="round"
+														stroke-linejoin="round"
+														d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
+													/>
+												</svg>
+											</button>
+										</Tooltip>
+
+										<Tooltip content="Copy" placement="bottom">
+											<button
+												class="{isLastMessage
+													? 'visible'
+													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition copy-response-button"
+												on:click={() => {
+													copyToClipboard(message.content);
+												}}
+											>
+												<svg
+													xmlns="http://www.w3.org/2000/svg"
+													fill="none"
+													viewBox="0 0 24 24"
+													stroke-width="2"
+													stroke="currentColor"
+													class="w-4 h-4"
+												>
+													<path
+														stroke-linecap="round"
+														stroke-linejoin="round"
+														d="M15.666 3.888A2.25 2.25 0 0013.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 01-.75.75H9a.75.75 0 01-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 01-2.25 2.25H6.75A2.25 2.25 0 014.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 011.927-.184"
+													/>
+												</svg>
+											</button>
+										</Tooltip>
+
+										<Tooltip content="Good Response" placement="bottom">
+											<button
+												class="{isLastMessage
+													? 'visible'
+													: 'invisible group-hover:visible'} p-1 rounded {message.rating === 1
+													? 'bg-gray-100 dark:bg-gray-800'
+													: ''} dark:hover:text-white hover:text-black transition"
+												on:click={() => {
+													rateMessage(message.id, 1);
+												}}
+											>
+												<svg
+													stroke="currentColor"
+													fill="none"
+													stroke-width="2"
+													viewBox="0 0 24 24"
+													stroke-linecap="round"
+													stroke-linejoin="round"
+													class="w-4 h-4"
+													xmlns="http://www.w3.org/2000/svg"
+													><path
+														d="M14 9V5a3 3 0 0 0-3-3l-4 9v11h11.28a2 2 0 0 0 2-1.7l1.38-9a2 2 0 0 0-2-2.3zM7 22H4a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2h3"
+													/></svg
+												>
+											</button>
+										</Tooltip>
+
+										<Tooltip content="Bad Response" placement="bottom">
+											<button
+												class="{isLastMessage
+													? 'visible'
+													: 'invisible group-hover:visible'} p-1 rounded {message.rating === -1
+													? 'bg-gray-100 dark:bg-gray-800'
+													: ''} dark:hover:text-white hover:text-black transition"
+												on:click={() => {
+													rateMessage(message.id, -1);
+												}}
+											>
+												<svg
+													stroke="currentColor"
+													fill="none"
+													stroke-width="2"
+													viewBox="0 0 24 24"
+													stroke-linecap="round"
+													stroke-linejoin="round"
+													class="w-4 h-4"
+													xmlns="http://www.w3.org/2000/svg"
+													><path
+														d="M10 15v4a3 3 0 0 0 3 3l4-9V2H5.72a2 2 0 0 0-2 1.7l-1.38 9a2 2 0 0 0 2 2.3zm7-13h2.67A2.31 2.31 0 0 1 22 4v7a2.31 2.31 0 0 1-2.33 2H17"
+													/></svg
+												>
+											</button>
+										</Tooltip>
+
+										<Tooltip content="Read Aloud" placement="bottom">
+											<button
+												id="speak-button-{message.id}"
+												class="{isLastMessage
+													? 'visible'
+													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition"
+												on:click={() => {
+													if (!loadingSpeech) {
+														toggleSpeakMessage(message);
 													}
 												}}
 											>
-												{#if generatingImage}
+												{#if loadingSpeech}
 													<svg
 														class=" w-4 h-4"
 														fill="currentColor"
@@ -679,105 +620,193 @@
 															r="3"
 														/></svg
 													>
-												{:else}
+												{:else if speaking}
 													<svg
 														xmlns="http://www.w3.org/2000/svg"
 														fill="none"
 														viewBox="0 0 24 24"
-														stroke-width="1.5"
+														stroke-width="2"
 														stroke="currentColor"
 														class="w-4 h-4"
 													>
 														<path
 															stroke-linecap="round"
 															stroke-linejoin="round"
-															d="m2.25 15.75 5.159-5.159a2.25 2.25 0 0 1 3.182 0l5.159 5.159m-1.5-1.5 1.409-1.409a2.25 2.25 0 0 1 3.182 0l2.909 2.909m-18 3.75h16.5a1.5 1.5 0 0 0 1.5-1.5V6a1.5 1.5 0 0 0-1.5-1.5H3.75A1.5 1.5 0 0 0 2.25 6v12a1.5 1.5 0 0 0 1.5 1.5Zm10.5-11.25h.008v.008h-.008V8.25Zm.375 0a.375.375 0 1 1-.75 0 .375.375 0 0 1 .75 0Z"
+															d="M17.25 9.75 19.5 12m0 0 2.25 2.25M19.5 12l2.25-2.25M19.5 12l-2.25 2.25m-10.5-6 4.72-4.72a.75.75 0 0 1 1.28.53v15.88a.75.75 0 0 1-1.28.53l-4.72-4.72H4.51c-.88 0-1.704-.507-1.938-1.354A9.009 9.009 0 0 1 2.25 12c0-.83.112-1.633.322-2.396C2.806 8.756 3.63 8.25 4.51 8.25H6.75Z"
+														/>
+													</svg>
+												{:else}
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														fill="none"
+														viewBox="0 0 24 24"
+														stroke-width="2"
+														stroke="currentColor"
+														class="w-4 h-4"
+													>
+														<path
+															stroke-linecap="round"
+															stroke-linejoin="round"
+															d="M19.114 5.636a9 9 0 010 12.728M16.463 8.288a5.25 5.25 0 010 7.424M6.75 8.25l4.72-4.72a.75.75 0 011.28.53v15.88a.75.75 0 01-1.28.53l-4.72-4.72H4.51c-.88 0-1.704-.507-1.938-1.354A9.01 9.01 0 012.25 12c0-.83.112-1.633.322-2.396C2.806 8.756 3.63 8.25 4.51 8.25H6.75z"
 														/>
 													</svg>
 												{/if}
 											</button>
+										</Tooltip>
+
+										{#if $config.images}
+											<Tooltip content="Generate Image" placement="bottom">
+												<button
+													class="{isLastMessage
+														? 'visible'
+														: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition"
+													on:click={() => {
+														if (!generatingImage) {
+															generateImage(message);
+														}
+													}}
+												>
+													{#if generatingImage}
+														<svg
+															class=" w-4 h-4"
+															fill="currentColor"
+															viewBox="0 0 24 24"
+															xmlns="http://www.w3.org/2000/svg"
+															><style>
+																.spinner_S1WN {
+																	animation: spinner_MGfb 0.8s linear infinite;
+																	animation-delay: -0.8s;
+																}
+																.spinner_Km9P {
+																	animation-delay: -0.65s;
+																}
+																.spinner_JApP {
+																	animation-delay: -0.5s;
+																}
+																@keyframes spinner_MGfb {
+																	93.75%,
+																	100% {
+																		opacity: 0.2;
+																	}
+																}
+															</style><circle class="spinner_S1WN" cx="4" cy="12" r="3" /><circle
+																class="spinner_S1WN spinner_Km9P"
+																cx="12"
+																cy="12"
+																r="3"
+															/><circle
+																class="spinner_S1WN spinner_JApP"
+																cx="20"
+																cy="12"
+																r="3"
+															/></svg
+														>
+													{:else}
+														<svg
+															xmlns="http://www.w3.org/2000/svg"
+															fill="none"
+															viewBox="0 0 24 24"
+															stroke-width="2"
+															stroke="currentColor"
+															class="w-4 h-4"
+														>
+															<path
+																stroke-linecap="round"
+																stroke-linejoin="round"
+																d="m2.25 15.75 5.159-5.159a2.25 2.25 0 0 1 3.182 0l5.159 5.159m-1.5-1.5 1.409-1.409a2.25 2.25 0 0 1 3.182 0l2.909 2.909m-18 3.75h16.5a1.5 1.5 0 0 0 1.5-1.5V6a1.5 1.5 0 0 0-1.5-1.5H3.75A1.5 1.5 0 0 0 2.25 6v12a1.5 1.5 0 0 0 1.5 1.5Zm10.5-11.25h.008v.008h-.008V8.25Zm.375 0a.375.375 0 1 1-.75 0 .375.375 0 0 1 .75 0Z"
+															/>
+														</svg>
+													{/if}
+												</button>
+											</Tooltip>
 										{/if}
 
 										{#if message.info}
-											<button
-												class=" {isLastMessage
-													? 'visible'
-													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition whitespace-pre-wrap"
-												on:click={() => {
-													console.log(message);
-												}}
-												id="info-{message.id}"
-											>
-												<svg
-													xmlns="http://www.w3.org/2000/svg"
-													fill="none"
-													viewBox="0 0 24 24"
-													stroke-width="1.5"
-													stroke="currentColor"
-													class="w-4 h-4"
+											<Tooltip content="Generation Info" placement="bottom">
+												<button
+													class=" {isLastMessage
+														? 'visible'
+														: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition whitespace-pre-wrap"
+													on:click={() => {
+														console.log(message);
+													}}
+													id="info-{message.id}"
 												>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M11.25 11.25l.041-.02a.75.75 0 011.063.852l-.708 2.836a.75.75 0 001.063.853l.041-.021M21 12a9 9 0 11-18 0 9 9 0 0118 0zm-9-3.75h.008v.008H12V8.25z"
-													/>
-												</svg>
-											</button>
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														fill="none"
+														viewBox="0 0 24 24"
+														stroke-width="2"
+														stroke="currentColor"
+														class="w-4 h-4"
+													>
+														<path
+															stroke-linecap="round"
+															stroke-linejoin="round"
+															d="M11.25 11.25l.041-.02a.75.75 0 011.063.852l-.708 2.836a.75.75 0 001.063.853l.041-.021M21 12a9 9 0 11-18 0 9 9 0 0118 0zm-9-3.75h.008v.008H12V8.25z"
+														/>
+													</svg>
+												</button>
+											</Tooltip>
 										{/if}
 
 										{#if isLastMessage}
-											<button
-												type="button"
-												class="{isLastMessage
-													? 'visible'
-													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition regenerate-response-button"
-												on:click={() => {
-													continueGeneration();
-												}}
-											>
-												<svg
-													xmlns="http://www.w3.org/2000/svg"
-													fill="none"
-													viewBox="0 0 24 24"
-													stroke-width="1.5"
-													stroke="currentColor"
-													class="w-4 h-4"
+											<Tooltip content="Continue Response" placement="bottom">
+												<button
+													type="button"
+													class="{isLastMessage
+														? 'visible'
+														: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition regenerate-response-button"
+													on:click={() => {
+														continueGeneration();
+													}}
 												>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z"
-													/>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M15.91 11.672a.375.375 0 0 1 0 .656l-5.603 3.113a.375.375 0 0 1-.557-.328V8.887c0-.286.307-.466.557-.327l5.603 3.112Z"
-													/>
-												</svg>
-											</button>
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														fill="none"
+														viewBox="0 0 24 24"
+														stroke-width="2"
+														stroke="currentColor"
+														class="w-4 h-4"
+													>
+														<path
+															stroke-linecap="round"
+															stroke-linejoin="round"
+															d="M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z"
+														/>
+														<path
+															stroke-linecap="round"
+															stroke-linejoin="round"
+															d="M15.91 11.672a.375.375 0 0 1 0 .656l-5.603 3.113a.375.375 0 0 1-.557-.328V8.887c0-.286.307-.466.557-.327l5.603 3.112Z"
+														/>
+													</svg>
+												</button>
+											</Tooltip>
 
-											<button
-												type="button"
-												class="{isLastMessage
-													? 'visible'
-													: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition regenerate-response-button"
-												on:click={regenerateResponse}
-											>
-												<svg
-													xmlns="http://www.w3.org/2000/svg"
-													fill="none"
-													viewBox="0 0 24 24"
-													stroke-width="1.5"
-													stroke="currentColor"
-													class="w-4 h-4"
+											<Tooltip content="Regenerate" placement="bottom">
+												<button
+													type="button"
+													class="{isLastMessage
+														? 'visible'
+														: 'invisible group-hover:visible'} p-1 rounded dark:hover:text-white hover:text-black transition regenerate-response-button"
+													on:click={regenerateResponse}
 												>
-													<path
-														stroke-linecap="round"
-														stroke-linejoin="round"
-														d="M16.023 9.348h4.992v-.001M2.985 19.644v-4.992m0 0h4.992m-4.993 0l3.181 3.183a8.25 8.25 0 0013.803-3.7M4.031 9.865a8.25 8.25 0 0113.803-3.7l3.181 3.182m0-4.991v4.99"
-													/>
-												</svg>
-											</button>
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														fill="none"
+														viewBox="0 0 24 24"
+														stroke-width="2"
+														stroke="currentColor"
+														class="w-4 h-4"
+													>
+														<path
+															stroke-linecap="round"
+															stroke-linejoin="round"
+															d="M16.023 9.348h4.992v-.001M2.985 19.644v-4.992m0 0h4.992m-4.993 0l3.181 3.183a8.25 8.25 0 0013.803-3.7M4.031 9.865a8.25 8.25 0 0113.803-3.7l3.181 3.182m0-4.991v4.99"
+														/>
+													</svg>
+												</button>
+											</Tooltip>
 										{/if}
 									</div>
 								{/if}

src/lib/components/chat/Messages/UserMessage.svelte

@@ -5,6 +5,7 @@
 	import Name from './Name.svelte';
 	import ProfileImage from './ProfileImage.svelte';
 	import { modelfiles, settings } from '$lib/stores';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
 	const dispatch = createEventDispatcher();
 
@@ -20,18 +21,17 @@
 
 	let edit = false;
 	let editedContent = '';
-
+	let messageEditTextAreaElement: HTMLTextAreaElement;
 	const editMessageHandler = async () => {
 		edit = true;
 		editedContent = message.content;
 
 		await tick();
-		const editElement = document.getElementById(`message-edit-${message.id}`);
 
-		editElement.style.height = '';
-		editElement.style.height = `${editElement.scrollHeight}px`;
+		messageEditTextAreaElement.style.height = '';
+		messageEditTextAreaElement.style.height = `${messageEditTextAreaElement.scrollHeight}px`;
 
-		editElement?.focus();
+		messageEditTextAreaElement?.focus();
 	};
 
 	const editMessageConfirmHandler = async () => {
@@ -165,9 +165,11 @@
 				<div class=" w-full">
 					<textarea
 						id="message-edit-{message.id}"
+						bind:this={messageEditTextAreaElement}
 						class=" bg-transparent outline-none w-full resize-none"
 						bind:value={editedContent}
 						on:input={(e) => {
+							e.target.style.height = '';
 							e.target.style.height = `${e.target.scrollHeight}px`;
 						}}
 					/>
@@ -245,72 +247,78 @@
 							</div>
 						{/if}
 
-						<button
-							class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition edit-user-message-button"
-							on:click={() => {
-								editMessageHandler();
-							}}
-						>
-							<svg
-								xmlns="http://www.w3.org/2000/svg"
-								fill="none"
-								viewBox="0 0 24 24"
-								stroke-width="1.5"
-								stroke="currentColor"
-								class="w-4 h-4"
-							>
-								<path
-									stroke-linecap="round"
-									stroke-linejoin="round"
-									d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
-								/>
-							</svg>
-						</button>
-
-						<button
-							class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition"
-							on:click={() => {
-								copyToClipboard(message.content);
-							}}
-						>
-							<svg
-								xmlns="http://www.w3.org/2000/svg"
-								fill="none"
-								viewBox="0 0 24 24"
-								stroke-width="1.5"
-								stroke="currentColor"
-								class="w-4 h-4"
-							>
-								<path
-									stroke-linecap="round"
-									stroke-linejoin="round"
-									d="M15.666 3.888A2.25 2.25 0 0013.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 01-.75.75H9a.75.75 0 01-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 01-2.25 2.25H6.75A2.25 2.25 0 014.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 011.927-.184"
-								/>
-							</svg>
-						</button>
-
-						{#if !isFirstMessage}
+						<Tooltip content="Edit" placement="bottom">
 							<button
-								class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition"
+								class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition edit-user-message-button"
 								on:click={() => {
-									deleteMessageHandler();
+									editMessageHandler();
 								}}
 							>
 								<svg
 									xmlns="http://www.w3.org/2000/svg"
 									fill="none"
 									viewBox="0 0 24 24"
-									stroke-width="1.5"
+									stroke-width="2"
 									stroke="currentColor"
 									class="w-4 h-4"
 								>
 									<path
 										stroke-linecap="round"
 										stroke-linejoin="round"
-										d="m14.74 9-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 0 1-2.244 2.077H8.084a2.25 2.25 0 0 1-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 0 0-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 0 1 3.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 0 0-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 0 0-7.5 0"
+										d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
 									/>
 								</svg>
 							</button>
+						</Tooltip>
+
+						<Tooltip content="Copy" placement="bottom">
+							<button
+								class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition"
+								on:click={() => {
+									copyToClipboard(message.content);
+								}}
+							>
+								<svg
+									xmlns="http://www.w3.org/2000/svg"
+									fill="none"
+									viewBox="0 0 24 24"
+									stroke-width="2"
+									stroke="currentColor"
+									class="w-4 h-4"
+								>
+									<path
+										stroke-linecap="round"
+										stroke-linejoin="round"
+										d="M15.666 3.888A2.25 2.25 0 0013.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 01-.75.75H9a.75.75 0 01-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 01-2.25 2.25H6.75A2.25 2.25 0 014.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 011.927-.184"
+									/>
+								</svg>
+							</button>
+						</Tooltip>
+
+						{#if !isFirstMessage}
+							<Tooltip content="Delete" placement="bottom">
+								<button
+									class="invisible group-hover:visible p-1 rounded dark:hover:text-white hover:text-black transition"
+									on:click={() => {
+										deleteMessageHandler();
+									}}
+								>
+									<svg
+										xmlns="http://www.w3.org/2000/svg"
+										fill="none"
+										viewBox="0 0 24 24"
+										stroke-width="2"
+										stroke="currentColor"
+										class="w-4 h-4"
+									>
+										<path
+											stroke-linecap="round"
+											stroke-linejoin="round"
+											d="m14.74 9-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 0 1-2.244 2.077H8.084a2.25 2.25 0 0 1-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 0 0-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 0 1 3.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 0 0-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 0 0-7.5 0"
+										/>
+									</svg>
+								</button>
+							</Tooltip>
 						{/if}
 					</div>
 				</div>

src/lib/components/chat/Settings/Account.svelte

@@ -15,6 +15,7 @@
 	let name = '';
 	let showJWTToken = false;
 	let JWTTokenCopied = false;
+	let profileImageInputElement: HTMLInputElement;
 
 	const submitHandler = async () => {
 		const updatedUser = await updateUserProfile(localStorage.token, name, profileImageUrl).catch(
@@ -40,11 +41,12 @@
 	<div class=" space-y-3 pr-1.5 overflow-y-scroll max-h-80">
 		<input
 			id="profile-image-input"
+			bind:this={profileImageInputElement}
 			type="file"
 			hidden
 			accept="image/*"
 			on:change={(e) => {
-				const files = e?.target?.files ?? [];
+				const files = profileImageInputElement.files ?? [];
 				let reader = new FileReader();
 				reader.onload = (event) => {
 					let originalImageUrl = `${event.target.result}`;
@@ -86,7 +88,7 @@
 						// Display the compressed image
 						profileImageUrl = compressedSrc;
 
-						e.target.files = null;
+						profileImageInputElement.files = null;
 					};
 				};
 
@@ -108,7 +110,7 @@
 						class="relative rounded-full dark:bg-gray-700"
 						type="button"
 						on:click={() => {
-							document.getElementById('profile-image-input')?.click();
+							profileImageInputElement.click();
 						}}
 					>
 						<img
@@ -269,7 +271,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
+			class="  px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
 			on:click={async () => {
 				const res = await submitHandler();
 

src/lib/components/chat/Settings/Audio.svelte

@@ -251,7 +251,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
+			class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
 			type="submit"
 		>
 			Save

src/lib/components/chat/Settings/Chats.svelte

@@ -22,6 +22,7 @@
 	let saveChatHistory = true;
 	let importFiles;
 	let showDeleteConfirm = false;
+	let chatImportInputElement: HTMLInputElement;
 
 	$: if (importFiles) {
 		console.log(importFiles);
@@ -159,11 +160,18 @@
 		<hr class=" dark:border-gray-700" />
 
 		<div class="flex flex-col">
-			<input id="chat-import-input" bind:files={importFiles} type="file" accept=".json" hidden />
+			<input
+				id="chat-import-input"
+				bind:this={chatImportInputElement}
+				bind:files={importFiles}
+				type="file"
+				accept=".json"
+				hidden
+			/>
 			<button
 				class=" flex rounded-md py-2 px-3.5 w-full hover:bg-gray-200 dark:hover:bg-gray-800 transition"
 				on:click={() => {
-					document.getElementById('chat-import-input').click();
+					chatImportInputElement.click();
 				}}
 			>
 				<div class=" self-center mr-3">

src/lib/components/chat/Settings/Connections.svelte

@@ -3,30 +3,38 @@
 	import { createEventDispatcher, onMount } from 'svelte';
 	const dispatch = createEventDispatcher();
 
-	import { getOllamaAPIUrl, getOllamaVersion, updateOllamaAPIUrl } from '$lib/apis/ollama';
-	import { getOpenAIKey, getOpenAIUrl, updateOpenAIKey, updateOpenAIUrl } from '$lib/apis/openai';
+	import { getOllamaUrls, getOllamaVersion, updateOllamaUrls } from '$lib/apis/ollama';
+	import {
+		getOpenAIKeys,
+		getOpenAIUrls,
+		updateOpenAIKeys,
+		updateOpenAIUrls
+	} from '$lib/apis/openai';
 	import { toast } from 'svelte-sonner';
 
 	export let getModels: Function;
 
 	// External
-	let API_BASE_URL = '';
+	let OLLAMA_BASE_URL = '';
+	let OLLAMA_BASE_URLS = [''];
 
 	let OPENAI_API_KEY = '';
 	let OPENAI_API_BASE_URL = '';
 
+	let OPENAI_API_KEYS = [''];
+	let OPENAI_API_BASE_URLS = [''];
+
 	let showOpenAI = false;
-	let showLiteLLM = false;
 
 	const updateOpenAIHandler = async () => {
-		OPENAI_API_BASE_URL = await updateOpenAIUrl(localStorage.token, OPENAI_API_BASE_URL);
-		OPENAI_API_KEY = await updateOpenAIKey(localStorage.token, OPENAI_API_KEY);
+		OPENAI_API_BASE_URLS = await updateOpenAIUrls(localStorage.token, OPENAI_API_BASE_URLS);
+		OPENAI_API_KEYS = await updateOpenAIKeys(localStorage.token, OPENAI_API_KEYS);
 
 		await models.set(await getModels());
 	};
 
-	const updateOllamaAPIUrlHandler = async () => {
-		API_BASE_URL = await updateOllamaAPIUrl(localStorage.token, API_BASE_URL);
+	const updateOllamaUrlsHandler = async () => {
+		OLLAMA_BASE_URLS = await updateOllamaUrls(localStorage.token, OLLAMA_BASE_URLS);
 
 		const ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => {
 			toast.error(error);
@@ -41,9 +49,9 @@
 
 	onMount(async () => {
 		if ($user.role === 'admin') {
-			API_BASE_URL = await getOllamaAPIUrl(localStorage.token);
-			OPENAI_API_BASE_URL = await getOpenAIUrl(localStorage.token);
-			OPENAI_API_KEY = await getOpenAIKey(localStorage.token);
+			OLLAMA_BASE_URLS = await getOllamaUrls(localStorage.token);
+			OPENAI_API_BASE_URLS = await getOpenAIUrls(localStorage.token);
+			OPENAI_API_KEYS = await getOpenAIKeys(localStorage.token);
 		}
 	});
 </script>
@@ -53,11 +61,6 @@
 	on:submit|preventDefault={() => {
 		updateOpenAIHandler();
 		dispatch('save');
-
-		// saveSettings({
-		// 	OPENAI_API_KEY: OPENAI_API_KEY !== '' ? OPENAI_API_KEY : undefined,
-		// 	OPENAI_API_BASE_URL: OPENAI_API_BASE_URL !== '' ? OPENAI_API_BASE_URL : undefined
-		// });
 	}}
 >
 	<div class="  pr-1.5 overflow-y-scroll max-h-[20.5rem] space-y-3">
@@ -75,37 +78,74 @@
 				</div>
 
 				{#if showOpenAI}
-					<div>
-						<div class=" mb-2.5 text-sm font-medium">API Key</div>
-						<div class="flex w-full">
-							<div class="flex-1">
-								<input
-									class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
-									placeholder="Enter OpenAI API Key"
-									bind:value={OPENAI_API_KEY}
-									autocomplete="off"
-								/>
-							</div>
-						</div>
-					</div>
+					<div class="flex flex-col gap-1">
+						{#each OPENAI_API_BASE_URLS as url, idx}
+							<div class="flex w-full gap-2">
+								<div class="flex-1">
+									<input
+										class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+										placeholder="API Base URL"
+										bind:value={url}
+										autocomplete="off"
+									/>
+								</div>
 
-					<div>
-						<div class=" mb-2.5 text-sm font-medium">API Base URL</div>
-						<div class="flex w-full">
-							<div class="flex-1">
-								<input
-									class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
-									placeholder="Enter OpenAI API Base URL"
-									bind:value={OPENAI_API_BASE_URL}
-									autocomplete="off"
-								/>
+								<div class="flex-1">
+									<input
+										class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+										placeholder="API Key"
+										bind:value={OPENAI_API_KEYS[idx]}
+										autocomplete="off"
+									/>
+								</div>
+								<div class="self-center flex items-center">
+									{#if idx === 0}
+										<button
+											class="px-1"
+											on:click={() => {
+												OPENAI_API_BASE_URLS = [...OPENAI_API_BASE_URLS, ''];
+												OPENAI_API_KEYS = [...OPENAI_API_KEYS, ''];
+											}}
+											type="button"
+										>
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 16 16"
+												fill="currentColor"
+												class="w-4 h-4"
+											>
+												<path
+													d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+												/>
+											</svg>
+										</button>
+									{:else}
+										<button
+											class="px-1"
+											on:click={() => {
+												OPENAI_API_BASE_URLS = OPENAI_API_BASE_URLS.filter(
+													(url, urlIdx) => idx !== urlIdx
+												);
+												OPENAI_API_KEYS = OPENAI_API_KEYS.filter((key, keyIdx) => idx !== keyIdx);
+											}}
+											type="button"
+										>
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 16 16"
+												fill="currentColor"
+												class="w-4 h-4"
+											>
+												<path d="M3.75 7.25a.75.75 0 0 0 0 1.5h8.5a.75.75 0 0 0 0-1.5h-8.5Z" />
+											</svg>
+										</button>
+									{/if}
+								</div>
 							</div>
-						</div>
-						<div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
-							WebUI will make requests to <span class=" text-gray-200"
-								>'{OPENAI_API_BASE_URL}/chat'</span
-							>
-						</div>
+							<div class=" mb-1 text-xs text-gray-400 dark:text-gray-500">
+								WebUI will make requests to <span class=" text-gray-200">'{url}/models'</span>
+							</div>
+						{/each}
 					</div>
 				{/if}
 			</div>
@@ -115,34 +155,81 @@
 
 		<div>
 			<div class=" mb-2.5 text-sm font-medium">Ollama Base URL</div>
-			<div class="flex w-full">
-				<div class="flex-1 mr-2">
-					<input
-						class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
-						placeholder="Enter URL (e.g. http://localhost:11434)"
-						bind:value={API_BASE_URL}
-					/>
+			<div class="flex w-full gap-1.5">
+				<div class="flex-1 flex flex-col gap-2">
+					{#each OLLAMA_BASE_URLS as url, idx}
+						<div class="flex gap-1.5">
+							<input
+								class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+								placeholder="Enter URL (e.g. http://localhost:11434)"
+								bind:value={url}
+							/>
+
+							<div class="self-center flex items-center">
+								{#if idx === 0}
+									<button
+										class="px-1"
+										on:click={() => {
+											OLLAMA_BASE_URLS = [...OLLAMA_BASE_URLS, ''];
+										}}
+										type="button"
+									>
+										<svg
+											xmlns="http://www.w3.org/2000/svg"
+											viewBox="0 0 16 16"
+											fill="currentColor"
+											class="w-4 h-4"
+										>
+											<path
+												d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+											/>
+										</svg>
+									</button>
+								{:else}
+									<button
+										class="px-1"
+										on:click={() => {
+											OLLAMA_BASE_URLS = OLLAMA_BASE_URLS.filter((url, urlIdx) => idx !== urlIdx);
+										}}
+										type="button"
+									>
+										<svg
+											xmlns="http://www.w3.org/2000/svg"
+											viewBox="0 0 16 16"
+											fill="currentColor"
+											class="w-4 h-4"
+										>
+											<path d="M3.75 7.25a.75.75 0 0 0 0 1.5h8.5a.75.75 0 0 0 0-1.5h-8.5Z" />
+										</svg>
+									</button>
+								{/if}
+							</div>
+						</div>
+					{/each}
 				</div>
-				<button
-					class="px-3 bg-gray-200 hover:bg-gray-300 dark:bg-gray-600 dark:hover:bg-gray-700 rounded transition"
-					on:click={() => {
-						updateOllamaAPIUrlHandler();
-					}}
-					type="button"
-				>
-					<svg
-						xmlns="http://www.w3.org/2000/svg"
-						viewBox="0 0 20 20"
-						fill="currentColor"
-						class="w-4 h-4"
+
+				<div class="">
+					<button
+						class="p-2.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-850 dark:hover:bg-gray-800 rounded-lg transition"
+						on:click={() => {
+							updateOllamaUrlsHandler();
+						}}
+						type="button"
 					>
-						<path
-							fill-rule="evenodd"
-							d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
-							clip-rule="evenodd"
-						/>
-					</svg>
-				</button>
+						<svg
+							xmlns="http://www.w3.org/2000/svg"
+							viewBox="0 0 20 20"
+							fill="currentColor"
+							class="w-4 h-4"
+						>
+							<path
+								fill-rule="evenodd"
+								d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
+								clip-rule="evenodd"
+							/>
+						</svg>
+					</button>
+				</div>
 			</div>
 
 			<div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
@@ -160,7 +247,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
+			class="  px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
 			type="submit"
 		>
 			Save

src/lib/components/chat/Settings/General.svelte

@@ -176,7 +176,7 @@
 			<div class=" my-2.5 text-sm font-medium">System Prompt</div>
 			<textarea
 				bind:value={system}
-				class="w-full rounded p-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
+				class="w-full rounded-lg p-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none resize-none"
 				rows="4"
 			/>
 		</div>
@@ -262,7 +262,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
+			class="  px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
 			on:click={() => {
 				saveSettings({
 					system: system !== '' ? system : undefined,

src/lib/components/chat/Settings/Images.svelte

@@ -5,16 +5,18 @@
 	import { config, user } from '$lib/stores';
 	import {
 		getAUTOMATIC1111Url,
-		getDefaultDiffusionModel,
-		getDiffusionModels,
-		getImageGenerationEnabledStatus,
+		getImageGenerationModels,
+		getDefaultImageGenerationModel,
+		updateDefaultImageGenerationModel,
 		getImageSize,
-		toggleImageGenerationEnabledStatus,
+		getImageGenerationConfig,
+		updateImageGenerationConfig,
 		updateAUTOMATIC1111Url,
-		updateDefaultDiffusionModel,
 		updateImageSize,
 		getImageSteps,
-		updateImageSteps
+		updateImageSteps,
+		getOpenAIKey,
+		updateOpenAIKey
 	} from '$lib/apis/images';
 	import { getBackendConfig } from '$lib/apis';
 	const dispatch = createEventDispatcher();
@@ -23,8 +25,11 @@
 
 	let loading = false;
 
+	let imageGenerationEngine = '';
 	let enableImageGeneration = false;
+
 	let AUTOMATIC1111_BASE_URL = '';
+	let OPENAI_API_KEY = '';
 
 	let selectedModel = '';
 	let models = null;
@@ -33,11 +38,11 @@
 	let steps = 50;
 
 	const getModels = async () => {
-		models = await getDiffusionModels(localStorage.token).catch((error) => {
+		models = await getImageGenerationModels(localStorage.token).catch((error) => {
 			toast.error(error);
 			return null;
 		});
-		selectedModel = await getDefaultDiffusionModel(localStorage.token).catch((error) => {
+		selectedModel = await getDefaultImageGenerationModel(localStorage.token).catch((error) => {
 			return '';
 		});
 	};
@@ -62,33 +67,45 @@
 			AUTOMATIC1111_BASE_URL = await getAUTOMATIC1111Url(localStorage.token);
 		}
 	};
-	const toggleImageGeneration = async () => {
-		if (AUTOMATIC1111_BASE_URL) {
-			enableImageGeneration = await toggleImageGenerationEnabledStatus(localStorage.token).catch(
-				(error) => {
-					toast.error(error);
-					return false;
-				}
-			);
+	const updateImageGeneration = async () => {
+		const res = await updateImageGenerationConfig(
+			localStorage.token,
+			imageGenerationEngine,
+			enableImageGeneration
+		).catch((error) => {
+			toast.error(error);
+			return null;
+		});
 
-			if (enableImageGeneration) {
-				config.set(await getBackendConfig(localStorage.token));
-				getModels();
-			}
-		} else {
-			enableImageGeneration = false;
-			toast.error('AUTOMATIC1111_BASE_URL not provided');
+		if (res) {
+			imageGenerationEngine = res.engine;
+			enableImageGeneration = res.enabled;
+		}
+
+		if (enableImageGeneration) {
+			config.set(await getBackendConfig(localStorage.token));
+			getModels();
 		}
 	};
 
 	onMount(async () => {
 		if ($user.role === 'admin') {
-			enableImageGeneration = await getImageGenerationEnabledStatus(localStorage.token);
-			AUTOMATIC1111_BASE_URL = await getAUTOMATIC1111Url(localStorage.token);
+			const res = await getImageGenerationConfig(localStorage.token).catch((error) => {
+				toast.error(error);
+				return null;
+			});
 
-			if (enableImageGeneration && AUTOMATIC1111_BASE_URL) {
-				imageSize = await getImageSize(localStorage.token);
-				steps = await getImageSteps(localStorage.token);
+			if (res) {
+				imageGenerationEngine = res.engine;
+				enableImageGeneration = res.enabled;
+			}
+			AUTOMATIC1111_BASE_URL = await getAUTOMATIC1111Url(localStorage.token);
+			OPENAI_API_KEY = await getOpenAIKey(localStorage.token);
+
+			imageSize = await getImageSize(localStorage.token);
+			steps = await getImageSteps(localStorage.token);
+
+			if (enableImageGeneration) {
 				getModels();
 			}
 		}
@@ -99,7 +116,13 @@
 	class="flex flex-col h-full justify-between space-y-3 text-sm"
 	on:submit|preventDefault={async () => {
 		loading = true;
-		await updateDefaultDiffusionModel(localStorage.token, selectedModel);
+
+		if (imageGenerationEngine === 'openai') {
+			await updateOpenAIKey(localStorage.token, OPENAI_API_KEY);
+		}
+
+		await updateDefaultImageGenerationModel(localStorage.token, selectedModel);
+
 		await updateImageSize(localStorage.token, imageSize).catch((error) => {
 			toast.error(error);
 			return null;
@@ -117,6 +140,23 @@
 		<div>
 			<div class=" mb-1 text-sm font-medium">Image Settings</div>
 
+			<div class=" py-0.5 flex w-full justify-between">
+				<div class=" self-center text-xs font-medium">Image Generation Engine</div>
+				<div class="flex items-center relative">
+					<select
+						class="w-fit pr-8 rounded px-2 p-1 text-xs bg-transparent outline-none text-right"
+						bind:value={imageGenerationEngine}
+						placeholder="Select a mode"
+						on:change={async () => {
+							await updateImageGeneration();
+						}}
+					>
+						<option value="">Default (Automatic1111)</option>
+						<option value="openai">Open AI (Dall-E)</option>
+					</select>
+				</div>
+			</div>
+
 			<div>
 				<div class=" py-0.5 flex w-full justify-between">
 					<div class=" self-center text-xs font-medium">Image Generation (Experimental)</div>
@@ -124,7 +164,17 @@
 					<button
 						class="p-1 px-3 text-xs flex rounded transition"
 						on:click={() => {
-							toggleImageGeneration();
+							if (imageGenerationEngine === '' && AUTOMATIC1111_BASE_URL === '') {
+								toast.error('AUTOMATIC1111 Base URL is required.');
+								enableImageGeneration = false;
+							} else if (imageGenerationEngine === 'openai' && OPENAI_API_KEY === '') {
+								toast.error('OpenAI API Key is required.');
+								enableImageGeneration = false;
+							} else {
+								enableImageGeneration = !enableImageGeneration;
+							}
+
+							updateImageGeneration();
 						}}
 						type="button"
 					>
@@ -139,49 +189,62 @@
 		</div>
 		<hr class=" dark:border-gray-700" />
 
-		<div class=" mb-2.5 text-sm font-medium">AUTOMATIC1111 Base URL</div>
-		<div class="flex w-full">
-			<div class="flex-1 mr-2">
-				<input
-					class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
-					placeholder="Enter URL (e.g. http://127.0.0.1:7860/)"
-					bind:value={AUTOMATIC1111_BASE_URL}
-				/>
-			</div>
-			<button
-				class="px-3 bg-gray-200 hover:bg-gray-300 dark:bg-gray-600 dark:hover:bg-gray-700 rounded transition"
-				type="button"
-				on:click={() => {
-					// updateOllamaAPIUrlHandler();
-
-					updateAUTOMATIC1111UrlHandler();
-				}}
-			>
-				<svg
-					xmlns="http://www.w3.org/2000/svg"
-					viewBox="0 0 20 20"
-					fill="currentColor"
-					class="w-4 h-4"
-				>
-					<path
-						fill-rule="evenodd"
-						d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
-						clip-rule="evenodd"
+		{#if imageGenerationEngine === ''}
+			<div class=" mb-2.5 text-sm font-medium">AUTOMATIC1111 Base URL</div>
+			<div class="flex w-full">
+				<div class="flex-1 mr-2">
+					<input
+						class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+						placeholder="Enter URL (e.g. http://127.0.0.1:7860/)"
+						bind:value={AUTOMATIC1111_BASE_URL}
 					/>
-				</svg>
-			</button>
-		</div>
+				</div>
+				<button
+					class="px-3 bg-gray-200 hover:bg-gray-300 dark:bg-gray-600 dark:hover:bg-gray-700 rounded-lg transition"
+					type="button"
+					on:click={() => {
+						// updateOllamaAPIUrlHandler();
 
-		<div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
-			Include `--api` flag when running stable-diffusion-webui
-			<a
-				class=" text-gray-300 font-medium"
-				href="https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/3734"
-				target="_blank"
-			>
-				(e.g. `sh webui.sh --api`)
-			</a>
-		</div>
+						updateAUTOMATIC1111UrlHandler();
+					}}
+				>
+					<svg
+						xmlns="http://www.w3.org/2000/svg"
+						viewBox="0 0 20 20"
+						fill="currentColor"
+						class="w-4 h-4"
+					>
+						<path
+							fill-rule="evenodd"
+							d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
+							clip-rule="evenodd"
+						/>
+					</svg>
+				</button>
+			</div>
+
+			<div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
+				Include `--api` flag when running stable-diffusion-webui
+				<a
+					class=" text-gray-300 font-medium"
+					href="https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/3734"
+					target="_blank"
+				>
+					(e.g. `sh webui.sh --api`)
+				</a>
+			</div>
+		{:else if imageGenerationEngine === 'openai'}
+			<div class=" mb-2.5 text-sm font-medium">OpenAI API Key</div>
+			<div class="flex w-full">
+				<div class="flex-1 mr-2">
+					<input
+						class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+						placeholder="Enter API Key"
+						bind:value={OPENAI_API_KEY}
+					/>
+				</div>
+			</div>
+		{/if}
 
 		{#if enableImageGeneration}
 			<hr class=" dark:border-gray-700" />
@@ -191,7 +254,7 @@
 				<div class="flex w-full">
 					<div class="flex-1 mr-2">
 						<select
-							class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
+							class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
 							bind:value={selectedModel}
 							placeholder="Select a model"
 						>
@@ -199,9 +262,7 @@
 								<option value="" disabled selected>Select a model</option>
 							{/if}
 							{#each models ?? [] as model}
-								<option value={model.title} class="bg-gray-100 dark:bg-gray-700"
-									>{model.model_name}</option
-								>
+								<option value={model.id} class="bg-gray-100 dark:bg-gray-700">{model.name}</option>
 							{/each}
 						</select>
 					</div>
@@ -213,7 +274,7 @@
 				<div class="flex w-full">
 					<div class="flex-1 mr-2">
 						<input
-							class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
+							class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
 							placeholder="Enter Image Size (e.g. 512x512)"
 							bind:value={imageSize}
 						/>
@@ -226,7 +287,7 @@
 				<div class="flex w-full">
 					<div class="flex-1 mr-2">
 						<input
-							class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
+							class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
 							placeholder="Enter Number of Steps (e.g. 50)"
 							bind:value={steps}
 						/>
@@ -238,7 +299,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded flex flex-row space-x-1 items-center {loading
+			class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg flex flex-row space-x-1 items-center {loading
 				? ' cursor-not-allowed'
 				: ''}"
 			type="submit"

src/lib/components/chat/Settings/Interface.svelte

@@ -63,6 +63,7 @@
 		}
 
 		saveSettings({
+			titleAutoGenerateModel: titleAutoGenerateModel !== '' ? titleAutoGenerateModel : undefined,
 			titleGenerationPrompt: titleGenerationPrompt ? titleGenerationPrompt : undefined
 		});
 	};
@@ -186,7 +187,7 @@
 			<div class="flex w-full">
 				<div class="flex-1 mr-2">
 					<select
-						class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
+						class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
 						bind:value={titleAutoGenerateModel}
 						placeholder="Select a model"
 					>
@@ -200,35 +201,12 @@
 						{/each}
 					</select>
 				</div>
-				<button
-					class="px-3 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-800 dark:text-gray-100 rounded transition"
-					on:click={() => {
-						saveSettings({
-							titleAutoGenerateModel:
-								titleAutoGenerateModel !== '' ? titleAutoGenerateModel : undefined
-						});
-					}}
-					type="button"
-				>
-					<svg
-						xmlns="http://www.w3.org/2000/svg"
-						viewBox="0 0 16 16"
-						fill="currentColor"
-						class="w-3.5 h-3.5"
-					>
-						<path
-							fill-rule="evenodd"
-							d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.932.75.75 0 0 1-1.3-.75 6 6 0 0 1 9.44-1.242l.842.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44 1.241l-.84-.84v1.371a.75.75 0 0 1-1.5 0V9.591a.75.75 0 0 1 .75-.75H5.35a.75.75 0 0 1 0 1.5H3.98l.841.841a4.5 4.5 0 0 0 7.08-.932.75.75 0 0 1 1.025-.273Z"
-							clip-rule="evenodd"
-						/>
-					</svg>
-				</button>
 			</div>
-			<div class="mt-3">
+			<div class="mt-3 mr-2">
 				<div class=" mb-2.5 text-sm font-medium">Title Generation Prompt</div>
 				<textarea
 					bind:value={titleGenerationPrompt}
-					class="w-full rounded p-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
+					class="w-full rounded-lg p-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none resize-none"
 					rows="3"
 				/>
 			</div>
@@ -321,7 +299,7 @@
 
 	<div class="flex justify-end pt-3 text-sm font-medium">
 		<button
-			class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
+			class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
 			type="submit"
 		>
 			Save

src/lib/components/chat/SettingsModal.svelte

@@ -326,7 +326,7 @@
 						{getModels}
 						{saveSettings}
 						on:save={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'models'}
@@ -335,28 +335,28 @@
 					<Connections
 						{getModels}
 						on:save={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'interface'}
 					<Interface
 						{saveSettings}
 						on:save={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'audio'}
 					<Audio
 						{saveSettings}
 						on:save={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'images'}
 					<Images
 						{saveSettings}
 						on:save={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'chats'}
@@ -364,7 +364,7 @@
 				{:else if selectedTab === 'account'}
 					<Account
 						saveHandler={() => {
-							show = false;
+							toast.success('Settings saved successfully!');
 						}}
 					/>
 				{:else if selectedTab === 'about'}

src/lib/components/common/Image.svelte

@@ -1,18 +1,23 @@
 <script lang="ts">
+	import { WEBUI_BASE_URL } from '$lib/constants';
 	import ImagePreview from './ImagePreview.svelte';
 
 	export let src = '';
 	export let alt = '';
 
+	let _src = '';
+
+	$: _src = src.startsWith('/') ? `${WEBUI_BASE_URL}${src}` : src;
+
 	let showImagePreview = false;
 </script>
 
-<ImagePreview bind:show={showImagePreview} {src} {alt} />
+<ImagePreview bind:show={showImagePreview} src={_src} {alt} />
 <button
 	on:click={() => {
 		console.log('image preview');
 		showImagePreview = true;
 	}}
 >
-	<img {src} {alt} class=" max-h-96 rounded-lg" draggable="false" />
+	<img src={_src} {alt} class=" max-h-96 rounded-lg" draggable="false" />
 </button>

src/lib/components/common/Tooltip.svelte

@@ -29,6 +29,6 @@
 	});
 </script>
 
-<div bind:this={tooltipElement}>
+<div bind:this={tooltipElement} aria-label={content}>
 	<slot />
 </div>

src/lib/components/documents/AddDocModal.svelte

@@ -15,7 +15,7 @@
 
 	export let show = false;
 	export let selectedDoc;
-
+	let uploadDocInputElement: HTMLInputElement;
 	let inputFiles;
 	let tags = [];
 
@@ -69,7 +69,7 @@
 			}
 
 			inputFiles = null;
-			document.getElementById('upload-doc-input').value = '';
+			uploadDocInputElement.value = '';
 		} else {
 			toast.error(`File not found.`);
 		}
@@ -126,13 +126,20 @@
 					}}
 				>
 					<div class="mb-3 w-full">
-						<input id="upload-doc-input" hidden bind:files={inputFiles} type="file" multiple />
+						<input
+							id="upload-doc-input"
+							bind:this={uploadDocInputElement}
+							hidden
+							bind:files={inputFiles}
+							type="file"
+							multiple
+						/>
 
 						<button
 							class="w-full text-sm font-medium py-3 bg-gray-850 hover:bg-gray-800 text-center rounded-xl"
 							type="button"
 							on:click={() => {
-								document.getElementById('upload-doc-input')?.click();
+								uploadDocInputElement.click();
 							}}
 						>
 							{#if inputFiles}

src/lib/components/documents/Settings/General.svelte

@@ -1,10 +1,10 @@
 <script lang="ts">
 	import { getDocs } from '$lib/apis/documents';
 	import {
-		getChunkParams,
+		getRAGConfig,
+		updateRAGConfig,
 		getQuerySettings,
 		scanDocs,
-		updateChunkParams,
 		updateQuerySettings
 	} from '$lib/apis/rag';
 	import { documents } from '$lib/stores';
@@ -17,6 +17,7 @@
 
 	let chunkSize = 0;
 	let chunkOverlap = 0;
+	let pdfExtractImages = true;
 
 	let querySettings = {
 		template: '',
@@ -35,16 +36,24 @@
 	};
 
 	const submitHandler = async () => {
-		const res = await updateChunkParams(localStorage.token, chunkSize, chunkOverlap);
+		const res = await updateRAGConfig(localStorage.token, {
+			pdf_extract_images: pdfExtractImages,
+			chunk: {
+				chunk_overlap: chunkOverlap,
+				chunk_size: chunkSize
+			}
+		});
 		querySettings = await updateQuerySettings(localStorage.token, querySettings);
 	};
 
 	onMount(async () => {
-		const res = await getChunkParams(localStorage.token);
+		const res = await getRAGConfig(localStorage.token);
 
 		if (res) {
-			chunkSize = res.chunk_size;
-			chunkOverlap = res.chunk_overlap;
+			pdfExtractImages = res.pdf_extract_images;
+
+			chunkSize = res.chunk.chunk_size;
+			chunkOverlap = res.chunk.chunk_overlap;
 		}
 
 		querySettings = await getQuerySettings(localStorage.token);
@@ -124,82 +133,100 @@
 
 		<hr class=" dark:border-gray-700" />
 
-		<div class=" ">
-			<div class=" text-sm font-medium">Chunk Params</div>
+		<div class=" space-y-3">
+			<div class=" space-y-3">
+				<div class=" text-sm font-medium">Chunk Params</div>
 
-			<div class=" flex">
-				<div class="  flex w-full justify-between">
-					<div class="self-center text-xs font-medium min-w-fit">Chunk Size</div>
+				<div class=" flex gap-2">
+					<div class="  flex w-full justify-between gap-2">
+						<div class="self-center text-xs font-medium min-w-fit">Chunk Size</div>
 
-					<div class="self-center p-3">
-						<input
-							class=" w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
-							type="number"
-							placeholder="Enter Chunk Size"
-							bind:value={chunkSize}
-							autocomplete="off"
-							min="0"
-						/>
+						<div class="self-center">
+							<input
+								class=" w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
+								type="number"
+								placeholder="Enter Chunk Size"
+								bind:value={chunkSize}
+								autocomplete="off"
+								min="0"
+							/>
+						</div>
+					</div>
+
+					<div class="flex w-full gap-2">
+						<div class=" self-center text-xs font-medium min-w-fit">Chunk Overlap</div>
+
+						<div class="self-center">
+							<input
+								class="w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
+								type="number"
+								placeholder="Enter Chunk Overlap"
+								bind:value={chunkOverlap}
+								autocomplete="off"
+								min="0"
+							/>
+						</div>
 					</div>
 				</div>
 
-				<div class="flex w-full">
-					<div class=" self-center text-xs font-medium min-w-fit">Chunk Overlap</div>
+				<div>
+					<div class="flex justify-between items-center text-xs">
+						<div class=" text-xs font-medium">PDF Extract Images (OCR)</div>
 
-					<div class="self-center p-3">
-						<input
-							class="w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
-							type="number"
-							placeholder="Enter Chunk Overlap"
-							bind:value={chunkOverlap}
-							autocomplete="off"
-							min="0"
-						/>
+						<button
+							class=" text-xs font-medium text-gray-500"
+							type="button"
+							on:click={() => {
+								pdfExtractImages = !pdfExtractImages;
+							}}>{pdfExtractImages ? 'On' : 'Off'}</button
+						>
 					</div>
 				</div>
 			</div>
 
-			<div class=" text-sm font-medium">Query Params</div>
-
-			<div class=" flex">
-				<div class="  flex w-full justify-between">
-					<div class="self-center text-xs font-medium flex-1">Top K</div>
-
-					<div class="self-center p-3">
-						<input
-							class=" w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
-							type="number"
-							placeholder="Enter Top K"
-							bind:value={querySettings.k}
-							autocomplete="off"
-							min="0"
-						/>
-					</div>
-				</div>
-
-				<!-- <div class="flex w-full">
-					<div class=" self-center text-xs font-medium min-w-fit">Chunk Overlap</div>
-
-					<div class="self-center p-3">
-						<input
-							class="w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
-							type="number"
-							placeholder="Enter Chunk Overlap"
-							bind:value={chunkOverlap}
-							autocomplete="off"
-							min="0"
-						/>
-					</div>
-				</div> -->
-			</div>
-
 			<div>
-				<div class=" mb-2.5 text-sm font-medium">RAG Template</div>
-				<textarea
-					bind:value={querySettings.template}
-					class="w-full rounded p-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
-					rows="4"
-				/>
+				<div class=" text-sm font-medium">Query Params</div>
+
+				<div class=" flex py-2">
+					<div class="  flex w-full justify-between gap-2">
+						<div class="self-center text-xs font-medium flex-1">Top K</div>
+
+						<div class="self-center">
+							<input
+								class=" w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
+								type="number"
+								placeholder="Enter Top K"
+								bind:value={querySettings.k}
+								autocomplete="off"
+								min="0"
+							/>
+						</div>
+					</div>
+
+					<!-- <div class="flex w-full">
+						<div class=" self-center text-xs font-medium min-w-fit">Chunk Overlap</div>
+	
+						<div class="self-center p-3">
+							<input
+								class="w-full rounded py-1.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none border border-gray-100 dark:border-gray-600"
+								type="number"
+								placeholder="Enter Chunk Overlap"
+								bind:value={chunkOverlap}
+								autocomplete="off"
+								min="0"
+							/>
+						</div>
+					</div> -->
+				</div>
+
+				<div>
+					<div class=" mb-2.5 text-sm font-medium">RAG Template</div>
+					<textarea
+						bind:value={querySettings.template}
+						class="w-full rounded p-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
+						rows="4"
+					/>
+				</div>
 			</div>
 		</div>
 	</div>

src/lib/components/layout/Sidebar.svelte

@@ -61,12 +61,16 @@
 	};
 
 	const editChatTitle = async (id, _title) => {
-		title = _title;
+		if (_title === '') {
+			toast.error('Title cannot be an empty string.');
+		} else {
+			title = _title;
 
-		await updateChatById(localStorage.token, id, {
-			title: _title
-		});
-		await chats.set(await getChatList(localStorage.token));
+			await updateChatById(localStorage.token, id, {
+				title: _title
+			});
+			await chats.set(await getChatList(localStorage.token));
+		}
 	};
 
 	const deleteChat = async (id) => {
@@ -388,12 +392,13 @@
 										show = false;
 									}
 								}}
+								draggable="false"
 							>
 								<div class=" flex self-center flex-1 w-full">
 									<div
 										class=" text-left self-center overflow-hidden {chat.id === $chatId
 											? 'w-[160px]'
-											: 'w-full'} "
+											: 'w-full'}  h-[20px]"
 									>
 										{chat.title}
 									</div>

src/lib/components/playground/ChatCompletion.svelte

@@ -2,12 +2,11 @@
 	import { onMount } from 'svelte';
 
 	export let messages = [];
-
+	let textAreaElement: HTMLTextAreaElement;
 	onMount(() => {
 		messages.forEach((message, idx) => {
-			let textareaElement = document.getElementById(`${message.role}-${idx}-textarea`);
-			textareaElement.style.height = '';
-			textareaElement.style.height = textareaElement.scrollHeight + 'px';
+			textAreaElement.style.height = '';
+			textAreaElement.style.height = textAreaElement.scrollHeight + 'px';
 		});
 	});
 </script>
@@ -27,16 +26,17 @@
 			<div class="flex-1">
 				<textarea
 					id="{message.role}-{idx}-textarea"
+					bind:this={textAreaElement}
 					class="w-full bg-transparent outline-none rounded-lg p-2 text-sm resize-none overflow-hidden"
 					placeholder="Enter {message.role === 'user' ? 'a user' : 'an assistant'} message here"
 					rows="1"
 					on:input={(e) => {
-						e.target.style.height = '';
-						e.target.style.height = e.target.scrollHeight + 'px';
+						textAreaElement.style.height = '';
+						textAreaElement.style.height = textAreaElement.scrollHeight + 'px';
 					}}
 					on:focus={(e) => {
-						e.target.style.height = '';
-						e.target.style.height = e.target.scrollHeight + 'px';
+						textAreaElement.style.height = '';
+						textAreaElement.style.height = textAreaElement.scrollHeight + 'px';
 
 						// e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
 					}}

src/lib/constants.ts

@@ -7,7 +7,7 @@ export const WEBUI_BASE_URL = dev ? `http://${location.hostname}:8080` : ``;
 export const WEBUI_API_BASE_URL = `${WEBUI_BASE_URL}/api/v1`;
 
 export const LITELLM_API_BASE_URL = `${WEBUI_BASE_URL}/litellm/api`;
-export const OLLAMA_API_BASE_URL = `${WEBUI_BASE_URL}/ollama/api`;
+export const OLLAMA_API_BASE_URL = `${WEBUI_BASE_URL}/ollama`;
 export const OPENAI_API_BASE_URL = `${WEBUI_BASE_URL}/openai/api`;
 export const AUDIO_API_BASE_URL = `${WEBUI_BASE_URL}/audio/api/v1`;
 export const IMAGES_API_BASE_URL = `${WEBUI_BASE_URL}/images/api/v1`;
@@ -90,8 +90,3 @@ export const SUPPORTED_FILE_EXTENSIONS = [
 // This feature, akin to $env/static/private, exclusively incorporates environment variables
 // that are prefixed with config.kit.env.publicPrefix (usually set to PUBLIC_).
 // Consequently, these variables can be securely exposed to client-side code.
-
-// Example of the .env configuration:
-// OLLAMA_API_BASE_URL="http://localhost:11434/api"
-// # Public
-// PUBLIC_API_BASE_URL=$OLLAMA_API_BASE_URL

src/routes/(app)/+layout.svelte

@@ -34,10 +34,11 @@
 	import Sidebar from '$lib/components/layout/Sidebar.svelte';
 	import ShortcutsModal from '$lib/components/chat/ShortcutsModal.svelte';
 	import ChangelogModal from '$lib/components/ChangelogModal.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
 	let ollamaVersion = '';
 	let loaded = false;
-
+	let showShortcutsButtonElement: HTMLButtonElement;
 	let DB = null;
 	let localDBChats = [];
 
@@ -96,14 +97,11 @@
 					if (localDBChats.length === 0) {
 						await deleteDB('Chats');
 					}
-
-					console.log('localdb', localDBChats);
 				}
 
 				console.log(DB);
 			} catch (error) {
 				// IndexedDB Not Found
-				console.log('IDB Not Found');
 			}
 
 			console.log();
@@ -184,7 +182,7 @@
 				if (isCtrlPressed && event.key === '/') {
 					event.preventDefault();
 					console.log('showShortcuts');
-					document.getElementById('show-shortcuts-button')?.click();
+					showShortcutsButtonElement.click();
 				}
 			});
 
@@ -201,15 +199,18 @@
 
 {#if loaded}
 	<div class=" hidden lg:flex fixed bottom-0 right-0 px-3 py-3 z-10">
-		<button
-			id="show-shortcuts-button"
-			class="text-gray-600 dark:text-gray-300 bg-gray-300/20 w-6 h-6 flex items-center justify-center text-xs rounded-full"
-			on:click={() => {
-				showShortcuts = !showShortcuts;
-			}}
-		>
-			?
-		</button>
+		<Tooltip content="help" placement="left">
+			<button
+				id="show-shortcuts-button"
+				bind:this={showShortcutsButtonElement}
+				class="text-gray-600 dark:text-gray-300 bg-gray-300/20 w-6 h-6 flex items-center justify-center text-xs rounded-full"
+				on:click={() => {
+					showShortcuts = !showShortcuts;
+				}}
+			>
+				?
+			</button>
+		</Tooltip>
 	</div>
 
 	<ShortcutsModal bind:show={showShortcuts} />

src/routes/(app)/+page.svelte

@@ -42,7 +42,7 @@
 	let stopResponseFlag = false;
 	let autoScroll = true;
 	let processing = '';
-
+	let messagesContainerElement: HTMLDivElement;
 	let currentRequestId = null;
 
 	let selectedModels = [''];
@@ -140,8 +140,9 @@
 	};
 
 	const scrollToBottom = () => {
-		const element = document.getElementById('messages-container');
-		element.scrollTop = element.scrollHeight;
+		if (messagesContainerElement) {
+			messagesContainerElement.scrollTop = messagesContainerElement.scrollHeight;
+		}
 	};
 
 	//////////////////////////
@@ -233,53 +234,6 @@
 	const sendPrompt = async (prompt, parentId) => {
 		const _chatId = JSON.parse(JSON.stringify($chatId));
 
-		const docs = messages
-			.filter((message) => message?.files ?? null)
-			.map((message) =>
-				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
-			)
-			.flat(1);
-
-		console.log(docs);
-		if (docs.length > 0) {
-			processing = 'Reading';
-			const query = history.messages[parentId].content;
-
-			let relevantContexts = await Promise.all(
-				docs.map(async (doc) => {
-					if (doc.type === 'collection') {
-						return await queryCollection(localStorage.token, doc.collection_names, query).catch(
-							(error) => {
-								console.log(error);
-								return null;
-							}
-						);
-					} else {
-						return await queryDoc(localStorage.token, doc.collection_name, query).catch((error) => {
-							console.log(error);
-							return null;
-						});
-					}
-				})
-			);
-			relevantContexts = relevantContexts.filter((context) => context);
-
-			const contextString = relevantContexts.reduce((a, context, i, arr) => {
-				return `${a}${context.documents.join(' ')}\n`;
-			}, '');
-
-			console.log(contextString);
-
-			history.messages[parentId].raContent = await RAGTemplate(
-				localStorage.token,
-				contextString,
-				query
-			);
-			history.messages[parentId].contexts = relevantContexts;
-			await tick();
-			processing = '';
-		}
-
 		await Promise.all(
 			selectedModels.map(async (modelId) => {
 				const model = $models.filter((m) => m.id === modelId).at(0);
@@ -340,18 +294,28 @@
 						content: $settings.system
 				  }
 				: undefined,
-			...messages.filter((message) => !message.deleted)
+			...messages
 		]
 			.filter((message) => message)
-			.map((message, idx, arr) => ({
-				role: message.role,
-				content: arr.length - 2 !== idx ? message.content : message?.raContent ?? message.content,
-				...(message.files && {
-					images: message.files
-						.filter((file) => file.type === 'image')
-						.map((file) => file.url.slice(file.url.indexOf(',') + 1))
-				})
-			}));
+			.map((message, idx, arr) => {
+				// Prepare the base message object
+				const baseMessage = {
+					role: message.role,
+					content: arr.length - 2 !== idx ? message.content : message?.raContent ?? message.content
+				};
+
+				// Extract and format image URLs if any exist
+				const imageUrls = message.files
+					?.filter((file) => file.type === 'image')
+					.map((file) => file.url.slice(file.url.indexOf(',') + 1));
+
+				// Add images array only if it contains elements
+				if (imageUrls && imageUrls.length > 0 && message.role === 'user') {
+					baseMessage.images = imageUrls;
+				}
+
+				return baseMessage;
+			});
 
 		let lastImageIndex = -1;
 
@@ -369,6 +333,13 @@
 			}
 		});
 
+		const docs = messages
+			.filter((message) => message?.files ?? null)
+			.map((message) =>
+				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+			)
+			.flat(1);
+
 		const [res, controller] = await generateChatCompletion(localStorage.token, {
 			model: model,
 			messages: messagesBody,
@@ -376,7 +347,8 @@
 				...($settings.options ?? {})
 			},
 			format: $settings.requestFormat ?? undefined,
-			keep_alive: $settings.keepAlive ?? undefined
+			keep_alive: $settings.keepAlive ?? undefined,
+			docs: docs.length > 0 ? docs : undefined
 		});
 
 		if (res && res.ok) {
@@ -536,6 +508,15 @@
 		const responseMessage = history.messages[responseMessageId];
 		scrollToBottom();
 
+		const docs = messages
+			.filter((message) => message?.files ?? null)
+			.map((message) =>
+				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+			)
+			.flat(1);
+
+		console.log(docs);
+
 		const res = await generateOpenAIChatCompletion(
 			localStorage.token,
 			{
@@ -548,12 +529,13 @@
 								content: $settings.system
 						  }
 						: undefined,
-					...messages.filter((message) => !message.deleted)
+					...messages
 				]
 					.filter((message) => message)
 					.map((message, idx, arr) => ({
 						role: message.role,
-						...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false
+						...((message.files?.filter((file) => file.type === 'image').length > 0 ?? false) &&
+						message.role === 'user'
 							? {
 									content: [
 										{
@@ -584,7 +566,8 @@
 				top_p: $settings?.options?.top_p ?? undefined,
 				num_ctx: $settings?.options?.num_ctx ?? undefined,
 				frequency_penalty: $settings?.options?.repeat_penalty ?? undefined,
-				max_tokens: $settings?.options?.num_predict ?? undefined
+				max_tokens: $settings?.options?.num_predict ?? undefined,
+				docs: docs.length > 0 ? docs : undefined
 			},
 			model.source === 'litellm' ? `${LITELLM_API_BASE_URL}/v1` : `${OPENAI_API_BASE_URL}`
 		);
@@ -695,7 +678,12 @@
 
 		if (messages.length == 2) {
 			window.history.replaceState(history.state, '', `/c/${_chatId}`);
-			await setChatTitle(_chatId, userPrompt);
+
+			if ($settings?.titleAutoGenerateModel) {
+				await generateChatTitle(_chatId, userPrompt);
+			} else {
+				await setChatTitle(_chatId, userPrompt);
+			}
 		}
 	};
 
@@ -821,8 +809,11 @@
 		<div
 			class=" pb-2.5 flex flex-col justify-between w-full flex-auto overflow-auto h-0"
 			id="messages-container"
+			bind:this={messagesContainerElement}
 			on:scroll={(e) => {
-				autoScroll = e.target.scrollHeight - e.target.scrollTop <= e.target.clientHeight + 50;
+				autoScroll =
+					messagesContainerElement.scrollHeight - messagesContainerElement.scrollTop <=
+					messagesContainerElement.clientHeight + 50;
 			}}
 		>
 			<div
@@ -830,10 +821,7 @@
 					? 'max-w-full'
 					: 'max-w-2xl md:px-0'} mx-auto w-full px-4"
 			>
-				<ModelSelector
-					bind:selectedModels
-					disabled={messages.length > 0 && !selectedModels.includes('')}
-				/>
+				<ModelSelector bind:selectedModels />
 			</div>
 
 			<div class=" h-full w-full flex flex-col py-8">

src/routes/(app)/c/[id]/+page.svelte

@@ -45,7 +45,7 @@
 	let stopResponseFlag = false;
 	let autoScroll = true;
 	let processing = '';
-
+	let messagesContainerElement: HTMLDivElement;
 	let currentRequestId = null;
 
 	// let chatId = $page.params.id;
@@ -160,8 +160,9 @@
 	};
 
 	const scrollToBottom = () => {
-		const element = document.getElementById('messages-container');
-		element.scrollTop = element.scrollHeight;
+		if (messagesContainerElement) {
+			messagesContainerElement.scrollTop = messagesContainerElement.scrollHeight;
+		}
 	};
 
 	//////////////////////////
@@ -246,53 +247,6 @@
 	const sendPrompt = async (prompt, parentId) => {
 		const _chatId = JSON.parse(JSON.stringify($chatId));
 
-		const docs = messages
-			.filter((message) => message?.files ?? null)
-			.map((message) =>
-				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
-			)
-			.flat(1);
-
-		console.log(docs);
-		if (docs.length > 0) {
-			processing = 'Reading';
-			const query = history.messages[parentId].content;
-
-			let relevantContexts = await Promise.all(
-				docs.map(async (doc) => {
-					if (doc.type === 'collection') {
-						return await queryCollection(localStorage.token, doc.collection_names, query).catch(
-							(error) => {
-								console.log(error);
-								return null;
-							}
-						);
-					} else {
-						return await queryDoc(localStorage.token, doc.collection_name, query).catch((error) => {
-							console.log(error);
-							return null;
-						});
-					}
-				})
-			);
-			relevantContexts = relevantContexts.filter((context) => context);
-
-			const contextString = relevantContexts.reduce((a, context, i, arr) => {
-				return `${a}${context.documents.join(' ')}\n`;
-			}, '');
-
-			console.log(contextString);
-
-			history.messages[parentId].raContent = await RAGTemplate(
-				localStorage.token,
-				contextString,
-				query
-			);
-			history.messages[parentId].contexts = relevantContexts;
-			await tick();
-			processing = '';
-		}
-
 		await Promise.all(
 			selectedModels.map(async (modelId) => {
 				const model = $models.filter((m) => m.id === modelId).at(0);
@@ -353,18 +307,28 @@
 						content: $settings.system
 				  }
 				: undefined,
-			...messages.filter((message) => !message.deleted)
+			...messages
 		]
 			.filter((message) => message)
-			.map((message, idx, arr) => ({
-				role: message.role,
-				content: arr.length - 2 !== idx ? message.content : message?.raContent ?? message.content,
-				...(message.files && {
-					images: message.files
-						.filter((file) => file.type === 'image')
-						.map((file) => file.url.slice(file.url.indexOf(',') + 1))
-				})
-			}));
+			.map((message, idx, arr) => {
+				// Prepare the base message object
+				const baseMessage = {
+					role: message.role,
+					content: arr.length - 2 !== idx ? message.content : message?.raContent ?? message.content
+				};
+
+				// Extract and format image URLs if any exist
+				const imageUrls = message.files
+					?.filter((file) => file.type === 'image')
+					.map((file) => file.url.slice(file.url.indexOf(',') + 1));
+
+				// Add images array only if it contains elements
+				if (imageUrls && imageUrls.length > 0 && message.role === 'user') {
+					baseMessage.images = imageUrls;
+				}
+
+				return baseMessage;
+			});
 
 		let lastImageIndex = -1;
 
@@ -382,6 +346,13 @@
 			}
 		});
 
+		const docs = messages
+			.filter((message) => message?.files ?? null)
+			.map((message) =>
+				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+			)
+			.flat(1);
+
 		const [res, controller] = await generateChatCompletion(localStorage.token, {
 			model: model,
 			messages: messagesBody,
@@ -389,7 +360,8 @@
 				...($settings.options ?? {})
 			},
 			format: $settings.requestFormat ?? undefined,
-			keep_alive: $settings.keepAlive ?? undefined
+			keep_alive: $settings.keepAlive ?? undefined,
+			docs: docs.length > 0 ? docs : undefined
 		});
 
 		if (res && res.ok) {
@@ -549,6 +521,15 @@
 		const responseMessage = history.messages[responseMessageId];
 		scrollToBottom();
 
+		const docs = messages
+			.filter((message) => message?.files ?? null)
+			.map((message) =>
+				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+			)
+			.flat(1);
+
+		console.log(docs);
+
 		const res = await generateOpenAIChatCompletion(
 			localStorage.token,
 			{
@@ -561,12 +542,13 @@
 								content: $settings.system
 						  }
 						: undefined,
-					...messages.filter((message) => !message.deleted)
+					...messages
 				]
 					.filter((message) => message)
 					.map((message, idx, arr) => ({
 						role: message.role,
-						...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false
+						...((message.files?.filter((file) => file.type === 'image').length > 0 ?? false) &&
+						message.role === 'user'
 							? {
 									content: [
 										{
@@ -597,7 +579,8 @@
 				top_p: $settings?.options?.top_p ?? undefined,
 				num_ctx: $settings?.options?.num_ctx ?? undefined,
 				frequency_penalty: $settings?.options?.repeat_penalty ?? undefined,
-				max_tokens: $settings?.options?.num_predict ?? undefined
+				max_tokens: $settings?.options?.num_predict ?? undefined,
+				docs: docs.length > 0 ? docs : undefined
 			},
 			model.source === 'litellm' ? `${LITELLM_API_BASE_URL}/v1` : `${OPENAI_API_BASE_URL}`
 		);
@@ -708,9 +691,15 @@
 
 		if (messages.length == 2) {
 			window.history.replaceState(history.state, '', `/c/${_chatId}`);
-			await setChatTitle(_chatId, userPrompt);
+
+			if ($settings?.titleAutoGenerateModel) {
+				await generateChatTitle(_chatId, userPrompt);
+			} else {
+				await setChatTitle(_chatId, userPrompt);
+			}
 		}
 	};
+
 	const stopResponse = () => {
 		stopResponseFlag = true;
 		console.log('stopResponse');
@@ -852,8 +841,11 @@
 			<div
 				class=" pb-2.5 flex flex-col justify-between w-full flex-auto overflow-auto h-0"
 				id="messages-container"
+				bind:this={messagesContainerElement}
 				on:scroll={(e) => {
-					autoScroll = e.target.scrollHeight - e.target.scrollTop <= e.target.clientHeight + 50;
+					autoScroll =
+						messagesContainerElement.scrollHeight - messagesContainerElement.scrollTop <=
+						messagesContainerElement.clientHeight + 50;
 				}}
 			>
 				<div
@@ -861,10 +853,7 @@
 						? 'max-w-full'
 						: 'max-w-2xl md:px-0'} mx-auto w-full px-4"
 				>
-					<ModelSelector
-						bind:selectedModels
-						disabled={messages.length > 0 && !selectedModels.includes('')}
-					/>
+					<ModelSelector bind:selectedModels />
 				</div>
 
 				<div class=" h-full w-full flex flex-col py-8">

src/routes/(app)/documents/+page.svelte

@@ -21,7 +21,7 @@
 
 	let inputFiles = '';
 	let query = '';
-
+	let documentsImportInputElement: HTMLInputElement;
 	let tags = [];
 
 	let showSettingsModal = false;
@@ -524,6 +524,7 @@
 				<div class="flex space-x-2">
 					<input
 						id="documents-import-input"
+						bind:this={documentsImportInputElement}
 						bind:files={importFiles}
 						type="file"
 						accept=".json"
@@ -558,8 +559,8 @@
 
 					<button
 						class="flex text-xs items-center space-x-1 px-3 py-1.5 rounded-xl bg-gray-50 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 dark:text-gray-200 transition"
-						on:click={async () => {
-							document.getElementById('documents-import-input')?.click();
+						on:click={() => {
+							documentsImportInputElement.click();
 						}}
 					>
 						<div class=" self-center mr-2 font-medium">Import Documents Mapping</div>

src/routes/(app)/modelfiles/+page.svelte

@@ -16,11 +16,14 @@
 
 	let localModelfiles = [];
 	let importFiles;
-
+	let modelfilesImportInputElement: HTMLInputElement;
 	const deleteModelHandler = async (tagName) => {
 		let success = null;
 
-		success = await deleteModel(localStorage.token, tagName);
+		success = await deleteModel(localStorage.token, tagName).catch((err) => {
+			toast.error(err);
+			return null;
+		});
 
 		if (success) {
 			toast.success(`Deleted ${tagName}`);
@@ -235,6 +238,7 @@
 				<div class="flex space-x-1">
 					<input
 						id="modelfiles-import-input"
+						bind:this={modelfilesImportInputElement}
 						bind:files={importFiles}
 						type="file"
 						accept=".json"
@@ -262,8 +266,8 @@
 
 					<button
 						class="flex text-xs items-center space-x-1 px-3 py-1.5 rounded-xl bg-gray-50 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 dark:text-gray-200 transition"
-						on:click={async () => {
-							document.getElementById('modelfiles-import-input')?.click();
+						on:click={() => {
+							modelfilesImportInputElement.click();
 						}}
 					>
 						<div class=" self-center mr-2 font-medium">Import Modelfiles</div>

src/routes/(app)/playground/+page.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
 	import { goto } from '$app/navigation';
 
 	import { onMount, tick } from 'svelte';
@@ -21,15 +21,17 @@
 
 	let mode = 'chat';
 	let loaded = false;
-
 	let text = '';
 
 	let selectedModelId = '';
 
 	let loading = false;
-	let currentRequestId;
+	let currentRequestId = null;
 	let stopResponseFlag = false;
 
+	let messagesContainerElement: HTMLDivElement;
+	let textCompletionAreaElement: HTMLTextAreaElement;
+
 	let system = '';
 	let messages = [
 		{
@@ -39,13 +41,7 @@
 	];
 
 	const scrollToBottom = () => {
-		let element;
-
-		if (mode === 'chat') {
-			element = document.getElementById('messages-container');
-		} else {
-			element = document.getElementById('text-completion-textarea');
-		}
+		const element = mode === 'chat' ? messagesContainerElement : textCompletionAreaElement;
 
 		if (element) {
 			element.scrollTop = element?.scrollHeight;
@@ -96,6 +92,10 @@
 			while (true) {
 				const { value, done } = await reader.read();
 				if (done || stopResponseFlag) {
+					if (stopResponseFlag) {
+						await cancelChatCompletion(localStorage.token, currentRequestId);
+					}
+
 					currentRequestId = null;
 					break;
 				}
@@ -112,7 +112,11 @@
 								let data = JSON.parse(line.replace(/^data: /, ''));
 								console.log(data);
 
-								text += data.choices[0].delta.content ?? '';
+								if ('request_id' in data) {
+									currentRequestId = data.request_id;
+								} else {
+									text += data.choices[0].delta.content ?? '';
+								}
 							}
 						}
 					}
@@ -150,16 +154,6 @@
 				: `${OLLAMA_API_BASE_URL}/v1`
 		);
 
-		// const [res, controller] = await generateChatCompletion(localStorage.token, {
-		// 	model: selectedModelId,
-		// 	messages: [
-		// 		{
-		// 			role: 'assistant',
-		// 			content: text
-		// 		}
-		// 	]
-		// });
-
 		let responseMessage;
 		if (messages.at(-1)?.role === 'assistant') {
 			responseMessage = messages.at(-1);
@@ -184,6 +178,11 @@
 			while (true) {
 				const { value, done } = await reader.read();
 				if (done || stopResponseFlag) {
+					if (stopResponseFlag) {
+						await cancelChatCompletion(localStorage.token, currentRequestId);
+					}
+
+					currentRequestId = null;
 					break;
 				}
 
@@ -200,17 +199,21 @@
 								let data = JSON.parse(line.replace(/^data: /, ''));
 								console.log(data);
 
-								if (responseMessage.content == '' && data.choices[0].delta.content == '\n') {
-									continue;
+								if ('request_id' in data) {
+									currentRequestId = data.request_id;
 								} else {
-									textareaElement.style.height = textareaElement.scrollHeight + 'px';
+									if (responseMessage.content == '' && data.choices[0].delta.content == '\n') {
+										continue;
+									} else {
+										textareaElement.style.height = textareaElement.scrollHeight + 'px';
 
-									responseMessage.content += data.choices[0].delta.content ?? '';
-									messages = messages;
+										responseMessage.content += data.choices[0].delta.content ?? '';
+										messages = messages;
 
-									textareaElement.style.height = textareaElement.scrollHeight + 'px';
+										textareaElement.style.height = textareaElement.scrollHeight + 'px';
 
-									await tick();
+										await tick();
+									}
 								}
 							}
 						}
@@ -221,48 +224,6 @@
 
 				scrollToBottom();
 			}
-
-			// while (true) {
-			// 	const { value, done } = await reader.read();
-			// 	if (done || stopResponseFlag) {
-			// 		if (stopResponseFlag) {
-			// 			await cancelChatCompletion(localStorage.token, currentRequestId);
-			// 		}
-
-			// 		currentRequestId = null;
-			// 		break;
-			// 	}
-
-			// 	try {
-			// 		let lines = value.split('\n');
-
-			// 		for (const line of lines) {
-			// 			if (line !== '') {
-			// 				console.log(line);
-			// 				let data = JSON.parse(line);
-
-			// 				if ('detail' in data) {
-			// 					throw data;
-			// 				}
-
-			// 				if ('id' in data) {
-			// 					console.log(data);
-			// 					currentRequestId = data.id;
-			// 				} else {
-			// 					if (data.done == false) {
-			// 						text += data.message.content;
-			// 					} else {
-			// 						console.log('done');
-			// 					}
-			// 				}
-			// 			}
-			// 		}
-			// 	} catch (error) {
-			// 		console.log(error);
-			// 	}
-
-			// 	scrollToBottom();
-			// }
 		}
 	};
 
@@ -306,7 +267,7 @@
 
 <div class="min-h-screen max-h-[100dvh] w-full flex justify-center dark:text-white">
 	<div class=" flex flex-col justify-between w-full overflow-y-auto h-[100dvh]">
-		<div class="max-w-2xl mx-auto w-full px-3 p-3 md:px-0 h-full">
+		<div class="max-w-2xl mx-auto w-full px-3 md:px-0 my-10 h-full">
 			<div class=" flex flex-col h-full">
 				<div class="flex flex-col justify-between mb-2.5 gap-1">
 					<div class="flex justify-between items-center gap-2">
@@ -417,12 +378,14 @@
 				<div
 					class=" pb-2.5 flex flex-col justify-between w-full flex-auto overflow-auto h-0"
 					id="messages-container"
+					bind:this={messagesContainerElement}
 				>
 					<div class=" h-full w-full flex flex-col">
 						<div class="flex-1 p-1">
 							{#if mode === 'complete'}
 								<textarea
 									id="text-completion-textarea"
+									bind:this={textCompletionAreaElement}
 									class="w-full h-full p-3 bg-transparent outline outline-1 outline-gray-200 dark:outline-gray-800 resize-none rounded-lg text-sm"
 									bind:value={text}
 									placeholder="You're a helpful assistant."

src/routes/(app)/prompts/+page.svelte

@@ -11,7 +11,7 @@
 
 	let importFiles = '';
 	let query = '';
-
+	let promptsImportInputElement: HTMLInputElement;
 	const sharePrompt = async (prompt) => {
 		toast.success('Redirecting you to OpenWebUI Community');
 
@@ -208,6 +208,7 @@
 				<div class="flex space-x-2">
 					<input
 						id="prompts-import-input"
+						bind:this={promptsImportInputElement}
 						bind:files={importFiles}
 						type="file"
 						accept=".json"
@@ -241,8 +242,8 @@
 
 					<button
 						class="flex text-xs items-center space-x-1 px-3 py-1.5 rounded-xl bg-gray-50 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 dark:text-gray-200 transition"
-						on:click={async () => {
-							document.getElementById('prompts-import-input')?.click();
+						on:click={() => {
+							promptsImportInputElement.click();
 						}}
 					>
 						<div class=" self-center mr-2 font-medium">Import Prompts</div>
@@ -266,7 +267,7 @@
 					<button
 						class="flex text-xs items-center space-x-1 px-3 py-1.5 rounded-xl bg-gray-50 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 dark:text-gray-200 transition"
 						on:click={async () => {
-							// document.getElementById('modelfiles-import-input')?.click();
+							// promptsImportInputElement.click();
 							let blob = new Blob([JSON.stringify($prompts)], {
 								type: 'application/json'
 							});

src/tailwind.css

@@ -3,16 +3,13 @@
 @tailwind utilities;
 
 @layer base {
-	html {
+	html, pre {
 		font-family: -apple-system, 'Arimo', ui-sans-serif, system-ui, 'Segoe UI', Roboto, Ubuntu,
 			Cantarell, 'Noto Sans', sans-serif, 'Helvetica Neue', Arial, 'Apple Color Emoji',
 			'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji';
 	}
 
-	pre {
-		font-family: -apple-system, 'Arimo', ui-sans-serif, system-ui, 'Segoe UI', Roboto, Ubuntu,
-			Cantarell, 'Noto Sans', sans-serif, 'Helvetica Neue', Arial, 'Apple Color Emoji',
-			'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji';
-		white-space: pre-wrap;
-	}
+  pre {
+    white-space: pre-wrap;
+  }
 }