refac

backend/open_webui/config.py

@@ -857,6 +857,16 @@ EXTERNAL_DOCUMENT_LOADER_URL = os.getenv('EXTERNAL_DOCUMENT_LOADER_URL', '')
 
 EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '')
 
+external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '')
+try:
+    external_document_loader_headers = json.loads(external_document_loader_headers)
+except json.JSONDecodeError:
+    external_document_loader_headers = {}
+if not isinstance(external_document_loader_headers, dict):
+    external_document_loader_headers = {}
+
+EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
+
 TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
 
 DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
@@ -2652,6 +2662,7 @@ DEFAULT_CONFIG = {
     'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS,
     'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL,
     'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
+    'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
     'rag.tika_server_url': TIKA_SERVER_URL,
     'rag.docling_server_url': DOCLING_SERVER_URL,
     'rag.docling_api_key': DOCLING_API_KEY,

backend/open_webui/models/knowledge.py

@@ -286,6 +286,17 @@ class KnowledgeTable:
                     elif view_option == 'shared':
                         stmt = stmt.filter(Knowledge.user_id != user_id)
 
+                    source = filter.get('source')
+                    if source == 'external':
+                        stmt = stmt.filter(Knowledge.meta['source'].as_string() == 'external')
+                    elif source == 'local':
+                        stmt = stmt.filter(
+                            or_(
+                                Knowledge.meta.is_(None),
+                                Knowledge.meta['source'].as_string() != 'external',
+                            )
+                        )
+
                     stmt = AccessGrants.has_permission_filter(
                         db=db,
                         query=stmt,
@@ -765,6 +776,25 @@ class KnowledgeTable:
             log.exception(e)
             return None
 
+    async def update_knowledge_meta_by_id(
+        self, id: str, meta: dict, db: Optional[AsyncSession] = None
+    ) -> Optional[KnowledgeModel]:
+        try:
+            async with get_async_db_context(db) as db:
+                await db.execute(
+                    update(Knowledge)
+                    .filter_by(id=id)
+                    .values(
+                        meta=meta,
+                        updated_at=int(time.time()),
+                    )
+                )
+                await db.commit()
+                return await self.get_knowledge_by_id(id=id, db=db)
+        except Exception as e:
+            log.exception(e)
+            return None
+
     async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
         try:
             async with get_async_db_context(db) as db:

backend/open_webui/retrieval/external.py

@@ -0,0 +1,378 @@
+import asyncio
+import logging
+import re
+import time
+from typing import Any, Optional
+
+from open_webui.models.config import Config
+from open_webui.models.knowledge import KnowledgeModel
+
+log = logging.getLogger(__name__)
+
+EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
+IDENTIFIER_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
+
+
+async def _get_external_connection(connection_id: str) -> Optional[dict]:
+    connections = await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
+    return next((connection for connection in connections if connection.get('id') == connection_id), None)
+
+
+def _get_path(data: Any, path: Optional[str], default=None):
+    if not path:
+        return default
+    value = data
+    for part in path.split('.'):
+        if isinstance(value, dict):
+            value = value.get(part, default)
+        else:
+            return default
+    return value
+
+
+def _normalize_result(result: dict, mapping: dict, knowledge: KnowledgeModel, distance: Optional[float] = None) -> dict:
+    content = _get_path(result, mapping.get('content_field', 'content'), '')
+    title = _get_path(result, mapping.get('title_field', 'title'), None)
+    source = _get_path(result, mapping.get('source_field', 'source'), None)
+    url = _get_path(result, mapping.get('url_field', 'url'), None)
+    document_id = _get_path(result, mapping.get('document_id_field', 'document_id'), None)
+    page = _get_path(result, mapping.get('page_field', 'page'), None)
+    metadata = _get_path(result, mapping.get('metadata_field', 'metadata'), {}) or {}
+    score = _get_path(result, mapping.get('score_field', 'score'), distance)
+
+    if not isinstance(metadata, dict):
+        metadata = {'external_metadata': metadata}
+
+    source_name = source or title or metadata.get('source') or metadata.get('name') or knowledge.name
+    metadata.update(
+        {
+            'name': title or source_name,
+            'source': source_name,
+            'url': url,
+            'file_id': document_id or f'external-{knowledge.id}',
+            'knowledge_id': knowledge.id,
+            'knowledge_name': knowledge.name,
+            'external': True,
+        }
+    )
+    if page is not None:
+        metadata['page'] = page
+    if document_id is not None:
+        metadata['document_id'] = document_id
+
+    return {
+        'content': content,
+        'metadata': metadata,
+        'distance': score,
+    }
+
+
+def _source_config(knowledge: KnowledgeModel) -> dict:
+    external = (knowledge.meta or {}).get('external', {})
+    source = external.get('source') or {}
+    return source.get('config') or {}
+
+
+def _root_field(path: Optional[str]) -> Optional[str]:
+    if not path:
+        return None
+    return path.split('.')[0]
+
+
+def _safe_identifier(value: str, label: str) -> str:
+    if not value or not IDENTIFIER_RE.match(value):
+        raise RuntimeError(f'Invalid {label}')
+    return value
+
+
+async def _retrieve_qdrant(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
+    try:
+        from qdrant_client import QdrantClient
+    except ImportError as exc:
+        raise RuntimeError('qdrant-client is not installed') from exc
+
+    if not embedding_function:
+        raise RuntimeError('Embedding function is not configured')
+
+    config = connection.get('config') or {}
+    external = (knowledge.meta or {}).get('external', {})
+    source = external.get('source') or {}
+    collection_name = source.get('name')
+    if not collection_name:
+        raise RuntimeError('External source collection is not configured')
+    source_config = _source_config(knowledge)
+    vector_field = source_config.get('vector_field') or None
+
+    vector = await embedding_function(query)
+
+    def _search():
+        client = QdrantClient(
+            url=connection.get('endpoint'),
+            api_key=(auth_config or {}).get('api_key'),
+            timeout=config.get('timeout') or 30,
+        )
+        return client.query_points(
+            collection_name=collection_name,
+            query=vector,
+            using=vector_field,
+            limit=count,
+        )
+
+    response = await asyncio.to_thread(_search)
+    mapping = {
+        'content_field': source_config.get('content_field') or 'payload.text',
+        'metadata_field': source_config.get('metadata_field') or 'payload.metadata',
+        'document_id_field': source_config.get('document_id_field') or 'id',
+        'score_field': 'score',
+    }
+
+    normalized = []
+    for point in response.points:
+        normalized.append(_normalize_result(point.model_dump(), mapping, knowledge, distance=point.score))
+    return normalized
+
+
+async def _retrieve_milvus(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
+    try:
+        from pymilvus import MilvusClient
+    except ImportError as exc:
+        raise RuntimeError('pymilvus is not installed') from exc
+
+    if not embedding_function:
+        raise RuntimeError('Embedding function is not configured')
+
+    config = connection.get('config') or {}
+    external = (knowledge.meta or {}).get('external', {})
+    source = external.get('source') or {}
+    collection_name = source.get('name')
+    if not collection_name:
+        raise RuntimeError('Milvus collection is not configured')
+    source_config = _source_config(knowledge)
+    vector_field = source_config.get('vector_field') or 'vector'
+    content_field = source_config.get('content_field') or 'data.text'
+    metadata_field = source_config.get('metadata_field') or 'metadata'
+
+    vector = await embedding_function(query)
+
+    def _search():
+        client_kwargs = {
+            'uri': connection.get('endpoint'),
+        }
+        token = (auth_config or {}).get('api_key') or (auth_config or {}).get('token')
+        if token:
+            client_kwargs['token'] = token
+        if config.get('db_name'):
+            client_kwargs['db_name'] = config.get('db_name')
+
+        client = MilvusClient(**client_kwargs)
+        output_fields = {
+            field
+            for field in (
+                _root_field(content_field),
+                _root_field(metadata_field),
+                _root_field(source_config.get('document_id_field')),
+            )
+            if field and field != vector_field
+        }
+        kwargs = {
+            'collection_name': collection_name,
+            'data': [vector],
+            'anns_field': vector_field,
+            'limit': count,
+            'output_fields': list(output_fields),
+        }
+        return client.search(**kwargs)
+
+    response = await asyncio.to_thread(_search)
+    mapping = {
+        'content_field': content_field,
+        'metadata_field': metadata_field,
+        'document_id_field': source_config.get('document_id_field') or 'id',
+        'score_field': 'distance',
+    }
+
+    normalized = []
+    for hit in (response[0] if response else []):
+        item = dict(hit)
+        entity = item.get('entity') or {}
+        result = {
+            **entity,
+            'id': item.get('id') or entity.get('id'),
+            'distance': item.get('distance'),
+        }
+        normalized.append(_normalize_result(result, mapping, knowledge, distance=item.get('distance')))
+    return normalized
+
+
+async def _retrieve_pgvector(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
+    try:
+        import psycopg
+        from pgvector.psycopg import register_vector
+        from psycopg.rows import dict_row
+    except ImportError as exc:
+        raise RuntimeError('psycopg and pgvector are required for pgvector retrieval') from exc
+
+    if not embedding_function:
+        raise RuntimeError('Embedding function is not configured')
+
+    config = connection.get('config') or {}
+    external = (knowledge.meta or {}).get('external', {})
+    source = external.get('source') or {}
+    collection_name = source.get('name')
+    if not collection_name:
+        raise RuntimeError('pgvector collection is not configured')
+    source_config = _source_config(knowledge)
+    table_name = source_config.get('table_name') or 'document_chunk'
+    collection_field = source_config.get('collection_field') or 'collection_name'
+    content_field = source_config.get('content_field') or 'text'
+    vector_field = source_config.get('vector_field') or 'vector'
+    metadata_field = source_config.get('metadata_field') or 'vmetadata'
+    document_id_field = source_config.get('document_id_field') or 'id'
+
+    vector = await embedding_function(query)
+
+    def _search():
+        from psycopg import sql
+
+        table_identifier = sql.SQL('.').join(
+            sql.Identifier(_safe_identifier(part, 'table name')) for part in table_name.split('.')
+        )
+        collection_identifier = sql.Identifier(_safe_identifier(collection_field, 'collection field'))
+        content_identifier = sql.Identifier(_safe_identifier(content_field, 'content field'))
+        vector_identifier = sql.Identifier(_safe_identifier(vector_field, 'vector field'))
+        document_id_identifier = sql.Identifier(_safe_identifier(document_id_field, 'document id field'))
+        metadata_sql = (
+            sql.Identifier(_safe_identifier(metadata_field, 'metadata field'))
+            if metadata_field
+            else sql.SQL("'{}'::jsonb")
+        )
+
+        with psycopg.connect(
+            connection.get('endpoint'),
+            row_factory=dict_row,
+            connect_timeout=config.get('timeout') or 30,
+        ) as conn:
+            register_vector(conn)
+            with conn.cursor() as cur:
+                cur.execute(
+                    sql.SQL(
+                        """
+                    SELECT {document_id} AS id,
+                           {content} AS content,
+                           {metadata} AS metadata,
+                           {vector_column} <=> %s AS distance
+                    FROM {table_name}
+                    WHERE {collection} = %s
+                    ORDER BY distance ASC
+                    LIMIT %s
+                    """
+                    ).format(
+                        document_id=document_id_identifier,
+                        content=content_identifier,
+                        metadata=metadata_sql,
+                        vector_column=vector_identifier,
+                        table_name=table_identifier,
+                        collection=collection_identifier,
+                    ),
+                    (vector, collection_name, count),
+                )
+                return cur.fetchall()
+
+    rows = await asyncio.to_thread(_search)
+    mapping = {
+        'content_field': 'content',
+        'metadata_field': 'metadata',
+        'document_id_field': 'id',
+        'score_field': 'distance',
+    }
+    return [_normalize_result(row, mapping, knowledge, distance=row.get('distance')) for row in rows]
+
+
+async def retrieve_external_knowledge(
+    request,
+    knowledge: KnowledgeModel,
+    queries: list[str],
+    count: int,
+    user=None,
+) -> dict:
+    external = (knowledge.meta or {}).get('external', {})
+    connection_id = external.get('connection_id')
+    if not connection_id:
+        raise RuntimeError('External knowledge connection is not configured')
+
+    connection = await _get_external_connection(connection_id)
+    if not connection:
+        raise RuntimeError('External knowledge connection not found')
+
+    return await retrieve_external_knowledge_for_connection(request, knowledge, connection, queries, count, user=user)
+
+
+async def retrieve_external_knowledge_for_connection(
+    request,
+    knowledge: KnowledgeModel,
+    connection: dict,
+    queries: list[str],
+    count: int,
+    user=None,
+) -> dict:
+    auth_config = connection.get('auth_config') or {}
+    if not connection.get('enabled', True):
+        raise RuntimeError('External knowledge connection is disabled')
+
+    started_at = time.monotonic()
+    chunks = []
+    provider = (connection.get('provider') or '').lower()
+
+    for query in queries:
+        if provider == 'qdrant':
+            chunks.extend(
+                await _retrieve_qdrant(
+                    connection,
+                    auth_config,
+                    knowledge,
+                    query,
+                    count,
+                    getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
+                )
+            )
+        elif provider == 'milvus':
+            chunks.extend(
+                await _retrieve_milvus(
+                    connection,
+                    auth_config,
+                    knowledge,
+                    query,
+                    count,
+                    getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
+                )
+            )
+        elif provider == 'pgvector':
+            chunks.extend(
+                await _retrieve_pgvector(
+                    connection,
+                    auth_config,
+                    knowledge,
+                    query,
+                    count,
+                    getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
+                )
+            )
+        else:
+            raise RuntimeError(f'Unsupported external knowledge provider: {connection.get("provider")}')
+
+    chunks = chunks[:count]
+    log.info(
+        'external_knowledge_retrieval knowledge_id=%s connection_id=%s provider=%s user_id=%s latency_ms=%s result_count=%s',
+        knowledge.id,
+        connection.get('id'),
+        connection.get('provider'),
+        getattr(user, 'id', None),
+        round((time.monotonic() - started_at) * 1000),
+        len(chunks),
+    )
+
+    return {
+        'documents': [[chunk['content'] for chunk in chunks]],
+        'metadatas': [[chunk['metadata'] for chunk in chunks]],
+        'distances': [[chunk['distance'] for chunk in chunks]],
+    }

backend/open_webui/retrieval/utils.py

@@ -43,6 +43,7 @@ from open_webui.models.config import Config
 from open_webui.models.users import UserModel
 from open_webui.retrieval.loaders.youtube import YoutubeLoader
 from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
+from open_webui.retrieval.external import retrieve_external_knowledge
 from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
 from open_webui.retrieval.vector.main import GetResult, SearchResult
 from open_webui.retrieval.web.utils import get_web_loader
@@ -84,6 +85,7 @@ LOADER_CONFIG_KEYS = {
     'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
     'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
     'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
+    'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
     'TIKA_SERVER_URL': 'rag.tika_server_url',
     'DOCLING_SERVER_URL': 'rag.docling_server_url',
     'DOCLING_API_KEY': 'rag.docling_api_key',
@@ -130,18 +132,16 @@ def build_loader_from_config(request, config: dict):
     """Build a Loader instance with the admin's configured extraction engine settings."""
     from open_webui.retrieval.loaders.main import Loader
 
-    loader_config = {
-        key: config.get(key)
-        for key in LOADER_CONFIG_KEYS
-        if key.isupper()
-    }
+    loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
     return Loader(
         engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
         **{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
     )
 
 
-def _extract_text_from_binary_response(request, response: requests.Response, url: str, loader_config: dict) -> tuple[str, list]:
+def _extract_text_from_binary_response(
+    request, response: requests.Response, url: str, loader_config: dict
+) -> tuple[str, list]:
     """Download response body to a temp file and extract text using the Loader pipeline."""
     import mimetypes
     import tempfile
@@ -774,11 +774,7 @@ async def query_collection_with_hybrid_search(
             return result
 
         native_task_results = await asyncio.gather(
-            *[
-                process_native_query(collection_name, query)
-                for collection_name in collection_names
-                for query in queries
-            ]
+            *[process_native_query(collection_name, query) for collection_name in collection_names for query in queries]
         )
         if native_task_results and all(result is not None for result in native_task_results):
             return merge_and_sort_query_results(native_task_results, k=k)
@@ -1485,50 +1481,61 @@ async def get_sources_from_items(
                     permission='read',
                 )
             ):
-                if item.get('context') == 'full' or bypass_embedding_and_retrieval:
-                    if knowledge_base and (
-                        user.role == 'admin'
-                        or knowledge_base.user_id == user.id
-                        or await AccessGrants.has_access(
-                            user_id=user.id,
-                            resource_type='knowledge',
-                            resource_id=knowledge_base.id,
-                            permission='read',
-                        )
-                    ):
-                        files = await Knowledges.get_files_by_id(knowledge_base.id)
+                if (knowledge_base.meta or {}).get('source') == 'external':
+                    query_result = await retrieve_external_knowledge(
+                        request,
+                        knowledge_base,
+                        queries=queries,
+                        count=k,
+                        user=user,
+                    )
+                    extracted_collections.append(knowledge_base.id)
 
-                        documents = []
-                        metadatas = []
-                        for file in files:
-                            documents.append(file.data.get('content', ''))
-                            metadatas.append(
-                                {
-                                    'file_id': file.id,
-                                    'name': file.filename,
-                                    'source': file.filename,
-                                }
-                            )
-
-                        query_result = {
-                            'documents': [documents],
-                            'metadatas': [metadatas],
-                        }
                 else:
-                    if item.get('legacy'):
-                        if BYPASS_RETRIEVAL_ACCESS_CONTROL:
-                            collection_names = item.get('collection_names', [])
-                        else:
-                            # Legacy KB: item.collection_names is client-supplied.
-                            # Validate against the KB's actual files to prevent
-                            # cross-tenant collection name substitution.
+                    if item.get('context') == 'full' or bypass_embedding_and_retrieval:
+                        if knowledge_base and (
+                            user.role == 'admin'
+                            or knowledge_base.user_id == user.id
+                            or await AccessGrants.has_access(
+                                user_id=user.id,
+                                resource_type='knowledge',
+                                resource_id=knowledge_base.id,
+                                permission='read',
+                            )
+                        ):
                             files = await Knowledges.get_files_by_id(knowledge_base.id)
-                            owned_names = {f'file-{f.id}' for f in files}
-                            owned_names.add(knowledge_base.id)
-                            valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
-                            collection_names = valid_names if valid_names else [knowledge_base.id]
+
+                            documents = []
+                            metadatas = []
+                            for file in files:
+                                documents.append(file.data.get('content', ''))
+                                metadatas.append(
+                                    {
+                                        'file_id': file.id,
+                                        'name': file.filename,
+                                        'source': file.filename,
+                                    }
+                                )
+
+                            query_result = {
+                                'documents': [documents],
+                                'metadatas': [metadatas],
+                            }
                     else:
-                        collection_names.append(item['id'])
+                        if item.get('legacy'):
+                            if BYPASS_RETRIEVAL_ACCESS_CONTROL:
+                                collection_names = item.get('collection_names', [])
+                            else:
+                                # Legacy KB: item.collection_names is client-supplied.
+                                # Validate against the KB's actual files to prevent
+                                # cross-tenant collection name substitution.
+                                files = await Knowledges.get_files_by_id(knowledge_base.id)
+                                owned_names = {f'file-{f.id}' for f in files}
+                                owned_names.add(knowledge_base.id)
+                                valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
+                                collection_names = valid_names if valid_names else [knowledge_base.id]
+                        else:
+                            collection_names.append(item['id'])
 
         elif item.get('docs'):
             # BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL

backend/open_webui/routers/knowledge.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import asyncio
 import io
 import logging
+import time
+import uuid
 import zipfile
 from typing import List, Optional
 from urllib.parse import quote
@@ -27,6 +29,7 @@ from open_webui.models.knowledge import (
 )
 from open_webui.models.models import ModelForm, Models
 from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
+from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection
 from open_webui.routers.retrieval import (
     BatchProcessFilesForm,
     ProcessFileForm,
@@ -110,6 +113,17 @@ class KnowledgeAccessListResponse(BaseModel):
     total: int
 
 
+def is_external_knowledge(knowledge) -> bool:
+    return (knowledge.meta or {}).get('source') == 'external'
+
+
+def external_knowledge_error():
+    raise HTTPException(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        detail='External knowledge bases are read-only.',
+    )
+
+
 @router.get('/', response_model=KnowledgeAccessListResponse)
 async def get_knowledge_bases(
     page: int | None = 1,
@@ -163,6 +177,7 @@ async def get_knowledge_bases(
 async def search_knowledge_bases(
     query: str | None = None,
     view_option: str | None = None,
+    source: str | None = None,
     page: int | None = 1,
     user=Depends(get_verified_user),
     db: AsyncSession = Depends(get_async_session),
@@ -176,6 +191,8 @@ async def search_knowledge_bases(
         filter['query'] = query
     if view_option:
         filter['view_option'] = view_option
+    if source in {'local', 'external'}:
+        filter['source'] = source
 
     groups = await Groups.get_groups_by_member_id(user.id, db=db)
     user_group_ids = {group.id for group in groups}
@@ -379,6 +396,574 @@ async def reindex_knowledge_base_metadata_embeddings(
     return {'total': len(knowledge_bases), 'success': success_count}
 
 
+############################
+# External Knowledge Sources
+############################
+
+
+class ExternalKnowledgeSourceForm(BaseModel):
+    type: str = 'collection'
+    name: str
+    config: Optional[dict] = None
+
+
+class ExternalKnowledgeCreateForm(BaseModel):
+    name: str
+    description: str = ''
+    connection_id: str
+    source: ExternalKnowledgeSourceForm
+    access_grants: Optional[list[dict]] = None
+
+
+class ExternalKnowledgeSourceCreateForm(BaseModel):
+    name: str
+    description: str = ''
+    connection: ExternalKnowledgeConnectionForm
+    source: ExternalKnowledgeSourceForm
+    access_grants: Optional[list[dict]] = None
+    test_query: str
+    test_count: int = 5
+
+
+class ExternalKnowledgeSourceUpdateForm(ExternalKnowledgeSourceCreateForm):
+    pass
+
+
+class ExternalKnowledgeSourceTestForm(BaseModel):
+    connection_id: Optional[str] = None
+    connection: ExternalKnowledgeConnectionForm
+    source: ExternalKnowledgeSourceForm
+    query: str
+    count: int = 5
+
+
+class ExternalKnowledgeRetrieveTestForm(BaseModel):
+    query: str
+    source: Optional[ExternalKnowledgeSourceForm] = None
+    count: int = 5
+
+
+class ExternalKnowledgeConnectionForm(BaseModel):
+    name: str
+    provider: str
+    endpoint: str
+    auth_config: Optional[dict] = None
+    config: Optional[dict] = None
+    capabilities: Optional[dict] = None
+    enabled: bool = True
+
+
+class ExternalKnowledgeConnectionListResponse(BaseModel):
+    items: list[dict]
+    total: int
+
+
+EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
+EXTERNAL_KNOWLEDGE_PROVIDERS = {'qdrant', 'milvus', 'pgvector'}
+
+
+def _validate_external_connection_form(form_data: ExternalKnowledgeConnectionForm) -> tuple[str, dict]:
+    provider = form_data.provider.lower().strip()
+    if provider not in EXTERNAL_KNOWLEDGE_PROVIDERS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail='Unsupported external knowledge provider.',
+        )
+
+    if not form_data.name.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source name is required.')
+
+    if not form_data.endpoint.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source endpoint is required.')
+
+    config = form_data.config or {}
+    allowed_config_keys = {'timeout'}
+    if provider == 'milvus':
+        allowed_config_keys.add('db_name')
+
+    return provider, {key: value for key, value in config.items() if key in allowed_config_keys}
+
+
+def _external_auth_config(provider: str, incoming: Optional[dict], existing: Optional[dict] = None) -> dict:
+    if provider == 'pgvector':
+        return {}
+    return existing if incoming is None else incoming or {}
+
+
+def _normalize_external_source(source: ExternalKnowledgeSourceForm, provider: str) -> ExternalKnowledgeSourceForm:
+    source.type = (source.type or 'collection').strip()
+    source.name = source.name.strip()
+
+    if source.type != 'collection':
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Only collection sources are supported.')
+    if not source.name:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Collection name is required.')
+
+    config = source.config or {}
+    allowed_keys = {'content_field', 'metadata_field', 'document_id_field'}
+    if provider in {'qdrant', 'milvus'}:
+        allowed_keys.add('vector_field')
+    if provider == 'pgvector':
+        allowed_keys.update({'table_name', 'collection_field', 'vector_field'})
+
+    normalized_config = {
+        key: value.strip() if isinstance(value, str) else value
+        for key, value in config.items()
+        if key in allowed_keys and value is not None and (not isinstance(value, str) or value.strip())
+    }
+
+    if not normalized_config.get('content_field'):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Content field is required.')
+    if provider in {'milvus', 'pgvector'} and not normalized_config.get('vector_field'):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Vector field is required.')
+
+    source.config = normalized_config
+    return source
+
+
+def _sanitize_external_connection(connection: dict) -> dict:
+    sanitized = {**connection}
+    sanitized.pop('auth_config', None)
+    sanitized['auth_configured'] = bool(connection.get('auth_config'))
+    return sanitized
+
+
+async def _get_external_connections() -> list[dict]:
+    return await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
+
+
+async def _set_external_connections(connections: list[dict]) -> None:
+    await Config.upsert({EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY: connections})
+
+
+def _external_connection_dict(form_data: ExternalKnowledgeConnectionForm, user_id: str, id: Optional[str] = None) -> dict:
+    provider, config = _validate_external_connection_form(form_data)
+    now = int(time.time())
+    return {
+        'id': id or str(uuid.uuid4()),
+        'name': form_data.name.strip(),
+        'provider': provider,
+        'endpoint': form_data.endpoint.strip(),
+        'auth_config': _external_auth_config(provider, form_data.auth_config),
+        'config': config,
+        'capabilities': form_data.capabilities or {'retrieve': True},
+        'health': None,
+        'enabled': form_data.enabled,
+        'created_by': user_id,
+        'created_at': now,
+        'updated_at': now,
+    }
+
+
+def _external_connection_update_dict(
+    form_data: ExternalKnowledgeConnectionForm,
+    existing: dict,
+) -> dict:
+    provider, config = _validate_external_connection_form(form_data)
+    return {
+        **existing,
+        'name': form_data.name.strip(),
+        'provider': provider,
+        'endpoint': form_data.endpoint.strip(),
+        'auth_config': _external_auth_config(provider, form_data.auth_config, existing.get('auth_config')) or {},
+        'config': config,
+        'capabilities': form_data.capabilities or {'retrieve': True},
+        'enabled': form_data.enabled,
+        'updated_at': int(time.time()),
+    }
+
+
+async def _get_external_connection(id: str) -> Optional[dict]:
+    connections = await _get_external_connections()
+    return next((connection for connection in connections if connection.get('id') == id), None)
+
+
+async def _count_external_connection_mappings(connection_id: str, db: Optional[AsyncSession] = None) -> int:
+    count = 0
+    for knowledge in await Knowledges.get_knowledge_bases(db=db):
+        if (knowledge.meta or {}).get('external', {}).get('connection_id') == connection_id:
+            count += 1
+    return count
+
+
+@router.get('/external/connections', response_model=ExternalKnowledgeConnectionListResponse)
+async def get_external_knowledge_connections(
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connections = [_sanitize_external_connection(connection) for connection in await _get_external_connections()]
+    return ExternalKnowledgeConnectionListResponse(items=connections, total=len(connections))
+
+
+@router.post('/external/connections', response_model=dict)
+async def create_external_knowledge_connection(
+    form_data: ExternalKnowledgeConnectionForm,
+    user=Depends(get_admin_user),
+):
+    connections = await _get_external_connections()
+    connection = _external_connection_dict(form_data, user.id)
+    connections.append(connection)
+    await _set_external_connections(connections)
+    return _sanitize_external_connection(connection)
+
+
+@router.get('/external/connections/{id}', response_model=dict)
+async def get_external_knowledge_connection(
+    id: str,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connection = await _get_external_connection(id)
+    if not connection:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+    return _sanitize_external_connection(connection)
+
+
+@router.patch('/external/connections/{id}', response_model=dict)
+async def update_external_knowledge_connection(
+    id: str,
+    form_data: ExternalKnowledgeConnectionForm,
+    user=Depends(get_admin_user),
+):
+    connections = await _get_external_connections()
+    idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == id), None)
+    if idx is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+
+    connection = _external_connection_update_dict(form_data, connections[idx])
+    connections[idx] = connection
+    await _set_external_connections(connections)
+    return _sanitize_external_connection(connection)
+
+
+@router.delete('/external/connections/{id}', response_model=bool)
+async def delete_external_knowledge_connection(
+    id: str,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connection = await _get_external_connection(id)
+    if not connection:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+
+    if await _count_external_connection_mappings(id, db=db) > 0:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail='External connection is still used by knowledge bases.',
+        )
+
+    connections = [connection for connection in await _get_external_connections() if connection.get('id') != id]
+    await _set_external_connections(connections)
+    return True
+
+
+@router.post('/external/connections/{id}/test', response_model=dict)
+async def test_external_knowledge_connection(
+    id: str,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connection = await _get_external_connection(id)
+    if not connection:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+
+    health = {
+        'ok': bool(connection.get('enabled') and connection.get('endpoint')),
+        'provider': connection.get('provider'),
+        'checked_at': int(time.time()),
+    }
+    connections = await _get_external_connections()
+    for item in connections:
+        if item.get('id') == id:
+            item['health'] = health
+            item['updated_at'] = int(time.time())
+            break
+    await _set_external_connections(connections)
+    return health
+
+
+async def _test_external_source_definition(
+    request: Request,
+    connection: dict,
+    source: ExternalKnowledgeSourceForm,
+    query: str,
+    count: int,
+    user,
+) -> dict:
+    if not query.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query is required.')
+
+    source = _normalize_external_source(source, connection.get('provider'))
+    test_knowledge = KnowledgeResponse(
+        id='external-test',
+        user_id=user.id,
+        name=connection.get('name'),
+        description='',
+        meta={
+            'source': 'external',
+            'read_only': True,
+            'external': {
+                'connection_id': connection.get('id'),
+                'source': source.model_dump(),
+                'provider': connection.get('provider'),
+                'auth_mode': 'service_account',
+                'capabilities': {'retrieve': True},
+            },
+        },
+        access_grants=[],
+        created_at=int(time.time()),
+        updated_at=int(time.time()),
+    )
+    result = await retrieve_external_knowledge_for_connection(
+        request,
+        test_knowledge,
+        connection,
+        [query.strip()],
+        count,
+        user=user,
+    )
+    return {
+        'documents': result.get('documents', [[]])[0],
+        'metadatas': result.get('metadatas', [[]])[0],
+        'distances': result.get('distances', [[]])[0],
+    }
+
+
+@router.post('/external/source/test', response_model=dict)
+async def test_external_knowledge_source(
+    request: Request,
+    form_data: ExternalKnowledgeSourceTestForm,
+    user=Depends(get_admin_user),
+):
+    if form_data.connection_id:
+        existing_connection = await _get_external_connection(form_data.connection_id)
+        if not existing_connection:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
+        connection = _external_connection_update_dict(form_data.connection, existing_connection)
+    else:
+        connection = _external_connection_dict(form_data.connection, user.id, id='external-test')
+
+    return await _test_external_source_definition(
+        request,
+        connection,
+        form_data.source,
+        form_data.query,
+        form_data.count,
+        user,
+    )
+
+
+@router.post('/external/connections/{id}/retrieve-test', response_model=dict)
+async def test_external_knowledge_retrieval(
+    request: Request,
+    id: str,
+    form_data: ExternalKnowledgeRetrieveTestForm,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connection = await _get_external_connection(id)
+    if not connection:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+
+    source = form_data.source or ExternalKnowledgeSourceForm(name='test', config={'content_field': 'payload.text'})
+    return await _test_external_source_definition(request, connection, source, form_data.query, form_data.count, user)
+
+
+@router.post('/external/knowledge/create', response_model=KnowledgeResponse | None)
+async def create_external_knowledge(
+    request: Request,
+    form_data: ExternalKnowledgeCreateForm,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    connection = await _get_external_connection(form_data.connection_id)
+    if not connection:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+    if not form_data.name.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
+    source = _normalize_external_source(form_data.source, connection.get('provider'))
+
+    form_data.access_grants = await filter_allowed_access_grants(
+        await Config.get('user.permissions'),
+        user.id,
+        user.role,
+        form_data.access_grants,
+        'sharing.public_knowledge',
+    )
+
+    knowledge = await Knowledges.insert_new_knowledge(
+        user.id,
+        KnowledgeForm(
+            name=form_data.name.strip(),
+            description=form_data.description,
+            access_grants=form_data.access_grants,
+        ),
+        db=db,
+    )
+    if not knowledge:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
+
+    meta = {
+        'source': 'external',
+        'read_only': True,
+        'external': {
+            'connection_id': form_data.connection_id,
+            'source': source.model_dump(),
+            'provider': connection.get('provider'),
+            'auth_mode': 'service_account',
+            'capabilities': {'retrieve': True},
+        },
+    }
+    knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
+    await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
+    return knowledge
+
+
+@router.post('/external/source/create', response_model=KnowledgeResponse | None)
+async def create_external_knowledge_source(
+    request: Request,
+    form_data: ExternalKnowledgeSourceCreateForm,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    if not form_data.name.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
+
+    connection = _external_connection_dict(form_data.connection, user.id)
+    source = _normalize_external_source(form_data.source, connection.get('provider'))
+    test_result = await _test_external_source_definition(
+        request,
+        connection,
+        source,
+        form_data.test_query,
+        form_data.test_count,
+        user,
+    )
+    if not test_result.get('documents'):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
+
+    form_data.access_grants = await filter_allowed_access_grants(
+        await Config.get('user.permissions'),
+        user.id,
+        user.role,
+        form_data.access_grants,
+        'sharing.public_knowledge',
+    )
+
+    connections = await _get_external_connections()
+    connections.append(connection)
+    await _set_external_connections(connections)
+
+    knowledge = await Knowledges.insert_new_knowledge(
+        user.id,
+        KnowledgeForm(
+            name=form_data.name.strip(),
+            description=form_data.description,
+            access_grants=form_data.access_grants,
+        ),
+        db=db,
+    )
+    if not knowledge:
+        connections = [item for item in await _get_external_connections() if item.get('id') != connection.get('id')]
+        await _set_external_connections(connections)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
+
+    meta = {
+        'source': 'external',
+        'read_only': True,
+        'external': {
+            'connection_id': connection.get('id'),
+            'source': source.model_dump(),
+            'provider': connection.get('provider'),
+            'auth_mode': 'service_account',
+            'capabilities': {'retrieve': True},
+        },
+    }
+    knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
+    await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
+    return knowledge
+
+
+@router.patch('/external/source/{id}', response_model=KnowledgeResponse | None)
+async def update_external_knowledge_source(
+    request: Request,
+    id: str,
+    form_data: ExternalKnowledgeSourceUpdateForm,
+    user=Depends(get_admin_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
+    if not knowledge or not is_external_knowledge(knowledge):
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
+    if not form_data.name.strip():
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
+
+    connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
+    connections = await _get_external_connections()
+    idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == connection_id), None)
+    if idx is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
+
+    existing_connection = connections[idx]
+    connection = _external_connection_update_dict(form_data.connection, existing_connection)
+    source = _normalize_external_source(form_data.source, connection.get('provider'))
+    test_result = await _test_external_source_definition(
+        request,
+        connection,
+        source,
+        form_data.test_query,
+        form_data.test_count,
+        user,
+    )
+    if not test_result.get('documents'):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
+
+    form_data.access_grants = await filter_allowed_access_grants(
+        await Config.get('user.permissions'),
+        user.id,
+        user.role,
+        form_data.access_grants,
+        'sharing.public_knowledge',
+    )
+
+    connections[idx] = connection
+    await _set_external_connections(connections)
+
+    updated = await Knowledges.update_knowledge_by_id(
+        id=id,
+        form_data=KnowledgeForm(
+            name=form_data.name.strip(),
+            description=form_data.description,
+            access_grants=form_data.access_grants,
+        ),
+        db=db,
+    )
+    if not updated:
+        connections[idx] = existing_connection
+        await _set_external_connections(connections)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
+
+    meta = {
+        'source': 'external',
+        'read_only': True,
+        'external': {
+            'connection_id': connection.get('id'),
+            'source': source.model_dump(),
+            'provider': connection.get('provider'),
+            'auth_mode': 'service_account',
+            'capabilities': {'retrieve': True},
+        },
+    }
+    updated = await Knowledges.update_knowledge_meta_by_id(id, meta, db=db)
+    if not updated:
+        connections[idx] = existing_connection
+        await _set_external_connections(connections)
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
+
+    await embed_knowledge_base_metadata(request, id, updated.name, updated.description)
+    return updated
+
+
 ############################
 # GetKnowledgeById
 ############################
@@ -711,6 +1296,8 @@ async def add_file_to_knowledge_by_id(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     if (
         knowledge.user_id != user.id
@@ -798,6 +1385,8 @@ async def update_file_from_knowledge_by_id(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     if (
         knowledge.user_id != user.id
@@ -877,6 +1466,8 @@ async def remove_file_from_knowledge_by_id(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     if (
         knowledge.user_id != user.id
@@ -1004,11 +1595,21 @@ async def delete_knowledge_by_id(
                 await Models.update_model_by_id(model.id, model_form, db=db)
 
     # Clean up vector DB
-    try:
-        await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
-    except Exception as e:
-        log.debug(e)
-        pass
+    if is_external_knowledge(knowledge):
+        connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
+        if connection_id:
+            connections = [
+                connection
+                for connection in await _get_external_connections()
+                if connection.get('id') != connection_id
+            ]
+            await _set_external_connections(connections)
+    else:
+        try:
+            await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
+        except Exception as e:
+            log.debug(e)
+            pass
 
     # Remove knowledge base embedding
     await remove_knowledge_base_metadata_embedding(id)
@@ -1035,6 +1636,8 @@ async def reset_knowledge_by_id(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     if (
         knowledge.user_id != user.id
@@ -1263,6 +1866,8 @@ async def add_files_to_knowledge_batch(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     if (
         knowledge.user_id != user.id
@@ -1380,6 +1985,8 @@ async def export_knowledge_by_id(id: str, user=Depends(get_admin_user), db: Asyn
             status_code=status.HTTP_404_NOT_FOUND,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
 
     files = await Knowledges.get_files_by_id(id, db=db)
 
@@ -1441,6 +2048,8 @@ async def _verify_knowledge_write_access(id: str, user, db: AsyncSession):
             status_code=status.HTTP_404_NOT_FOUND,
             detail=ERROR_MESSAGES.NOT_FOUND,
         )
+    if is_external_knowledge(knowledge):
+        external_knowledge_error()
     if (
         knowledge.user_id != user.id
         and not await AccessGrants.has_access(

backend/open_webui/routers/retrieval.py

@@ -288,6 +288,7 @@ RETRIEVAL_CONFIG_KEYS = {
     'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
     'EXA_API_KEY': 'rag.web.search.exa_api_key',
     'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
+    'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
     'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
     'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
     'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
@@ -632,6 +633,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
         'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
         'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
         'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
+        'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
         'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
         'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
         'DOCLING_API_KEY': config.DOCLING_API_KEY,
@@ -846,6 +848,7 @@ class ConfigForm(BaseModel):
 
     EXTERNAL_DOCUMENT_LOADER_URL: str | None = None
     EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None
+    EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
 
     TIKA_SERVER_URL: str | None = None
     DOCLING_SERVER_URL: str | None = None
@@ -1021,6 +1024,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
         if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None
         else config.EXTERNAL_DOCUMENT_LOADER_API_KEY
     )
+    config.EXTERNAL_DOCUMENT_LOADER_HEADERS = (
+        form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS
+        if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None
+        else config.EXTERNAL_DOCUMENT_LOADER_HEADERS
+    )
     config.TIKA_SERVER_URL = (
         form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
     )
@@ -1336,6 +1344,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
         'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
         'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
         'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
+        'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
         'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
         'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
         'DOCLING_API_KEY': config.DOCLING_API_KEY,
@@ -1843,6 +1852,11 @@ async def process_file(
                     loader_config = await get_loader_config()
                     loader = build_loader_from_config(request, loader_config)
                     loader.user = user
+                    loader.metadata = {
+                        'file_id': file.id,
+                        'file_name': file.filename,
+                        'file_content_type': file.meta.get('content_type'),
+                    }
                     docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path)
 
                     docs = [

backend/open_webui/tools/builtin.py

@@ -514,21 +514,14 @@ async def execute_code(
 
         elif engine == 'jupyter':
             from open_webui.utils.code_interpreter import execute_code_jupyter
+
             jupyter_auth = await Config.get('code_interpreter.jupyter.auth')
 
             output = await execute_code_jupyter(
                 await Config.get('code_interpreter.jupyter.url'),
                 code,
-                (
-                    await Config.get('code_interpreter.jupyter.auth_token')
-                    if jupyter_auth == 'token'
-                    else None
-                ),
-                (
-                    await Config.get('code_interpreter.jupyter.auth_password')
-                    if jupyter_auth == 'password'
-                    else None
-                ),
+                (await Config.get('code_interpreter.jupyter.auth_token') if jupyter_auth == 'token' else None),
+                (await Config.get('code_interpreter.jupyter.auth_password') if jupyter_auth == 'password' else None),
                 await Config.get('code_interpreter.jupyter.timeout'),
             )
 
@@ -2385,6 +2378,7 @@ async def query_knowledge_files(
         from open_webui.models.files import Files
         from open_webui.models.knowledge import Knowledges
         from open_webui.models.notes import Notes
+        from open_webui.retrieval.external import retrieve_external_knowledge
         from open_webui.retrieval.utils import query_collection
 
         user_id = __user__.get('id')
@@ -2396,6 +2390,7 @@ async def query_knowledge_files(
             return json.dumps({'error': 'Embedding function not configured'})
 
         collection_names = []
+        external_knowledges = []
         note_results = []  # Notes aren't vectorized, handle separately
 
         # If model has attached knowledge, use those
@@ -2418,7 +2413,10 @@ async def query_knowledge_files(
                             user_group_ids=set(user_group_ids),
                         )
                     ):
-                        collection_names.append(item_id)
+                        if (knowledge.meta or {}).get('source') == 'external':
+                            external_knowledges.append(knowledge)
+                        else:
+                            collection_names.append(item_id)
 
                 elif item_type == 'file':
                     # Individual file - use file-{id} as collection name
@@ -2464,7 +2462,10 @@ async def query_knowledge_files(
                         user_group_ids=set(user_group_ids),
                     )
                 ):
-                    collection_names.append(knowledge_id)
+                    if (knowledge.meta or {}).get('source') == 'external':
+                        external_knowledges.append(knowledge)
+                    else:
+                        collection_names.append(knowledge_id)
         else:
             # No model knowledge and no specific IDs - search all accessible KBs
             result = await Knowledges.search_knowledge_bases(
@@ -2477,7 +2478,11 @@ async def query_knowledge_files(
                 skip=0,
                 limit=50,
             )
-            collection_names = [knowledge_base.id for knowledge_base in result.items]
+            for knowledge_base in result.items:
+                if (knowledge_base.meta or {}).get('source') == 'external':
+                    external_knowledges.append(knowledge_base)
+                else:
+                    collection_names.append(knowledge_base.id)
 
         chunks = []
 
@@ -2509,6 +2514,31 @@ async def query_knowledge_files(
                         chunk_info['distance'] = distances[idx]
                     chunks.append(chunk_info)
 
+        for knowledge in external_knowledges:
+            query_results = await retrieve_external_knowledge(
+                __request__,
+                knowledge,
+                queries=[query],
+                count=count,
+                user=type('UserContext', (), {'id': user_id, 'role': user_role})(),
+            )
+            documents = query_results.get('documents', [[]])[0]
+            metadatas = query_results.get('metadatas', [[]])[0]
+            distances = query_results.get('distances', [[]])[0]
+
+            for idx, doc in enumerate(documents):
+                metadata = metadatas[idx] if idx < len(metadatas) else {}
+                chunk_info = {
+                    'content': doc,
+                    'source': metadata.get('source', metadata.get('name', knowledge.name)),
+                    'file_id': metadata.get('file_id', f'external-{knowledge.id}'),
+                    'type': 'external',
+                    'knowledge_id': knowledge.id,
+                }
+                if idx < len(distances):
+                    chunk_info['distance'] = distances[idx]
+                chunks.append(chunk_info)
+
         # Limit to requested count
         chunks = chunks[:count]