This commit is contained in:
Timothy Jaeryang Baek committed 2026-06-23 23:13:32 +02:00
1 parent b1c2536ed2
commit 15c7e37438
15 files changed
+2929 -394

No files matched your search

+11
View File
@@ -857,6 +857,16 @@ EXTERNAL_DOCUMENT_LOADER_URL = os.getenv('EXTERNAL_DOCUMENT_LOADER_URL', '')
EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '') EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '')
external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '')
try:
external_document_loader_headers = json.loads(external_document_loader_headers)
except json.JSONDecodeError:
external_document_loader_headers = {}
if not isinstance(external_document_loader_headers, dict):
external_document_loader_headers = {}
EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998') TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001') DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
@@ -2652,6 +2662,7 @@ DEFAULT_CONFIG = {
'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS, 'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS,
'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL, 'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL,
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY, 'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
'rag.tika_server_url': TIKA_SERVER_URL, 'rag.tika_server_url': TIKA_SERVER_URL,
'rag.docling_server_url': DOCLING_SERVER_URL, 'rag.docling_server_url': DOCLING_SERVER_URL,
'rag.docling_api_key': DOCLING_API_KEY, 'rag.docling_api_key': DOCLING_API_KEY,
+30
View File
@@ -286,6 +286,17 @@ class KnowledgeTable:
elif view_option == 'shared': elif view_option == 'shared':
stmt = stmt.filter(Knowledge.user_id != user_id) stmt = stmt.filter(Knowledge.user_id != user_id)
source = filter.get('source')
if source == 'external':
stmt = stmt.filter(Knowledge.meta['source'].as_string() == 'external')
elif source == 'local':
stmt = stmt.filter(
or_(
Knowledge.meta.is_(None),
Knowledge.meta['source'].as_string() != 'external',
)
)
stmt = AccessGrants.has_permission_filter( stmt = AccessGrants.has_permission_filter(
db=db, db=db,
query=stmt, query=stmt,
@@ -765,6 +776,25 @@ class KnowledgeTable:
log.exception(e) log.exception(e)
return None return None
async def update_knowledge_meta_by_id(
self, id: str, meta: dict, db: Optional[AsyncSession] = None
) -> Optional[KnowledgeModel]:
try:
async with get_async_db_context(db) as db:
await db.execute(
update(Knowledge)
.filter_by(id=id)
.values(
meta=meta,
updated_at=int(time.time()),
)
)
await db.commit()
return await self.get_knowledge_by_id(id=id, db=db)
except Exception as e:
log.exception(e)
return None
async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool: async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
try: try:
async with get_async_db_context(db) as db: async with get_async_db_context(db) as db:
+378
View File
@@ -0,0 +1,378 @@
import asyncio
import logging
import re
import time
from typing import Any, Optional
from open_webui.models.config import Config
from open_webui.models.knowledge import KnowledgeModel
log = logging.getLogger(__name__)
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
IDENTIFIER_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
async def _get_external_connection(connection_id: str) -> Optional[dict]:
connections = await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
return next((connection for connection in connections if connection.get('id') == connection_id), None)
def _get_path(data: Any, path: Optional[str], default=None):
if not path:
return default
value = data
for part in path.split('.'):
if isinstance(value, dict):
value = value.get(part, default)
else:
return default
return value
def _normalize_result(result: dict, mapping: dict, knowledge: KnowledgeModel, distance: Optional[float] = None) -> dict:
content = _get_path(result, mapping.get('content_field', 'content'), '')
title = _get_path(result, mapping.get('title_field', 'title'), None)
source = _get_path(result, mapping.get('source_field', 'source'), None)
url = _get_path(result, mapping.get('url_field', 'url'), None)
document_id = _get_path(result, mapping.get('document_id_field', 'document_id'), None)
page = _get_path(result, mapping.get('page_field', 'page'), None)
metadata = _get_path(result, mapping.get('metadata_field', 'metadata'), {}) or {}
score = _get_path(result, mapping.get('score_field', 'score'), distance)
if not isinstance(metadata, dict):
metadata = {'external_metadata': metadata}
source_name = source or title or metadata.get('source') or metadata.get('name') or knowledge.name
metadata.update(
{
'name': title or source_name,
'source': source_name,
'url': url,
'file_id': document_id or f'external-{knowledge.id}',
'knowledge_id': knowledge.id,
'knowledge_name': knowledge.name,
'external': True,
}
)
if page is not None:
metadata['page'] = page
if document_id is not None:
metadata['document_id'] = document_id
return {
'content': content,
'metadata': metadata,
'distance': score,
}
def _source_config(knowledge: KnowledgeModel) -> dict:
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
return source.get('config') or {}
def _root_field(path: Optional[str]) -> Optional[str]:
if not path:
return None
return path.split('.')[0]
def _safe_identifier(value: str, label: str) -> str:
if not value or not IDENTIFIER_RE.match(value):
raise RuntimeError(f'Invalid {label}')
return value
async def _retrieve_qdrant(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
from qdrant_client import QdrantClient
except ImportError as exc:
raise RuntimeError('qdrant-client is not installed') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('External source collection is not configured')
source_config = _source_config(knowledge)
vector_field = source_config.get('vector_field') or None
vector = await embedding_function(query)
def _search():
client = QdrantClient(
url=connection.get('endpoint'),
api_key=(auth_config or {}).get('api_key'),
timeout=config.get('timeout') or 30,
)
return client.query_points(
collection_name=collection_name,
query=vector,
using=vector_field,
limit=count,
)
response = await asyncio.to_thread(_search)
mapping = {
'content_field': source_config.get('content_field') or 'payload.text',
'metadata_field': source_config.get('metadata_field') or 'payload.metadata',
'document_id_field': source_config.get('document_id_field') or 'id',
'score_field': 'score',
}
normalized = []
for point in response.points:
normalized.append(_normalize_result(point.model_dump(), mapping, knowledge, distance=point.score))
return normalized
async def _retrieve_milvus(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
from pymilvus import MilvusClient
except ImportError as exc:
raise RuntimeError('pymilvus is not installed') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('Milvus collection is not configured')
source_config = _source_config(knowledge)
vector_field = source_config.get('vector_field') or 'vector'
content_field = source_config.get('content_field') or 'data.text'
metadata_field = source_config.get('metadata_field') or 'metadata'
vector = await embedding_function(query)
def _search():
client_kwargs = {
'uri': connection.get('endpoint'),
}
token = (auth_config or {}).get('api_key') or (auth_config or {}).get('token')
if token:
client_kwargs['token'] = token
if config.get('db_name'):
client_kwargs['db_name'] = config.get('db_name')
client = MilvusClient(**client_kwargs)
output_fields = {
field
for field in (
_root_field(content_field),
_root_field(metadata_field),
_root_field(source_config.get('document_id_field')),
)
if field and field != vector_field
}
kwargs = {
'collection_name': collection_name,
'data': [vector],
'anns_field': vector_field,
'limit': count,
'output_fields': list(output_fields),
}
return client.search(**kwargs)
response = await asyncio.to_thread(_search)
mapping = {
'content_field': content_field,
'metadata_field': metadata_field,
'document_id_field': source_config.get('document_id_field') or 'id',
'score_field': 'distance',
}
normalized = []
for hit in (response[0] if response else []):
item = dict(hit)
entity = item.get('entity') or {}
result = {
**entity,
'id': item.get('id') or entity.get('id'),
'distance': item.get('distance'),
}
normalized.append(_normalize_result(result, mapping, knowledge, distance=item.get('distance')))
return normalized
async def _retrieve_pgvector(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
import psycopg
from pgvector.psycopg import register_vector
from psycopg.rows import dict_row
except ImportError as exc:
raise RuntimeError('psycopg and pgvector are required for pgvector retrieval') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('pgvector collection is not configured')
source_config = _source_config(knowledge)
table_name = source_config.get('table_name') or 'document_chunk'
collection_field = source_config.get('collection_field') or 'collection_name'
content_field = source_config.get('content_field') or 'text'
vector_field = source_config.get('vector_field') or 'vector'
metadata_field = source_config.get('metadata_field') or 'vmetadata'
document_id_field = source_config.get('document_id_field') or 'id'
vector = await embedding_function(query)
def _search():
from psycopg import sql
table_identifier = sql.SQL('.').join(
sql.Identifier(_safe_identifier(part, 'table name')) for part in table_name.split('.')
)
collection_identifier = sql.Identifier(_safe_identifier(collection_field, 'collection field'))
content_identifier = sql.Identifier(_safe_identifier(content_field, 'content field'))
vector_identifier = sql.Identifier(_safe_identifier(vector_field, 'vector field'))
document_id_identifier = sql.Identifier(_safe_identifier(document_id_field, 'document id field'))
metadata_sql = (
sql.Identifier(_safe_identifier(metadata_field, 'metadata field'))
if metadata_field
else sql.SQL("'{}'::jsonb")
)
with psycopg.connect(
connection.get('endpoint'),
row_factory=dict_row,
connect_timeout=config.get('timeout') or 30,
) as conn:
register_vector(conn)
with conn.cursor() as cur:
cur.execute(
sql.SQL(
"""
SELECT {document_id} AS id,
{content} AS content,
{metadata} AS metadata,
{vector_column} <=> %s AS distance
FROM {table_name}
WHERE {collection} = %s
ORDER BY distance ASC
LIMIT %s
"""
).format(
document_id=document_id_identifier,
content=content_identifier,
metadata=metadata_sql,
vector_column=vector_identifier,
table_name=table_identifier,
collection=collection_identifier,
),
(vector, collection_name, count),
)
return cur.fetchall()
rows = await asyncio.to_thread(_search)
mapping = {
'content_field': 'content',
'metadata_field': 'metadata',
'document_id_field': 'id',
'score_field': 'distance',
}
return [_normalize_result(row, mapping, knowledge, distance=row.get('distance')) for row in rows]
async def retrieve_external_knowledge(
request,
knowledge: KnowledgeModel,
queries: list[str],
count: int,
user=None,
) -> dict:
external = (knowledge.meta or {}).get('external', {})
connection_id = external.get('connection_id')
if not connection_id:
raise RuntimeError('External knowledge connection is not configured')
connection = await _get_external_connection(connection_id)
if not connection:
raise RuntimeError('External knowledge connection not found')
return await retrieve_external_knowledge_for_connection(request, knowledge, connection, queries, count, user=user)
async def retrieve_external_knowledge_for_connection(
request,
knowledge: KnowledgeModel,
connection: dict,
queries: list[str],
count: int,
user=None,
) -> dict:
auth_config = connection.get('auth_config') or {}
if not connection.get('enabled', True):
raise RuntimeError('External knowledge connection is disabled')
started_at = time.monotonic()
chunks = []
provider = (connection.get('provider') or '').lower()
for query in queries:
if provider == 'qdrant':
chunks.extend(
await _retrieve_qdrant(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
elif provider == 'milvus':
chunks.extend(
await _retrieve_milvus(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
elif provider == 'pgvector':
chunks.extend(
await _retrieve_pgvector(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
else:
raise RuntimeError(f'Unsupported external knowledge provider: {connection.get("provider")}')
chunks = chunks[:count]
log.info(
'external_knowledge_retrieval knowledge_id=%s connection_id=%s provider=%s user_id=%s latency_ms=%s result_count=%s',
knowledge.id,
connection.get('id'),
connection.get('provider'),
getattr(user, 'id', None),
round((time.monotonic() - started_at) * 1000),
len(chunks),
)
return {
'documents': [[chunk['content'] for chunk in chunks]],
'metadatas': [[chunk['metadata'] for chunk in chunks]],
'distances': [[chunk['distance'] for chunk in chunks]],
}
+58 -51
View File
@@ -43,6 +43,7 @@ from open_webui.models.config import Config
from open_webui.models.users import UserModel from open_webui.models.users import UserModel
from open_webui.retrieval.loaders.youtube import YoutubeLoader from open_webui.retrieval.loaders.youtube import YoutubeLoader
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.retrieval.external import retrieve_external_knowledge
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
from open_webui.retrieval.vector.main import GetResult, SearchResult from open_webui.retrieval.vector.main import GetResult, SearchResult
from open_webui.retrieval.web.utils import get_web_loader from open_webui.retrieval.web.utils import get_web_loader
@@ -84,6 +85,7 @@ LOADER_CONFIG_KEYS = {
'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format', 'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'TIKA_SERVER_URL': 'rag.tika_server_url', 'TIKA_SERVER_URL': 'rag.tika_server_url',
'DOCLING_SERVER_URL': 'rag.docling_server_url', 'DOCLING_SERVER_URL': 'rag.docling_server_url',
'DOCLING_API_KEY': 'rag.docling_api_key', 'DOCLING_API_KEY': 'rag.docling_api_key',
@@ -130,18 +132,16 @@ def build_loader_from_config(request, config: dict):
"""Build a Loader instance with the admin's configured extraction engine settings.""" """Build a Loader instance with the admin's configured extraction engine settings."""
from open_webui.retrieval.loaders.main import Loader from open_webui.retrieval.loaders.main import Loader
loader_config = { loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
key: config.get(key)
for key in LOADER_CONFIG_KEYS
if key.isupper()
}
return Loader( return Loader(
engine=loader_config['CONTENT_EXTRACTION_ENGINE'], engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'}, **{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
) )
def _extract_text_from_binary_response(request, response: requests.Response, url: str, loader_config: dict) -> tuple[str, list]: def _extract_text_from_binary_response(
request, response: requests.Response, url: str, loader_config: dict
) -> tuple[str, list]:
"""Download response body to a temp file and extract text using the Loader pipeline.""" """Download response body to a temp file and extract text using the Loader pipeline."""
import mimetypes import mimetypes
import tempfile import tempfile
@@ -774,11 +774,7 @@ async def query_collection_with_hybrid_search(
return result return result
native_task_results = await asyncio.gather( native_task_results = await asyncio.gather(
*[ *[process_native_query(collection_name, query) for collection_name in collection_names for query in queries]
process_native_query(collection_name, query)
for collection_name in collection_names
for query in queries
]
) )
if native_task_results and all(result is not None for result in native_task_results): if native_task_results and all(result is not None for result in native_task_results):
return merge_and_sort_query_results(native_task_results, k=k) return merge_and_sort_query_results(native_task_results, k=k)
@@ -1485,50 +1481,61 @@ async def get_sources_from_items(
permission='read', permission='read',
) )
): ):
if item.get('context') == 'full' or bypass_embedding_and_retrieval: if (knowledge_base.meta or {}).get('source') == 'external':
if knowledge_base and ( query_result = await retrieve_external_knowledge(
user.role == 'admin' request,
or knowledge_base.user_id == user.id knowledge_base,
or await AccessGrants.has_access( queries=queries,
user_id=user.id, count=k,
resource_type='knowledge', user=user,
resource_id=knowledge_base.id, )
permission='read', extracted_collections.append(knowledge_base.id)
)
):
files = await Knowledges.get_files_by_id(knowledge_base.id)
documents = []
metadatas = []
for file in files:
documents.append(file.data.get('content', ''))
metadatas.append(
{
'file_id': file.id,
'name': file.filename,
'source': file.filename,
}
)
query_result = {
'documents': [documents],
'metadatas': [metadatas],
}
else: else:
if item.get('legacy'): if item.get('context') == 'full' or bypass_embedding_and_retrieval:
if BYPASS_RETRIEVAL_ACCESS_CONTROL: if knowledge_base and (
collection_names = item.get('collection_names', []) user.role == 'admin'
else: or knowledge_base.user_id == user.id
# Legacy KB: item.collection_names is client-supplied. or await AccessGrants.has_access(
# Validate against the KB's actual files to prevent user_id=user.id,
# cross-tenant collection name substitution. resource_type='knowledge',
resource_id=knowledge_base.id,
permission='read',
)
):
files = await Knowledges.get_files_by_id(knowledge_base.id) files = await Knowledges.get_files_by_id(knowledge_base.id)
owned_names = {f'file-{f.id}' for f in files}
owned_names.add(knowledge_base.id) documents = []
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names] metadatas = []
collection_names = valid_names if valid_names else [knowledge_base.id] for file in files:
documents.append(file.data.get('content', ''))
metadatas.append(
{
'file_id': file.id,
'name': file.filename,
'source': file.filename,
}
)
query_result = {
'documents': [documents],
'metadatas': [metadatas],
}
else: else:
collection_names.append(item['id']) if item.get('legacy'):
if BYPASS_RETRIEVAL_ACCESS_CONTROL:
collection_names = item.get('collection_names', [])
else:
# Legacy KB: item.collection_names is client-supplied.
# Validate against the KB's actual files to prevent
# cross-tenant collection name substitution.
files = await Knowledges.get_files_by_id(knowledge_base.id)
owned_names = {f'file-{f.id}' for f in files}
owned_names.add(knowledge_base.id)
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
collection_names = valid_names if valid_names else [knowledge_base.id]
else:
collection_names.append(item['id'])
elif item.get('docs'): elif item.get('docs'):
# BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL # BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
+614 -5
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import asyncio import asyncio
import io import io
import logging import logging
import time
import uuid
import zipfile import zipfile
from typing import List, Optional from typing import List, Optional
from urllib.parse import quote from urllib.parse import quote
@@ -27,6 +29,7 @@ from open_webui.models.knowledge import (
) )
from open_webui.models.models import ModelForm, Models from open_webui.models.models import ModelForm, Models
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection
from open_webui.routers.retrieval import ( from open_webui.routers.retrieval import (
BatchProcessFilesForm, BatchProcessFilesForm,
ProcessFileForm, ProcessFileForm,
@@ -110,6 +113,17 @@ class KnowledgeAccessListResponse(BaseModel):
total: int total: int
def is_external_knowledge(knowledge) -> bool:
return (knowledge.meta or {}).get('source') == 'external'
def external_knowledge_error():
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='External knowledge bases are read-only.',
)
@router.get('/', response_model=KnowledgeAccessListResponse) @router.get('/', response_model=KnowledgeAccessListResponse)
async def get_knowledge_bases( async def get_knowledge_bases(
page: int | None = 1, page: int | None = 1,
@@ -163,6 +177,7 @@ async def get_knowledge_bases(
async def search_knowledge_bases( async def search_knowledge_bases(
query: str | None = None, query: str | None = None,
view_option: str | None = None, view_option: str | None = None,
source: str | None = None,
page: int | None = 1, page: int | None = 1,
user=Depends(get_verified_user), user=Depends(get_verified_user),
db: AsyncSession = Depends(get_async_session), db: AsyncSession = Depends(get_async_session),
@@ -176,6 +191,8 @@ async def search_knowledge_bases(
filter['query'] = query filter['query'] = query
if view_option: if view_option:
filter['view_option'] = view_option filter['view_option'] = view_option
if source in {'local', 'external'}:
filter['source'] = source
groups = await Groups.get_groups_by_member_id(user.id, db=db) groups = await Groups.get_groups_by_member_id(user.id, db=db)
user_group_ids = {group.id for group in groups} user_group_ids = {group.id for group in groups}
@@ -379,6 +396,574 @@ async def reindex_knowledge_base_metadata_embeddings(
return {'total': len(knowledge_bases), 'success': success_count} return {'total': len(knowledge_bases), 'success': success_count}
############################
# External Knowledge Sources
############################
class ExternalKnowledgeSourceForm(BaseModel):
type: str = 'collection'
name: str
config: Optional[dict] = None
class ExternalKnowledgeCreateForm(BaseModel):
name: str
description: str = ''
connection_id: str
source: ExternalKnowledgeSourceForm
access_grants: Optional[list[dict]] = None
class ExternalKnowledgeSourceCreateForm(BaseModel):
name: str
description: str = ''
connection: ExternalKnowledgeConnectionForm
source: ExternalKnowledgeSourceForm
access_grants: Optional[list[dict]] = None
test_query: str
test_count: int = 5
class ExternalKnowledgeSourceUpdateForm(ExternalKnowledgeSourceCreateForm):
pass
class ExternalKnowledgeSourceTestForm(BaseModel):
connection_id: Optional[str] = None
connection: ExternalKnowledgeConnectionForm
source: ExternalKnowledgeSourceForm
query: str
count: int = 5
class ExternalKnowledgeRetrieveTestForm(BaseModel):
query: str
source: Optional[ExternalKnowledgeSourceForm] = None
count: int = 5
class ExternalKnowledgeConnectionForm(BaseModel):
name: str
provider: str
endpoint: str
auth_config: Optional[dict] = None
config: Optional[dict] = None
capabilities: Optional[dict] = None
enabled: bool = True
class ExternalKnowledgeConnectionListResponse(BaseModel):
items: list[dict]
total: int
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
EXTERNAL_KNOWLEDGE_PROVIDERS = {'qdrant', 'milvus', 'pgvector'}
def _validate_external_connection_form(form_data: ExternalKnowledgeConnectionForm) -> tuple[str, dict]:
provider = form_data.provider.lower().strip()
if provider not in EXTERNAL_KNOWLEDGE_PROVIDERS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Unsupported external knowledge provider.',
)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source name is required.')
if not form_data.endpoint.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source endpoint is required.')
config = form_data.config or {}
allowed_config_keys = {'timeout'}
if provider == 'milvus':
allowed_config_keys.add('db_name')
return provider, {key: value for key, value in config.items() if key in allowed_config_keys}
def _external_auth_config(provider: str, incoming: Optional[dict], existing: Optional[dict] = None) -> dict:
if provider == 'pgvector':
return {}
return existing if incoming is None else incoming or {}
def _normalize_external_source(source: ExternalKnowledgeSourceForm, provider: str) -> ExternalKnowledgeSourceForm:
source.type = (source.type or 'collection').strip()
source.name = source.name.strip()
if source.type != 'collection':
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Only collection sources are supported.')
if not source.name:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Collection name is required.')
config = source.config or {}
allowed_keys = {'content_field', 'metadata_field', 'document_id_field'}
if provider in {'qdrant', 'milvus'}:
allowed_keys.add('vector_field')
if provider == 'pgvector':
allowed_keys.update({'table_name', 'collection_field', 'vector_field'})
normalized_config = {
key: value.strip() if isinstance(value, str) else value
for key, value in config.items()
if key in allowed_keys and value is not None and (not isinstance(value, str) or value.strip())
}
if not normalized_config.get('content_field'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Content field is required.')
if provider in {'milvus', 'pgvector'} and not normalized_config.get('vector_field'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Vector field is required.')
source.config = normalized_config
return source
def _sanitize_external_connection(connection: dict) -> dict:
sanitized = {**connection}
sanitized.pop('auth_config', None)
sanitized['auth_configured'] = bool(connection.get('auth_config'))
return sanitized
async def _get_external_connections() -> list[dict]:
return await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
async def _set_external_connections(connections: list[dict]) -> None:
await Config.upsert({EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY: connections})
def _external_connection_dict(form_data: ExternalKnowledgeConnectionForm, user_id: str, id: Optional[str] = None) -> dict:
provider, config = _validate_external_connection_form(form_data)
now = int(time.time())
return {
'id': id or str(uuid.uuid4()),
'name': form_data.name.strip(),
'provider': provider,
'endpoint': form_data.endpoint.strip(),
'auth_config': _external_auth_config(provider, form_data.auth_config),
'config': config,
'capabilities': form_data.capabilities or {'retrieve': True},
'health': None,
'enabled': form_data.enabled,
'created_by': user_id,
'created_at': now,
'updated_at': now,
}
def _external_connection_update_dict(
form_data: ExternalKnowledgeConnectionForm,
existing: dict,
) -> dict:
provider, config = _validate_external_connection_form(form_data)
return {
**existing,
'name': form_data.name.strip(),
'provider': provider,
'endpoint': form_data.endpoint.strip(),
'auth_config': _external_auth_config(provider, form_data.auth_config, existing.get('auth_config')) or {},
'config': config,
'capabilities': form_data.capabilities or {'retrieve': True},
'enabled': form_data.enabled,
'updated_at': int(time.time()),
}
async def _get_external_connection(id: str) -> Optional[dict]:
connections = await _get_external_connections()
return next((connection for connection in connections if connection.get('id') == id), None)
async def _count_external_connection_mappings(connection_id: str, db: Optional[AsyncSession] = None) -> int:
count = 0
for knowledge in await Knowledges.get_knowledge_bases(db=db):
if (knowledge.meta or {}).get('external', {}).get('connection_id') == connection_id:
count += 1
return count
@router.get('/external/connections', response_model=ExternalKnowledgeConnectionListResponse)
async def get_external_knowledge_connections(
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connections = [_sanitize_external_connection(connection) for connection in await _get_external_connections()]
return ExternalKnowledgeConnectionListResponse(items=connections, total=len(connections))
@router.post('/external/connections', response_model=dict)
async def create_external_knowledge_connection(
form_data: ExternalKnowledgeConnectionForm,
user=Depends(get_admin_user),
):
connections = await _get_external_connections()
connection = _external_connection_dict(form_data, user.id)
connections.append(connection)
await _set_external_connections(connections)
return _sanitize_external_connection(connection)
@router.get('/external/connections/{id}', response_model=dict)
async def get_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
return _sanitize_external_connection(connection)
@router.patch('/external/connections/{id}', response_model=dict)
async def update_external_knowledge_connection(
id: str,
form_data: ExternalKnowledgeConnectionForm,
user=Depends(get_admin_user),
):
connections = await _get_external_connections()
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == id), None)
if idx is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
connection = _external_connection_update_dict(form_data, connections[idx])
connections[idx] = connection
await _set_external_connections(connections)
return _sanitize_external_connection(connection)
@router.delete('/external/connections/{id}', response_model=bool)
async def delete_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if await _count_external_connection_mappings(id, db=db) > 0:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='External connection is still used by knowledge bases.',
)
connections = [connection for connection in await _get_external_connections() if connection.get('id') != id]
await _set_external_connections(connections)
return True
@router.post('/external/connections/{id}/test', response_model=dict)
async def test_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
health = {
'ok': bool(connection.get('enabled') and connection.get('endpoint')),
'provider': connection.get('provider'),
'checked_at': int(time.time()),
}
connections = await _get_external_connections()
for item in connections:
if item.get('id') == id:
item['health'] = health
item['updated_at'] = int(time.time())
break
await _set_external_connections(connections)
return health
async def _test_external_source_definition(
request: Request,
connection: dict,
source: ExternalKnowledgeSourceForm,
query: str,
count: int,
user,
) -> dict:
if not query.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query is required.')
source = _normalize_external_source(source, connection.get('provider'))
test_knowledge = KnowledgeResponse(
id='external-test',
user_id=user.id,
name=connection.get('name'),
description='',
meta={
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
},
access_grants=[],
created_at=int(time.time()),
updated_at=int(time.time()),
)
result = await retrieve_external_knowledge_for_connection(
request,
test_knowledge,
connection,
[query.strip()],
count,
user=user,
)
return {
'documents': result.get('documents', [[]])[0],
'metadatas': result.get('metadatas', [[]])[0],
'distances': result.get('distances', [[]])[0],
}
@router.post('/external/source/test', response_model=dict)
async def test_external_knowledge_source(
request: Request,
form_data: ExternalKnowledgeSourceTestForm,
user=Depends(get_admin_user),
):
if form_data.connection_id:
existing_connection = await _get_external_connection(form_data.connection_id)
if not existing_connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
connection = _external_connection_update_dict(form_data.connection, existing_connection)
else:
connection = _external_connection_dict(form_data.connection, user.id, id='external-test')
return await _test_external_source_definition(
request,
connection,
form_data.source,
form_data.query,
form_data.count,
user,
)
@router.post('/external/connections/{id}/retrieve-test', response_model=dict)
async def test_external_knowledge_retrieval(
request: Request,
id: str,
form_data: ExternalKnowledgeRetrieveTestForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
source = form_data.source or ExternalKnowledgeSourceForm(name='test', config={'content_field': 'payload.text'})
return await _test_external_source_definition(request, connection, source, form_data.query, form_data.count, user)
@router.post('/external/knowledge/create', response_model=KnowledgeResponse | None)
async def create_external_knowledge(
request: Request,
form_data: ExternalKnowledgeCreateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(form_data.connection_id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
source = _normalize_external_source(form_data.source, connection.get('provider'))
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
knowledge = await Knowledges.insert_new_knowledge(
user.id,
KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not knowledge:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': form_data.connection_id,
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
return knowledge
@router.post('/external/source/create', response_model=KnowledgeResponse | None)
async def create_external_knowledge_source(
request: Request,
form_data: ExternalKnowledgeSourceCreateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
connection = _external_connection_dict(form_data.connection, user.id)
source = _normalize_external_source(form_data.source, connection.get('provider'))
test_result = await _test_external_source_definition(
request,
connection,
source,
form_data.test_query,
form_data.test_count,
user,
)
if not test_result.get('documents'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
connections = await _get_external_connections()
connections.append(connection)
await _set_external_connections(connections)
knowledge = await Knowledges.insert_new_knowledge(
user.id,
KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not knowledge:
connections = [item for item in await _get_external_connections() if item.get('id') != connection.get('id')]
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
return knowledge
@router.patch('/external/source/{id}', response_model=KnowledgeResponse | None)
async def update_external_knowledge_source(
request: Request,
id: str,
form_data: ExternalKnowledgeSourceUpdateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
if not knowledge or not is_external_knowledge(knowledge):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
connections = await _get_external_connections()
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == connection_id), None)
if idx is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
existing_connection = connections[idx]
connection = _external_connection_update_dict(form_data.connection, existing_connection)
source = _normalize_external_source(form_data.source, connection.get('provider'))
test_result = await _test_external_source_definition(
request,
connection,
source,
form_data.test_query,
form_data.test_count,
user,
)
if not test_result.get('documents'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
connections[idx] = connection
await _set_external_connections(connections)
updated = await Knowledges.update_knowledge_by_id(
id=id,
form_data=KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not updated:
connections[idx] = existing_connection
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
updated = await Knowledges.update_knowledge_meta_by_id(id, meta, db=db)
if not updated:
connections[idx] = existing_connection
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
await embed_knowledge_base_metadata(request, id, updated.name, updated.description)
return updated
############################ ############################
# GetKnowledgeById # GetKnowledgeById
############################ ############################
@@ -711,6 +1296,8 @@ async def add_file_to_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST, status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
@@ -798,6 +1385,8 @@ async def update_file_from_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST, status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
@@ -877,6 +1466,8 @@ async def remove_file_from_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST, status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
@@ -1004,11 +1595,21 @@ async def delete_knowledge_by_id(
await Models.update_model_by_id(model.id, model_form, db=db) await Models.update_model_by_id(model.id, model_form, db=db)
# Clean up vector DB # Clean up vector DB
try: if is_external_knowledge(knowledge):
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id) connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
except Exception as e: if connection_id:
log.debug(e) connections = [
pass connection
for connection in await _get_external_connections()
if connection.get('id') != connection_id
]
await _set_external_connections(connections)
else:
try:
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
except Exception as e:
log.debug(e)
pass
# Remove knowledge base embedding # Remove knowledge base embedding
await remove_knowledge_base_metadata_embedding(id) await remove_knowledge_base_metadata_embedding(id)
@@ -1035,6 +1636,8 @@ async def reset_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST, status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
@@ -1263,6 +1866,8 @@ async def add_files_to_knowledge_batch(
status_code=status.HTTP_400_BAD_REQUEST, status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
@@ -1380,6 +1985,8 @@ async def export_knowledge_by_id(id: str, user=Depends(get_admin_user), db: Asyn
status_code=status.HTTP_404_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
files = await Knowledges.get_files_by_id(id, db=db) files = await Knowledges.get_files_by_id(id, db=db)
@@ -1441,6 +2048,8 @@ async def _verify_knowledge_write_access(id: str, user, db: AsyncSession):
status_code=status.HTTP_404_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND,
detail=ERROR_MESSAGES.NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND,
) )
if is_external_knowledge(knowledge):
external_knowledge_error()
if ( if (
knowledge.user_id != user.id knowledge.user_id != user.id
and not await AccessGrants.has_access( and not await AccessGrants.has_access(
+14
View File
@@ -288,6 +288,7 @@ RETRIEVAL_CONFIG_KEYS = {
'ENABLE_WEB_SEARCH': 'rag.web.search.enable', 'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
'EXA_API_KEY': 'rag.web.search.exa_api_key', 'EXA_API_KEY': 'rag.web.search.exa_api_key',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key', 'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url', 'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
@@ -632,6 +633,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY, 'DOCLING_API_KEY': config.DOCLING_API_KEY,
@@ -846,6 +848,7 @@ class ConfigForm(BaseModel):
EXTERNAL_DOCUMENT_LOADER_URL: str | None = None EXTERNAL_DOCUMENT_LOADER_URL: str | None = None
EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
TIKA_SERVER_URL: str | None = None TIKA_SERVER_URL: str | None = None
DOCLING_SERVER_URL: str | None = None DOCLING_SERVER_URL: str | None = None
@@ -1021,6 +1024,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None
else config.EXTERNAL_DOCUMENT_LOADER_API_KEY else config.EXTERNAL_DOCUMENT_LOADER_API_KEY
) )
config.EXTERNAL_DOCUMENT_LOADER_HEADERS = (
form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS
if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None
else config.EXTERNAL_DOCUMENT_LOADER_HEADERS
)
config.TIKA_SERVER_URL = ( config.TIKA_SERVER_URL = (
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
) )
@@ -1336,6 +1344,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY, 'DOCLING_API_KEY': config.DOCLING_API_KEY,
@@ -1843,6 +1852,11 @@ async def process_file(
loader_config = await get_loader_config() loader_config = await get_loader_config()
loader = build_loader_from_config(request, loader_config) loader = build_loader_from_config(request, loader_config)
loader.user = user loader.user = user
loader.metadata = {
'file_id': file.id,
'file_name': file.filename,
'file_content_type': file.meta.get('content_type'),
}
docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path) docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path)
docs = [ docs = [
+43 -13
View File
@@ -514,21 +514,14 @@ async def execute_code(
elif engine == 'jupyter': elif engine == 'jupyter':
from open_webui.utils.code_interpreter import execute_code_jupyter from open_webui.utils.code_interpreter import execute_code_jupyter
jupyter_auth = await Config.get('code_interpreter.jupyter.auth') jupyter_auth = await Config.get('code_interpreter.jupyter.auth')
output = await execute_code_jupyter( output = await execute_code_jupyter(
await Config.get('code_interpreter.jupyter.url'), await Config.get('code_interpreter.jupyter.url'),
code, code,
( (await Config.get('code_interpreter.jupyter.auth_token') if jupyter_auth == 'token' else None),
await Config.get('code_interpreter.jupyter.auth_token') (await Config.get('code_interpreter.jupyter.auth_password') if jupyter_auth == 'password' else None),
if jupyter_auth == 'token'
else None
),
(
await Config.get('code_interpreter.jupyter.auth_password')
if jupyter_auth == 'password'
else None
),
await Config.get('code_interpreter.jupyter.timeout'), await Config.get('code_interpreter.jupyter.timeout'),
) )
@@ -2385,6 +2378,7 @@ async def query_knowledge_files(
from open_webui.models.files import Files from open_webui.models.files import Files
from open_webui.models.knowledge import Knowledges from open_webui.models.knowledge import Knowledges
from open_webui.models.notes import Notes from open_webui.models.notes import Notes
from open_webui.retrieval.external import retrieve_external_knowledge
from open_webui.retrieval.utils import query_collection from open_webui.retrieval.utils import query_collection
user_id = __user__.get('id') user_id = __user__.get('id')
@@ -2396,6 +2390,7 @@ async def query_knowledge_files(
return json.dumps({'error': 'Embedding function not configured'}) return json.dumps({'error': 'Embedding function not configured'})
collection_names = [] collection_names = []
external_knowledges = []
note_results = [] # Notes aren't vectorized, handle separately note_results = [] # Notes aren't vectorized, handle separately
# If model has attached knowledge, use those # If model has attached knowledge, use those
@@ -2418,7 +2413,10 @@ async def query_knowledge_files(
user_group_ids=set(user_group_ids), user_group_ids=set(user_group_ids),
) )
): ):
collection_names.append(item_id) if (knowledge.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge)
else:
collection_names.append(item_id)
elif item_type == 'file': elif item_type == 'file':
# Individual file - use file-{id} as collection name # Individual file - use file-{id} as collection name
@@ -2464,7 +2462,10 @@ async def query_knowledge_files(
user_group_ids=set(user_group_ids), user_group_ids=set(user_group_ids),
) )
): ):
collection_names.append(knowledge_id) if (knowledge.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge)
else:
collection_names.append(knowledge_id)
else: else:
# No model knowledge and no specific IDs - search all accessible KBs # No model knowledge and no specific IDs - search all accessible KBs
result = await Knowledges.search_knowledge_bases( result = await Knowledges.search_knowledge_bases(
@@ -2477,7 +2478,11 @@ async def query_knowledge_files(
skip=0, skip=0,
limit=50, limit=50,
) )
collection_names = [knowledge_base.id for knowledge_base in result.items] for knowledge_base in result.items:
if (knowledge_base.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge_base)
else:
collection_names.append(knowledge_base.id)
chunks = [] chunks = []
@@ -2509,6 +2514,31 @@ async def query_knowledge_files(
chunk_info['distance'] = distances[idx] chunk_info['distance'] = distances[idx]
chunks.append(chunk_info) chunks.append(chunk_info)
for knowledge in external_knowledges:
query_results = await retrieve_external_knowledge(
__request__,
knowledge,
queries=[query],
count=count,
user=type('UserContext', (), {'id': user_id, 'role': user_role})(),
)
documents = query_results.get('documents', [[]])[0]
metadatas = query_results.get('metadatas', [[]])[0]
distances = query_results.get('distances', [[]])[0]
for idx, doc in enumerate(documents):
metadata = metadatas[idx] if idx < len(metadatas) else {}
chunk_info = {
'content': doc,
'source': metadata.get('source', metadata.get('name', knowledge.name)),
'file_id': metadata.get('file_id', f'external-{knowledge.id}'),
'type': 'external',
'knowledge_id': knowledge.id,
}
if idx < len(distances):
chunk_info['distance'] = distances[idx]
chunks.append(chunk_info)
# Limit to requested count # Limit to requested count
chunks = chunks[:count] chunks = chunks[:count]
+301 -1
View File
@@ -38,6 +38,304 @@ export const createNewKnowledge = async (
return res; return res;
}; };
export const getExternalKnowledgeConnections = async (token: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledgeConnection = async (token: string, connection: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(connection)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const updateExternalKnowledgeConnection = async (
token: string,
id: string,
connection: object
) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
method: 'PATCH',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(connection)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const deleteExternalKnowledgeConnection = async (token: string, id: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeConnection = async (token: string, id: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/test`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeRetrieval = async (
token: string,
id: string,
payload: object
) => {
let error = null;
const res = await fetch(
`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/retrieve-test`,
{
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
}
)
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeSource = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/test`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledgeSource = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const updateExternalKnowledgeSource = async (token: string, id: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/${id}`, {
method: 'PATCH',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledge = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/knowledge/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getKnowledgeBases = async (token: string = '', page: number | null = null) => { export const getKnowledgeBases = async (token: string = '', page: number | null = null) => {
let error = null; let error = null;
@@ -76,13 +374,15 @@ export const searchKnowledgeBases = async (
token: string = '', token: string = '',
query: string | null = null, query: string | null = null,
viewOption: string | null = null, viewOption: string | null = null,
page: number | null = null page: number | null = null,
source: string | null = null
) => { ) => {
let error = null; let error = null;
const searchParams = new URLSearchParams(); const searchParams = new URLSearchParams();
if (query) searchParams.append('query', query); if (query) searchParams.append('query', query);
if (viewOption) searchParams.append('view_option', viewOption); if (viewOption) searchParams.append('view_option', viewOption);
if (source) searchParams.append('source', source);
if (page) searchParams.append('page', page.toString()); if (page) searchParams.append('page', page.toString());
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, { const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, {
+1
View File
@@ -54,6 +54,7 @@ type RAGConfigForm = {
PDF_EXTRACT_IMAGES?: boolean; PDF_EXTRACT_IMAGES?: boolean;
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean; ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
ENABLE_ONEDRIVE_INTEGRATION?: boolean; ENABLE_ONEDRIVE_INTEGRATION?: boolean;
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
chunk?: ChunkConfigForm; chunk?: ChunkConfigForm;
content_extraction?: ContentExtractConfigForm; content_extraction?: ContentExtractConfigForm;
web_loader_ssl_verification?: boolean; web_loader_ssl_verification?: boolean;
+15 -1
View File
@@ -155,7 +155,21 @@
id: 'integrations', id: 'integrations',
title: 'Integrations', title: 'Integrations',
route: '/admin/settings/integrations', route: '/admin/settings/integrations',
keywords: ['tools', 'integrations', 'plugins', 'extensions', 'functions', 'openapi', 'server'] keywords: [
'tools',
'integrations',
'plugins',
'extensions',
'functions',
'openapi',
'server',
'knowledge',
'vector db',
'qdrant',
'rag',
'retrieval',
'sources'
]
}, },
{ {
id: 'documents', id: 'documents',
@@ -37,6 +37,7 @@
let showResetConfirm = false; let showResetConfirm = false;
let showResetUploadDirConfirm = false; let showResetUploadDirConfirm = false;
let showReindexConfirm = false; let showReindexConfirm = false;
let showExternalDocumentLoaderHeadersHint = false;
let RAG_EMBEDDING_ENGINE = ''; let RAG_EMBEDDING_ENGINE = '';
let RAG_EMBEDDING_MODEL = ''; let RAG_EMBEDDING_MODEL = '';
@@ -149,6 +150,21 @@
toast.error($i18n.t('External Document Loader URL required.')); toast.error($i18n.t('External Document Loader URL required.'));
return; return;
} }
if (
RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external' &&
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS
) {
try {
const headers = JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS);
if (headers === null || typeof headers !== 'object' || Array.isArray(headers)) {
throw new Error('Headers must be a valid JSON object');
}
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS = JSON.stringify(headers, null, 2);
} catch (error) {
toast.error($i18n.t('Headers must be a valid JSON object'));
return;
}
}
if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') { if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') {
toast.error($i18n.t('Tika Server URL required.')); toast.error($i18n.t('Tika Server URL required.'));
return; return;
@@ -241,6 +257,11 @@
typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== '' typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== ''
? JSON.parse(RAGConfig.DOCLING_PARAMS) ? JSON.parse(RAGConfig.DOCLING_PARAMS)
: {}, : {},
EXTERNAL_DOCUMENT_LOADER_HEADERS:
typeof RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'string' &&
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS.trim() !== ''
? JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS)
: {},
MINERU_PARAMS: MINERU_PARAMS:
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== '' typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
? JSON.parse(RAGConfig.MINERU_PARAMS) ? JSON.parse(RAGConfig.MINERU_PARAMS)
@@ -289,6 +310,13 @@
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2) ? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
: config.MINERU_PARAMS; : config.MINERU_PARAMS;
config.EXTERNAL_DOCUMENT_LOADER_HEADERS =
typeof config.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'object'
? Object.keys(config.EXTERNAL_DOCUMENT_LOADER_HEADERS ?? {}).length > 0
? JSON.stringify(config.EXTERNAL_DOCUMENT_LOADER_HEADERS, null, 2)
: ''
: config.EXTERNAL_DOCUMENT_LOADER_HEADERS;
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', '); config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
RAGConfig = config; RAGConfig = config;
@@ -581,17 +609,78 @@
</div> </div>
</div> </div>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'} {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
<div class="my-0.5 flex gap-2 pr-2"> <div class="my-0.5 flex flex-col gap-2 pr-2">
<input <div class="flex gap-2">
class="flex-1 w-full text-sm bg-transparent outline-hidden" <input
placeholder={$i18n.t('Enter External Document Loader URL')} class="flex-1 w-full text-sm bg-transparent outline-hidden"
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL} placeholder={$i18n.t('Enter External Document Loader URL')}
/> bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL}
<SensitiveInput />
placeholder={$i18n.t('Enter External Document Loader API Key')} <SensitiveInput
required={false} placeholder={$i18n.t('Enter External Document Loader API Key')}
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY} required={false}
/> bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY}
/>
</div>
<div class="flex flex-col">
<div class="mb-0.5 text-xs text-gray-500">{$i18n.t('Headers')}</div>
<Tooltip
content={$i18n.t(
'Enter additional headers in JSON format (e.g. {"X-Custom-Header": "value"}'
)}
>
<Textarea
className="w-full text-sm outline-hidden"
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS}
placeholder={$i18n.t('Enter additional headers in JSON format')}
required={false}
/>
</Tooltip>
<button
type="button"
class="mt-1 flex items-center gap-1 text-xs text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 transition w-fit"
on:click={() =>
(showExternalDocumentLoaderHeadersHint =
!showExternalDocumentLoaderHeadersHint)}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-3 h-3 transition-transform {showExternalDocumentLoaderHeadersHint
? 'rotate-90'
: ''}"
>
<path
fill-rule="evenodd"
d="M7.21 14.77a.75.75 0 01.02-1.06L11.168 10 7.23 6.29a.75.75 0 111.04-1.08l4.5 4.25a.75.75 0 010 1.08l-4.5 4.25a.75.75 0 01-1.06-.02z"
clip-rule="evenodd"
/>
</svg>
{$i18n.t('Header variables')}
</button>
{#if showExternalDocumentLoaderHeadersHint}
<div class="mt-1 text-xs text-gray-500 dark:text-gray-400 leading-5">
<div>
{$i18n.t('No additional headers are sent unless configured.')}
</div>
<div>
{$i18n.t('Example')}:
<code class="text-gray-700 dark:text-gray-300"
>{'{"X-OpenWebUI-File-Id": "{{FILE_ID}}"}'}</code
>
</div>
<div>
{$i18n.t('Available variables')}:
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_ID}}'}</code>,
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_NAME}}'}</code>,
<code class="text-gray-700 dark:text-gray-300"
>{'{{FILE_CONTENT_TYPE}}'}</code
>
</div>
</div>
{/if}
</div>
</div> </div>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'} {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
<div class="flex w-full mt-1"> <div class="flex w-full mt-1">
@@ -0,0 +1,883 @@
<script lang="ts">
import { getContext, onMount } from 'svelte';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
import { toast } from 'svelte-sonner';
import AccessControl from '$lib/components/workspace/common/AccessControl.svelte';
import Modal from '$lib/components/common/Modal.svelte';
import Spinner from '$lib/components/common/Spinner.svelte';
import Switch from '$lib/components/common/Switch.svelte';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import Cog6 from '$lib/components/icons/Cog6.svelte';
import DatabaseSettings from '$lib/components/icons/DatabaseSettings.svelte';
import Plus from '$lib/components/icons/Plus.svelte';
import XMark from '$lib/components/icons/XMark.svelte';
import {
createExternalKnowledgeSource,
getExternalKnowledgeConnections,
searchKnowledgeBases,
testExternalKnowledgeSource,
updateExternalKnowledgeConnection,
updateExternalKnowledgeSource
} from '$lib/apis/knowledge';
const i18n = getContext<Writable<i18nType>>('i18n');
type ExternalKnowledgeConnection = {
id: string;
name: string;
provider: string;
endpoint: string;
config?: Record<string, any>;
capabilities?: Record<string, any>;
enabled?: boolean;
auth_configured?: boolean;
};
type ExternalKnowledgeItem = {
id: string;
name: string;
description?: string;
access_grants?: any[];
meta?: {
external?: {
connection_id?: string;
provider?: string;
source?: {
name?: string;
config?: Record<string, any>;
};
};
};
};
let loading = false;
let testing = false;
let creating = false;
let showSourceModal = false;
let connections: ExternalKnowledgeConnection[] = [];
let items: ExternalKnowledgeItem[] = [];
let editingItem: ExternalKnowledgeItem | null = null;
let editingConnection: ExternalKnowledgeConnection | null = null;
let accessGrants: any[] = [];
let testResult: {
documents?: string[];
metadatas?: Record<string, any>[];
distances?: number[];
} | null = null;
let sourceForm = {
name: '',
description: '',
provider: 'qdrant',
endpoint: '',
apiKey: '',
timeout: 30,
dbName: '',
sourceName: '',
contentField: 'payload.text',
vectorField: '',
metadataField: 'payload.metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name',
testQuery: ''
};
const schemaDefaults = (provider = sourceForm.provider) => {
if (provider === 'milvus') {
return {
contentField: 'data.text',
vectorField: 'vector',
metadataField: 'metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
}
if (provider === 'pgvector') {
return {
contentField: 'text',
vectorField: 'vector',
metadataField: 'vmetadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
}
return {
contentField: 'payload.text',
vectorField: '',
metadataField: 'payload.metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
};
const resetForm = () => {
sourceForm = {
name: '',
description: '',
provider: 'qdrant',
endpoint: '',
apiKey: '',
timeout: 30,
dbName: '',
sourceName: '',
testQuery: '',
...schemaDefaults('qdrant')
};
accessGrants = [];
testResult = null;
editingItem = null;
editingConnection = null;
};
const openCreateSource = () => {
resetForm();
showSourceModal = true;
};
const openEditSource = (item: ExternalKnowledgeItem) => {
const connection = connectionForItem(item);
if (!connection) {
toast.error($i18n.t('External connection not found.'));
return;
}
resetForm();
editingItem = item;
editingConnection = connection;
const source = item.meta?.external?.source;
const sourceConfig = source?.config ?? {};
const defaults = schemaDefaults(connection.provider);
sourceForm = {
name: item.name ?? '',
description: item.description ?? '',
provider: connection.provider ?? 'qdrant',
endpoint: connection.endpoint ?? '',
apiKey: '',
timeout: connection.config?.timeout ?? 30,
dbName: connection.config?.db_name ?? '',
sourceName: source?.name ?? '',
testQuery: '',
contentField: sourceConfig.content_field ?? defaults.contentField,
vectorField: sourceConfig.vector_field ?? defaults.vectorField,
metadataField: sourceConfig.metadata_field ?? defaults.metadataField,
documentIdField: sourceConfig.document_id_field ?? defaults.documentIdField,
tableName: sourceConfig.table_name ?? defaults.tableName,
collectionField: sourceConfig.collection_field ?? defaults.collectionField
};
accessGrants = item.access_grants ?? [];
showSourceModal = true;
};
const markUntested = () => {
testResult = null;
};
const refresh = async () => {
loading = true;
const [connectionRes, knowledgeRes] = await Promise.all([
getExternalKnowledgeConnections(localStorage.token).catch((error) => {
toast.error(`${error}`);
return null;
}),
searchKnowledgeBases(localStorage.token, null, null, 1, 'external').catch((error) => {
toast.error(`${error}`);
return null;
})
]);
connections = connectionRes?.items ?? [];
items = knowledgeRes?.items ?? [];
loading = false;
};
const endpointPlaceholder = () => {
if (sourceForm.provider === 'pgvector') {
return 'postgresql://user:password@host:5432/db';
}
if (sourceForm.provider === 'milvus') {
return 'http://milvus.example.com:19530';
}
return 'https://qdrant.example.com';
};
const connectionForItem = (item: ExternalKnowledgeItem) =>
connections.find((connection) => connection.id === item?.meta?.external?.connection_id);
const connectionPayload = () => ({
name:
sourceForm.name.trim() ||
sourceForm.sourceName.trim() ||
$i18n.t('External Knowledge Source'),
provider: sourceForm.provider,
endpoint: sourceForm.endpoint,
auth_config:
sourceForm.provider === 'pgvector'
? {}
: sourceForm.apiKey
? { type: 'bearer', api_key: sourceForm.apiKey }
: editingConnection
? null
: {},
config: {
timeout: Number(sourceForm.timeout) || 30,
...(sourceForm.provider === 'milvus' && sourceForm.dbName
? { db_name: sourceForm.dbName }
: {})
},
capabilities: { retrieve: true },
enabled: editingConnection?.enabled !== false
});
const sourcePayload = () => {
const config: Record<string, string> = {
content_field: sourceForm.contentField.trim(),
...(sourceForm.vectorField.trim() ? { vector_field: sourceForm.vectorField.trim() } : {}),
...(sourceForm.metadataField.trim()
? { metadata_field: sourceForm.metadataField.trim() }
: {}),
...(sourceForm.documentIdField.trim()
? { document_id_field: sourceForm.documentIdField.trim() }
: {})
};
if (sourceForm.provider === 'pgvector') {
config.table_name = sourceForm.tableName.trim();
config.collection_field = sourceForm.collectionField.trim();
}
return {
type: 'collection',
name: sourceForm.sourceName.trim(),
config
};
};
const testFormIsValid = () =>
!!sourceForm.endpoint.trim() &&
!!sourceForm.sourceName.trim() &&
!!sourceForm.contentField.trim() &&
(sourceForm.provider === 'qdrant' || !!sourceForm.vectorField.trim()) &&
(sourceForm.provider !== 'pgvector' ||
(!!sourceForm.tableName.trim() && !!sourceForm.collectionField.trim())) &&
!!sourceForm.testQuery.trim();
const sourceFormIsValid = () => !!sourceForm.name.trim() && testFormIsValid();
const testSource = async () => {
if (!testFormIsValid()) {
toast.error($i18n.t('Fill the source fields and test query first.'));
return;
}
testing = true;
testResult = null;
const res = await testExternalKnowledgeSource(localStorage.token, {
...(editingConnection?.id ? { connection_id: editingConnection.id } : {}),
connection: connectionPayload(),
source: sourcePayload(),
query: sourceForm.testQuery,
count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res?.documents?.length) {
testResult = res;
toast.success($i18n.t('Test succeeded.'));
} else if (res) {
toast.error($i18n.t('Test returned no results.'));
}
testing = false;
};
const createSource = async () => {
if (!sourceFormIsValid()) {
toast.error($i18n.t('Fill the required fields first.'));
return;
}
if (!testResult?.documents?.length) {
toast.error($i18n.t('Test the source before creating it.'));
return;
}
creating = true;
const res = await createExternalKnowledgeSource(localStorage.token, {
name: sourceForm.name,
description: sourceForm.description,
connection: connectionPayload(),
source: sourcePayload(),
access_grants: accessGrants,
test_query: sourceForm.testQuery,
test_count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
toast.success($i18n.t('Knowledge source created.'));
showSourceModal = false;
resetForm();
await refresh();
}
creating = false;
};
const updateSource = async () => {
if (!editingItem) return;
if (!sourceFormIsValid()) {
toast.error($i18n.t('Fill the required fields first.'));
return;
}
if (!testResult?.documents?.length) {
toast.error($i18n.t('Test the source before saving it.'));
return;
}
creating = true;
const res = await updateExternalKnowledgeSource(localStorage.token, editingItem.id, {
name: sourceForm.name,
description: sourceForm.description,
connection: connectionPayload(),
source: sourcePayload(),
access_grants: accessGrants,
test_query: sourceForm.testQuery,
test_count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
toast.success($i18n.t('Knowledge source updated.'));
showSourceModal = false;
resetForm();
await refresh();
}
creating = false;
};
const submitSource = async () => {
if (editingItem) {
await updateSource();
} else {
await createSource();
}
};
const toggleSource = async (item: ExternalKnowledgeItem) => {
const connection = connectionForItem(item);
if (!connection) return;
const res = await updateExternalKnowledgeConnection(localStorage.token, connection.id, {
name: connection.name,
provider: connection.provider,
endpoint: connection.endpoint,
auth_config: null,
config: connection.config ?? {},
capabilities: connection.capabilities ?? { retrieve: true },
enabled: !connection.enabled
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
await refresh();
}
};
const updateProvider = () => {
sourceForm = {
...sourceForm,
...schemaDefaults(sourceForm.provider)
};
markUntested();
};
onMount(refresh);
</script>
<Modal bind:show={showSourceModal} size="sm">
<div>
<div class="flex justify-between dark:text-gray-100 px-5 pt-4 pb-2">
<h1 class="text-lg font-medium self-center font-primary">
{editingItem ? $i18n.t('Edit Knowledge Connection') : $i18n.t('Add Knowledge Connection')}
</h1>
<button
class="self-center"
aria-label={$i18n.t('Close')}
type="button"
on:click={() => {
showSourceModal = false;
resetForm();
}}
>
<XMark className="size-5" />
</button>
</div>
<div class="flex flex-col md:flex-row w-full px-4 pb-4 md:space-x-4 dark:text-gray-200">
<div class="flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
<form class="flex flex-col w-full" on:submit|preventDefault={submitSource}>
<div class="px-1">
<div class="flex gap-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-name"
>{$i18n.t('Name')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-name"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.name}
on:input={markUntested}
placeholder={$i18n.t('Research Knowledge')}
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-provider"
>{$i18n.t('Provider')}</label
>
</div>
<div class="flex flex-1 items-center">
<select
id="external-source-provider"
class="w-full text-sm bg-transparent outline-hidden"
bind:value={sourceForm.provider}
on:change={updateProvider}
>
<option value="qdrant">Qdrant</option>
<option value="milvus">Milvus</option>
<option value="pgvector">pgvector</option>
</select>
</div>
</div>
</div>
<div class="flex flex-col w-full mt-2">
<label class="text-xs text-gray-500 mb-0.5" for="external-source-description"
>{$i18n.t('Description')}</label
>
<textarea
id="external-source-description"
class="w-full text-sm bg-transparent outline-hidden resize-none placeholder:text-gray-300 dark:placeholder:text-gray-700"
rows="2"
bind:value={sourceForm.description}
></textarea>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-endpoint"
>{$i18n.t('Endpoint')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-endpoint"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.endpoint}
on:input={markUntested}
placeholder={endpointPlaceholder()}
required
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-timeout"
>{$i18n.t('Timeout')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-timeout"
class="w-full text-sm bg-transparent outline-hidden"
type="number"
bind:value={sourceForm.timeout}
on:input={markUntested}
/>
</div>
</div>
{#if sourceForm.provider !== 'pgvector'}
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-api-key"
>{$i18n.t('API Key / Token')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-api-key"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
type="password"
bind:value={sourceForm.apiKey}
on:input={markUntested}
autocomplete="off"
/>
</div>
</div>
{/if}
</div>
{#if sourceForm.provider === 'milvus'}
<div class="flex gap-2 mt-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-db-name"
>{$i18n.t('Database')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-db-name"
class="w-full text-sm bg-transparent outline-hidden"
bind:value={sourceForm.dbName}
on:input={markUntested}
placeholder={$i18n.t('Default')}
/>
</div>
</div>
</div>
{/if}
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
<div class="flex gap-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-collection"
>{$i18n.t('Collection')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-collection"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.sourceName}
on:input={markUntested}
placeholder="research-docs"
required
/>
</div>
</div>
</div>
{#if sourceForm.provider === 'pgvector'}
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-table"
>{$i18n.t('Table')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-table"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.tableName}
on:input={markUntested}
placeholder="document_chunk"
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-collection-field"
>{$i18n.t('Collection Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-collection-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.collectionField}
on:input={markUntested}
placeholder="collection_name"
required
/>
</div>
</div>
</div>
{/if}
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-content-field"
>{$i18n.t('Content Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-content-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.contentField}
on:input={markUntested}
placeholder={sourceForm.provider === 'pgvector' ? 'text' : 'payload.text'}
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-vector-field">
{$i18n.t('Vector Field')}
</label>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-vector-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.vectorField}
on:input={markUntested}
placeholder={sourceForm.provider === 'qdrant' ? $i18n.t('Default') : 'vector'}
required={sourceForm.provider !== 'qdrant'}
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-metadata-field"
>{$i18n.t('Metadata Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-metadata-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.metadataField}
on:input={markUntested}
placeholder={sourceForm.provider === 'pgvector'
? 'vmetadata'
: 'payload.metadata'}
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-document-id-field"
>{$i18n.t('Document ID Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-document-id-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.documentIdField}
on:input={markUntested}
placeholder="id"
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-test-query"
>{$i18n.t('Test Query')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-test-query"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.testQuery}
on:input={markUntested}
placeholder={$i18n.t('Ask a test question')}
required
/>
<Tooltip
content={$i18n.t('Verify Connection')}
className="shrink-0 flex items-center mr-1"
>
<button
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition"
type="button"
on:click={testSource}
disabled={testing}
aria-label={$i18n.t('Verify Connection')}
>
{#if testing}
<Spinner />
{:else}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-4 h-4"
aria-hidden="true"
>
<path
fill-rule="evenodd"
d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
clip-rule="evenodd"
/>
</svg>
{/if}
</button>
</Tooltip>
</div>
<div class="text-xs text-gray-500 mt-1">
{$i18n.t(
'External vectors must be generated with the same embedding model configured in Open WebUI.'
)}
</div>
</div>
</div>
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
<AccessControl
bind:accessGrants
accessRoles={['read']}
share={true}
sharePublic={true}
shareUsers={true}
/>
<div class="flex justify-between items-center pt-3 text-sm font-medium">
<div></div>
<button
class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full flex items-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
type="submit"
disabled={creating || !sourceFormIsValid() || !testResult?.documents?.length}
>
{editingItem ? $i18n.t('Save') : $i18n.t('Create')}
{#if creating}<Spinner />{/if}
</button>
</div>
</div>
</form>
</div>
</div>
</div>
</Modal>
<div class="mb-2.5 flex flex-col w-full justify-between text-sm">
<div class="flex justify-between items-center mb-1">
<div class="flex items-center gap-2">
<div class="font-medium">{$i18n.t('External Knowledge Sources')}</div>
<span
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
>{$i18n.t('Experimental')}</span
>
</div>
<Tooltip content={$i18n.t('Add Connection')}>
<button class="px-1" on:click={openCreateSource} type="button">
<Plus />
</button>
</Tooltip>
</div>
<div class="flex flex-col gap-1.5">
{#each items as item}
{@const connection = connectionForItem(item)}
<div class="flex w-full gap-2 items-center">
<Tooltip className="w-full relative" content={''} placement="top-start">
<div class="flex w-full">
<div
class="flex-1 relative flex gap-1.5 items-center min-w-0 {connection?.enabled ===
false
? 'opacity-50'
: ''}"
>
<Tooltip content={$i18n.t('Knowledge')}>
<DatabaseSettings className="size-4" strokeWidth="1.5" />
</Tooltip>
<div class="outline-hidden w-full bg-transparent text-sm min-w-0 line-clamp-1">
<span>{item.name}</span>
{' '}
<span class="text-gray-500">
{item?.meta?.external?.provider ?? connection?.provider}
{item?.meta?.external?.source?.name ? `· ${item.meta.external.source.name}` : ''}
</span>
</div>
</div>
</div>
</Tooltip>
<div class="flex gap-1 items-center">
<Tooltip content={$i18n.t('Configure')}>
<button
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition disabled:opacity-50 disabled:cursor-not-allowed"
type="button"
disabled={!connection}
aria-label={$i18n.t('Configure')}
on:click={() => {
openEditSource(item);
}}
>
<Cog6 />
</button>
</Tooltip>
<Tooltip
content={connection?.enabled !== false ? $i18n.t('Enabled') : $i18n.t('Disabled')}
>
<Switch
state={connection?.enabled !== false}
on:change={() => {
toggleSource(item);
}}
/>
</Tooltip>
</div>
</div>
{/each}
</div>
{#if loading}
<div class="py-2">
<Spinner />
</div>
{:else if items.length === 0}
<div class="text-xs text-gray-400 dark:text-gray-500">
{$i18n.t('No external knowledge sources configured.')}
</div>
{/if}
<div class="my-1.5">
<div class="text-xs text-gray-500">
{$i18n.t(
'Create one read-only Knowledge source per external collection. Test must pass before the source is created.'
)}
</div>
</div>
</div>
@@ -1,11 +1,12 @@
<script lang="ts"> <script lang="ts">
import { toast } from 'svelte-sonner'; import { toast } from 'svelte-sonner';
import { createEventDispatcher, onMount, getContext, tick } from 'svelte'; import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
import { v4 as uuidv4 } from 'uuid';
import { getModels as _getModels } from '$lib/apis'; import { getModels as _getModels } from '$lib/apis';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const dispatch = createEventDispatcher(); const dispatch = createEventDispatcher();
const i18n = getContext('i18n'); const i18n = getContext<Writable<i18nType>>('i18n');
import { models, settings, user, terminalServers } from '$lib/stores'; import { models, settings, user, terminalServers } from '$lib/stores';
import { getTerminalServers } from '$lib/apis/terminal'; import { getTerminalServers } from '$lib/apis/terminal';
@@ -22,6 +23,7 @@
import AddToolServerModal from '$lib/components/AddToolServerModal.svelte'; import AddToolServerModal from '$lib/components/AddToolServerModal.svelte';
import AddTerminalServerModal from '$lib/components/AddTerminalServerModal.svelte'; import AddTerminalServerModal from '$lib/components/AddTerminalServerModal.svelte';
import ExternalKnowledge from './ExternalKnowledge.svelte';
import { import {
getToolServerConnections, getToolServerConnections,
@@ -32,16 +34,26 @@
export let saveSettings: Function; export let saveSettings: Function;
let servers = null; type ToolServerConnection = any;
type TerminalConnection = {
id?: string;
url?: string;
name?: string;
key?: string;
enabled?: boolean;
[key: string]: any;
};
let servers: ToolServerConnection[] | null = null;
let showConnectionModal = false; let showConnectionModal = false;
// Terminal server admin connections // Terminal server admin connections
let terminalConnections = []; let terminalConnections: TerminalConnection[] = [];
let showAddTerminalModal = false; let showAddTerminalModal = false;
let editTerminalIdx: number | null = null; let editTerminalIdx: number | null = null;
const addConnectionHandler = async (server) => { const addConnectionHandler = async (server: ToolServerConnection) => {
servers = [...servers, server]; servers = [...(servers ?? []), server];
await updateHandler(); await updateHandler();
}; };
@@ -71,7 +83,9 @@
// Refresh the terminalServers store so changes are reflected immediately // Refresh the terminalServers store so changes are reflected immediately
// Preserve user direct terminals, refresh system terminals from backend // Preserve user direct terminals, refresh system terminals from backend
const existingDirectTerminals = ($terminalServers ?? []).filter((t) => !t.id); const existingDirectTerminals = (($terminalServers ?? []) as TerminalConnection[]).filter(
(t) => !t.id
);
const systemTerminals = await getTerminalServers(localStorage.token); const systemTerminals = await getTerminalServers(localStorage.token);
const systemEntries = systemTerminals.map((t) => ({ const systemEntries = systemTerminals.map((t) => ({
id: t.id, id: t.id,
@@ -79,16 +93,19 @@
name: t.name, name: t.name,
key: localStorage.token key: localStorage.token
})); }));
terminalServers.set([...existingDirectTerminals, ...systemEntries]); terminalServers.set([...existingDirectTerminals, ...systemEntries] as any);
} }
}; };
const addTerminalConnection = (server) => { const addTerminalConnection = (server: TerminalConnection) => {
terminalConnections = [...terminalConnections, { ...server, id: server.id ?? uuidv4() }]; terminalConnections = [
...terminalConnections,
{ ...server, id: server.id ?? crypto.randomUUID() }
];
saveTerminalServers(); saveTerminalServers();
}; };
const updateTerminalConnection = (idx: number, updated) => { const updateTerminalConnection = (idx: number, updated: TerminalConnection) => {
terminalConnections = terminalConnections.map((c, i) => terminalConnections = terminalConnections.map((c, i) =>
i === idx ? { ...c, ...updated, id: updated.id ?? c.id } : c i === idx ? { ...c, ...updated, id: updated.id ?? c.id } : c
); );
@@ -102,13 +119,13 @@
onMount(async () => { onMount(async () => {
const res = await getToolServerConnections(localStorage.token); const res = await getToolServerConnections(localStorage.token);
servers = res.TOOL_SERVER_CONNECTIONS; servers = res.TOOL_SERVER_CONNECTIONS as ToolServerConnection[];
// Load terminal server connections // Load terminal server connections
try { try {
const terminalRes = await getTerminalServerConnections(localStorage.token); const terminalRes = await getTerminalServerConnections(localStorage.token);
if (terminalRes?.TERMINAL_SERVER_CONNECTIONS) { if (terminalRes?.TERMINAL_SERVER_CONNECTIONS) {
terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS; terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS as TerminalConnection[];
} }
} catch { } catch {
// Not configured yet // Not configured yet
@@ -122,7 +139,7 @@
bind:show={showAddTerminalModal} bind:show={showAddTerminalModal}
edit={editTerminalIdx !== null} edit={editTerminalIdx !== null}
connection={editTerminalIdx !== null ? terminalConnections[editTerminalIdx] : null} connection={editTerminalIdx !== null ? terminalConnections[editTerminalIdx] : null}
onSubmit={(c) => { onSubmit={(c: TerminalConnection) => {
if (editTerminalIdx !== null) { if (editTerminalIdx !== null) {
updateTerminalConnection(editTerminalIdx, c); updateTerminalConnection(editTerminalIdx, c);
editTerminalIdx = null; editTerminalIdx = null;
@@ -148,13 +165,13 @@
{#if servers !== null} {#if servers !== null}
<div class=""> <div class="">
<div class="mb-3"> <div class="mb-3">
<div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('General')}</div> <div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('Tools')}</div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" /> <hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
<div class="mb-2.5 flex flex-col w-full justify-between"> <div class="mb-2.5 flex flex-col w-full justify-between">
<div class="flex justify-between items-center mb-0.5"> <div class="flex justify-between items-center mb-0.5">
<div class="font-medium">{$i18n.t('Manage Tool Servers')}</div> <div class="font-medium">{$i18n.t('External Tool Servers')}</div>
<Tooltip content={$i18n.t(`Add Connection`)}> <Tooltip content={$i18n.t(`Add Connection`)}>
<button <button
@@ -170,21 +187,21 @@
</div> </div>
<div class="flex flex-col gap-1"> <div class="flex flex-col gap-1">
{#each servers as server, idx} {#each servers ?? [] as server, idx}
<Connection <Connection
bind:connection={server} bind:connection={server}
onSubmit={() => { onSubmit={() => {
updateHandler(); updateHandler();
}} }}
onDelete={() => { onDelete={() => {
servers = servers.filter((_, i) => i !== idx); servers = (servers ?? []).filter((_, i) => i !== idx);
updateHandler(); updateHandler();
}} }}
/> />
{/each} {/each}
</div> </div>
{#if servers.length === 0} {#if (servers ?? []).length === 0}
<div class="text-xs text-gray-400 dark:text-gray-500"> <div class="text-xs text-gray-400 dark:text-gray-500">
{$i18n.t('No tool server connections configured.')} {$i18n.t('No tool server connections configured.')}
</div> </div>
@@ -197,17 +214,9 @@
</div> </div>
</div> </div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-4" /> <div class="mt-4 mb-2.5 flex flex-col w-full">
<div class="mb-2.5 flex flex-col w-full">
<div class="flex justify-between items-center mb-1"> <div class="flex justify-between items-center mb-1">
<div class="flex items-center gap-2"> <div class="font-medium">{$i18n.t('Open Terminal')}</div>
<div class="font-medium">{$i18n.t('Open Terminal')}</div>
<span
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
>{$i18n.t('Experimental')}</span
>
</div>
<Tooltip content={$i18n.t('Add Connection')}> <Tooltip content={$i18n.t('Add Connection')}>
<button <button
@@ -300,6 +309,12 @@
</div> </div>
</div> </div>
</div> </div>
<div class="mt-8 mb-2.5 text-base font-medium">{$i18n.t('Knowledge')}</div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
<ExternalKnowledge />
</div> </div>
</div> </div>
{:else} {:else}
+67 -18
View File
@@ -5,7 +5,10 @@
import { toast } from 'svelte-sonner'; import { toast } from 'svelte-sonner';
import { onMount, getContext, tick, onDestroy } from 'svelte'; import { onMount, getContext, tick, onDestroy } from 'svelte';
const i18n = getContext('i18n'); import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const i18n = getContext<Writable<i18nType>>('i18n');
import { WEBUI_NAME, knowledge, user } from '$lib/stores'; import { WEBUI_NAME, knowledge, user } from '$lib/stores';
import { import {
@@ -28,19 +31,33 @@
import ViewSelector from './common/ViewSelector.svelte'; import ViewSelector from './common/ViewSelector.svelte';
import Loader from '../common/Loader.svelte'; import Loader from '../common/Loader.svelte';
type KnowledgeListItem = {
id: string;
name: string;
description?: string;
updated_at: number;
write_access?: boolean;
meta?: any;
user?: {
name?: string;
email?: string;
};
};
let loaded = false; let loaded = false;
let showDeleteConfirm = false; let showDeleteConfirm = false;
let tagsContainerElement: HTMLDivElement; let tagsContainerElement: HTMLDivElement;
let selectedItem = null; let selectedItem: KnowledgeListItem | null = null;
let page = 1; let page = 1;
let query = ''; let query = '';
let searchDebounceTimer: ReturnType<typeof setTimeout>; let searchDebounceTimer: ReturnType<typeof setTimeout>;
let viewOption = ''; let viewOption = '';
let sourceOption = '';
let items = null; let items: KnowledgeListItem[] | null = null;
let total = null; let total: number | null = null;
let allItemsLoaded = false; let allItemsLoaded = false;
let itemsLoading = false; let itemsLoading = false;
@@ -56,7 +73,7 @@
clearTimeout(searchDebounceTimer); clearTimeout(searchDebounceTimer);
}); });
$: if (loaded && viewOption !== undefined) { $: if (loaded && viewOption !== undefined && sourceOption !== undefined) {
init(); init();
} }
@@ -83,16 +100,20 @@
const getItemsPage = async () => { const getItemsPage = async () => {
itemsLoading = true; itemsLoading = true;
const res = await searchKnowledgeBases(localStorage.token, query, viewOption, page).catch( const res = await searchKnowledgeBases(
() => { localStorage.token,
return []; query,
} viewOption,
); page,
sourceOption
).catch(() => {
return [];
});
if (res) { if (res) {
console.log(res); console.log(res);
total = res.total; total = res.total;
const pageItems = res.items; const pageItems: KnowledgeListItem[] = res.items ?? [];
if ((pageItems ?? []).length === 0) { if ((pageItems ?? []).length === 0) {
allItemsLoaded = true; allItemsLoaded = true;
@@ -113,7 +134,9 @@
return res; return res;
}; };
const deleteHandler = async (item) => { const deleteHandler = async (item: KnowledgeListItem | null) => {
if (!item) return;
const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => { const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => {
toast.error(`${e}`); toast.error(`${e}`);
}); });
@@ -124,7 +147,7 @@
} }
}; };
const exportHandler = async (item) => { const exportHandler = async (item: KnowledgeListItem) => {
try { try {
const blob = await exportKnowledgeById(localStorage.token, item.id); const blob = await exportKnowledgeById(localStorage.token, item.id);
if (blob) { if (blob) {
@@ -145,6 +168,7 @@
onMount(async () => { onMount(async () => {
viewOption = localStorage?.workspaceViewOption || ''; viewOption = localStorage?.workspaceViewOption || '';
sourceOption = localStorage?.workspaceKnowledgeSourceOption || '';
loaded = true; loaded = true;
}); });
</script> </script>
@@ -241,6 +265,19 @@
await tick(); await tick();
}} }}
/> />
<select
class="relative w-full flex items-center gap-0.5 px-2.5 py-1.5 bg-gray-50 dark:bg-gray-850 rounded-xl outline-hidden"
bind:value={sourceOption}
on:change={async () => {
localStorage.workspaceKnowledgeSourceOption = sourceOption;
await tick();
}}
>
<option value="">{$i18n.t('All Sources')}</option>
<option value="local">{$i18n.t('Local')}</option>
<option value="external">{$i18n.t('Connected')}</option>
</select>
</div> </div>
</div> </div>
@@ -267,11 +304,23 @@
<div class=" self-center flex-1 justify-between"> <div class=" self-center flex-1 justify-between">
<div class="flex items-center justify-between -my-1 h-8"> <div class="flex items-center justify-between -my-1 h-8">
<div class=" flex gap-2 items-center justify-between w-full"> <div class=" flex gap-2 items-center justify-between w-full">
<div> {#if item?.meta?.source === 'external'}
<Badge type="success" content={$i18n.t('Collection')} /> <div>
</div> <Badge
type="muted"
content={item?.meta?.external?.provider ?? $i18n.t('Connected')}
/>
</div>
<div>
<Badge type="muted" content={$i18n.t('Read Only')} />
</div>
{:else}
<div>
<Badge type="success" content={$i18n.t('Collection')} />
</div>
{/if}
{#if !item?.write_access} {#if !item?.write_access && item?.meta?.source !== 'external'}
<div> <div>
<Badge type="muted" content={$i18n.t('Read Only')} /> <Badge type="muted" content={$i18n.t('Read Only')} />
</div> </div>
@@ -282,7 +331,7 @@
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
<div class=" flex self-center"> <div class=" flex self-center">
<ItemMenu <ItemMenu
onExport={$user.role === 'admin' onExport={$user?.role === 'admin'
? () => { ? () => {
exportHandler(item); exportHandler(item);
} }
@@ -5,7 +5,10 @@
import { PaneGroup, Pane, PaneResizer } from 'paneforge'; import { PaneGroup, Pane, PaneResizer } from 'paneforge';
import { onMount, getContext, onDestroy, tick } from 'svelte'; import { onMount, getContext, onDestroy, tick } from 'svelte';
const i18n = getContext('i18n'); import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const i18n = getContext<Writable<i18nType>>('i18n');
import { goto } from '$app/navigation'; import { goto } from '$app/navigation';
import { page } from '$app/stores'; import { page } from '$app/stores';
@@ -40,7 +43,8 @@
deleteKnowledgeDirectory, deleteKnowledgeDirectory,
moveFileInKnowledge, moveFileInKnowledge,
syncKnowledgeDiff, syncKnowledgeDiff,
syncKnowledgeCleanup syncKnowledgeCleanup,
testExternalKnowledgeRetrieval
} from '$lib/apis/knowledge'; } from '$lib/apis/knowledge';
import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval'; import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
@@ -98,11 +102,13 @@
files: any[]; files: any[];
access_grants?: any[]; access_grants?: any[];
write_access?: boolean; write_access?: boolean;
meta?: any;
}; };
let id = null; let id = null;
let knowledge: Knowledge | null = null; let knowledge: Knowledge | null = null;
let knowledgeId = null; let knowledgeId = null;
let isExternalKnowledge = false;
let selectedFileId = null; let selectedFileId = null;
let selectedFile = null; let selectedFile = null;
@@ -132,6 +138,14 @@
let deleteDirectoryContents = true; let deleteDirectoryContents = true;
let pendingPollTimer: ReturnType<typeof setInterval> | null = null; let pendingPollTimer: ReturnType<typeof setInterval> | null = null;
let externalTestQuery = '';
let externalTestResult: {
documents?: string[];
metadatas?: Record<string, any>[];
distances?: number[];
} | null = null;
$: isExternalKnowledge = knowledge?.meta?.source === 'external';
const reset = () => { const reset = () => {
currentPage = 1; currentPage = 1;
@@ -243,6 +257,24 @@
} }
}; };
const externalTestHandler = async () => {
if (!isExternalKnowledge || !externalTestQuery.trim()) return;
const external = knowledge?.meta?.external ?? {};
const res = await testExternalKnowledgeRetrieval(localStorage.token, external.connection_id, {
query: externalTestQuery,
source: external.source,
count: 5
}).catch((e) => {
toast.error(`${e}`);
return null;
});
if (res) {
externalTestResult = res;
}
};
const createFileFromText = (name, content) => { const createFileFromText = (name, content) => {
const blob = new Blob([content], { type: 'text/plain' }); const blob = new Blob([content], { type: 'text/plain' });
const file = blobToFile(blob, `${name}.txt`); const file = blobToFile(blob, `${name}.txt`);
@@ -1299,284 +1331,357 @@
<div <div
class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1" class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1"
> >
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2"> {#if isExternalKnowledge}
<div class="flex flex-1 items-center"> <div class="p-5 flex flex-col gap-4">
<div class=" self-center ml-1 mr-3"> <div class="flex flex-wrap gap-2 text-xs">
<Search className="size-3.5" /> <div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
</div> {$i18n.t('Connected')}
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
bind:value={query}
on:input={handleSearchInput}
aria-label={$i18n.t('Search Collection')}
placeholder={$i18n.t('Search Collection')}
on:focus={() => {
selectedFileId = null;
}}
/>
<Dropdown align="end">
<button
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
type="button"
>
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
</button>
<div slot="content">
<div
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
>
<button
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
type="button"
on:click={() => {
includeContent = !includeContent;
}}
>
<Checkbox
state={includeContent ? 'checked' : 'unchecked'}
on:change={(e) => {
includeContent = e.detail === 'checked';
}}
/>
{$i18n.t('File content')}
</button>
</div>
</div> </div>
</Dropdown> <div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{$i18n.t('Read Only')}
</div>
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{knowledge?.meta?.external?.provider ?? $i18n.t('Provider')}
</div>
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{$i18n.t('Service Account')}
</div>
</div>
{#if knowledge?.write_access} <div class="grid grid-cols-1 md:grid-cols-2 gap-3 text-sm">
<div> <div>
<AddContentMenu <div class="text-xs text-gray-500 mb-1">{$i18n.t('Mapped Source')}</div>
onUpload={(data) => { <div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
if (data.type === 'directory') { {knowledge?.meta?.external?.source?.name ?? $i18n.t('Not configured')}
uploadDirectoryHandler(); </div>
} else if (data.type === 'new_directory') {
showNewDirectoryModal = true;
} else if (data.type === 'web') {
showAddWebpageModal = true;
} else if (data.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
onSync={async () => {
pendingSyncFiles = await collectDirectoryFiles();
if (pendingSyncFiles?.length) {
showSyncConfirmModal = true;
}
}}
onReset={() => {
showResetConfirm = true;
}}
/>
</div> </div>
{/if} <div>
</div> <div class="text-xs text-gray-500 mb-1">{$i18n.t('Auth Mode')}</div>
</div> <div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
{$i18n.t('Admin-managed service account')}
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
on:wheel={(e) => {
if (e.deltaY !== 0) {
e.preventDefault();
e.currentTarget.scrollLeft += e.deltaY;
}
}}
>
<div
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
>
<DropdownOptions
align="start"
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
bind:value={viewOption}
items={[
{ value: null, label: $i18n.t('All') },
{ value: 'created', label: $i18n.t('Created by you') },
{ value: 'shared', label: $i18n.t('Shared with you') }
]}
onChange={(value) => {
if (value) {
localStorage.workspaceViewOption = value;
} else {
delete localStorage.workspaceViewOption;
}
}}
/>
<DropdownOptions
align="start"
bind:value={sortKey}
placeholder={$i18n.t('Sort')}
items={[
{ value: 'name', label: $i18n.t('Name') },
{ value: 'created_at', label: $i18n.t('Created At') },
{ value: 'updated_at', label: $i18n.t('Updated At') }
]}
/>
{#if sortKey}
<DropdownOptions
align="start"
bind:value={direction}
items={[
{ value: 'asc', label: $i18n.t('Asc') },
{ value: null, label: $i18n.t('Desc') }
]}
/>
{/if}
</div>
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 mt-2">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
{#if syncing}
<div class="mx-2.5 mt-2.5 -mb-0.5">
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
<Spinner className="size-3.5 shrink-0" />
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
{syncing}
</div>
</div>
</div>
{/if}
{#if fileItems !== null && fileItemsTotal !== null}
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
<div class="flex-1 flex">
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
<div class="w-full h-full flex flex-col min-h-full">
{#if fileItems.length > 0 || directoryItems.length > 0}
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
<Files
files={fileItems}
directories={directoryItems}
{knowledge}
{selectedFileId}
onClick={(fileId) => {
selectedFileId = fileId;
if (fileItems) {
const file = fileItems.find((file) => file.id === selectedFileId);
if (file) {
fileSelectHandler(file);
} else {
selectedFile = null;
}
}
}}
onDelete={(fileId) => {
selectedFileId = null;
selectedFile = null;
deleteFileHandler(fileId);
}}
onRename={(fileId, name) => renameFileHandler(fileId, name)}
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
onMoveFileToDirectory={(fileId, dirId) =>
moveFileToDirectoryHandler(fileId, dirId)}
onMoveDirectoryToDirectory={(dirId, targetId) =>
moveDirectoryHandler(dirId, targetId)}
/>
</div>
{#if fileItemsTotal > 30}
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
{/if}
{:else}
<div class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs">
<div>
{$i18n.t('No content found')}
</div>
</div>
{/if}
</div> </div>
</div> </div>
</div> </div>
{#if selectedFileId !== null} <div class="text-xs text-gray-500">
<Drawer {$i18n.t(
className="h-full" 'This knowledge base retrieves from a connected source. Open WebUI can query it, but cannot upload, sync, edit, delete, reset, or reindex its source data.'
show={selectedFileId !== null} )}
onClose={() => { </div>
selectedFileId = null;
selectedFile = null;
}}
>
<div class="flex flex-col justify-start h-full max-h-full">
<div class=" flex flex-col w-full h-full max-h-full">
<div class="shrink-0 flex items-center p-2">
<div class="mr-2">
<button
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
aria-label={$i18n.t('Close')}
on:click={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<ChevronLeft strokeWidth="2.5" />
</button>
</div>
<div class=" flex-1 text-lg line-clamp-1">
{selectedFile?.meta?.name}
</div>
{#if knowledge?.write_access} <div class="flex flex-col gap-2">
<div> <div class="font-medium text-sm">{$i18n.t('Test Query')}</div>
<button <div class="flex gap-2">
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed" <input
disabled={isSaving} class="w-full text-sm rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2 outline-hidden"
on:click={() => { bind:value={externalTestQuery}
updateFileContentHandler(); placeholder={$i18n.t('Ask this knowledge source a test question')}
}} />
> <button
{$i18n.t('Save')} class="px-3 py-2 rounded-xl bg-black text-white dark:bg-white dark:text-black text-sm"
{#if isSaving} on:click={externalTestHandler}
<div class="ml-2 self-center"> >
<Spinner /> {$i18n.t('Test')}
</div> </button>
{/if} </div>
</button> </div>
</div>
{/if} {#if externalTestResult}
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 p-3 text-xs">
<div class="font-medium mb-2">{$i18n.t('Preview')}</div>
{#each externalTestResult.documents ?? [] as document, idx}
<div class="border-t border-gray-100 dark:border-gray-800 py-2">
<div class="line-clamp-4">{document}</div>
<div class="text-gray-500 mt-1">
{externalTestResult.metadatas?.[idx]?.source ?? ''}
</div> </div>
{#key selectedFile.id}
<textarea
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
bind:value={selectedFileContent}
disabled={!knowledge?.write_access}
aria-label={$i18n.t('File content')}
placeholder={$i18n.t('Add content here')}
/>
{/key}
</div> </div>
</div> {/each}
</Drawer> </div>
{/if} {/if}
</div> </div>
{:else} {:else}
<div class="my-10"> <div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
<Spinner className="size-4" /> <div class="flex flex-1 items-center">
<div class=" self-center ml-1 mr-3">
<Search className="size-3.5" />
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
bind:value={query}
on:input={handleSearchInput}
aria-label={$i18n.t('Search Collection')}
placeholder={$i18n.t('Search Collection')}
on:focus={() => {
selectedFileId = null;
}}
/>
<Dropdown align="end">
<button
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
type="button"
>
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
</button>
<div slot="content">
<div
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
>
<button
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
type="button"
on:click={() => {
includeContent = !includeContent;
}}
>
<Checkbox
state={includeContent ? 'checked' : 'unchecked'}
on:change={(e) => {
includeContent = e.detail === 'checked';
}}
/>
{$i18n.t('File content')}
</button>
</div>
</div>
</Dropdown>
{#if knowledge?.write_access}
<div>
<AddContentMenu
onUpload={(data) => {
if (data.type === 'directory') {
uploadDirectoryHandler();
} else if (data.type === 'new_directory') {
showNewDirectoryModal = true;
} else if (data.type === 'web') {
showAddWebpageModal = true;
} else if (data.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
onSync={async () => {
pendingSyncFiles = await collectDirectoryFiles();
if (pendingSyncFiles?.length) {
showSyncConfirmModal = true;
}
}}
onReset={() => {
showResetConfirm = true;
}}
/>
</div>
{/if}
</div>
</div> </div>
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
on:wheel={(e) => {
if (e.deltaY !== 0) {
e.preventDefault();
e.currentTarget.scrollLeft += e.deltaY;
}
}}
>
<div
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
>
<DropdownOptions
align="start"
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
bind:value={viewOption}
items={[
{ value: null, label: $i18n.t('All') },
{ value: 'created', label: $i18n.t('Created by you') },
{ value: 'shared', label: $i18n.t('Shared with you') }
]}
onChange={(value) => {
if (value) {
localStorage.workspaceViewOption = value;
} else {
delete localStorage.workspaceViewOption;
}
}}
/>
<DropdownOptions
align="start"
bind:value={sortKey}
placeholder={$i18n.t('Sort')}
items={[
{ value: 'name', label: $i18n.t('Name') },
{ value: 'created_at', label: $i18n.t('Created At') },
{ value: 'updated_at', label: $i18n.t('Updated At') }
]}
/>
{#if sortKey}
<DropdownOptions
align="start"
bind:value={direction}
items={[
{ value: 'asc', label: $i18n.t('Asc') },
{ value: null, label: $i18n.t('Desc') }
]}
/>
{/if}
</div>
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 mt-2">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
{#if syncing}
<div class="mx-2.5 mt-2.5 -mb-0.5">
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
<Spinner className="size-3.5 shrink-0" />
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
{syncing}
</div>
</div>
</div>
{/if}
{#if fileItems !== null && fileItemsTotal !== null}
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
<div class="flex-1 flex">
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
<div class="w-full h-full flex flex-col min-h-full">
{#if fileItems.length > 0 || directoryItems.length > 0}
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
<Files
files={fileItems}
directories={directoryItems}
{knowledge}
{selectedFileId}
onClick={(fileId) => {
selectedFileId = fileId;
if (fileItems) {
const file = fileItems.find((file) => file.id === selectedFileId);
if (file) {
fileSelectHandler(file);
} else {
selectedFile = null;
}
}
}}
onDelete={(fileId) => {
selectedFileId = null;
selectedFile = null;
deleteFileHandler(fileId);
}}
onRename={(fileId, name) => renameFileHandler(fileId, name)}
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
onMoveFileToDirectory={(fileId, dirId) =>
moveFileToDirectoryHandler(fileId, dirId)}
onMoveDirectoryToDirectory={(dirId, targetId) =>
moveDirectoryHandler(dirId, targetId)}
/>
</div>
{#if fileItemsTotal > 30}
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
{/if}
{:else}
<div
class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs"
>
<div>
{$i18n.t('No content found')}
</div>
</div>
{/if}
</div>
</div>
</div>
{#if selectedFileId !== null}
<Drawer
className="h-full"
show={selectedFileId !== null}
onClose={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<div class="flex flex-col justify-start h-full max-h-full">
<div class=" flex flex-col w-full h-full max-h-full">
<div class="shrink-0 flex items-center p-2">
<div class="mr-2">
<button
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
aria-label={$i18n.t('Close')}
on:click={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<ChevronLeft strokeWidth="2.5" />
</button>
</div>
<div class=" flex-1 text-lg line-clamp-1">
{selectedFile?.meta?.name}
</div>
{#if knowledge?.write_access}
<div>
<button
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isSaving}
on:click={() => {
updateFileContentHandler();
}}
>
{$i18n.t('Save')}
{#if isSaving}
<div class="ml-2 self-center">
<Spinner />
</div>
{/if}
</button>
</div>
{/if}
</div>
{#key selectedFile.id}
<textarea
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
bind:value={selectedFileContent}
disabled={!knowledge?.write_access}
aria-label={$i18n.t('File content')}
placeholder={$i18n.t('Add content here')}
/>
{/key}
</div>
</div>
</Drawer>
{/if}
</div>
{:else}
<div class="my-10">
<Spinner className="size-4" />
</div>
{/if}
{/if} {/if}
</div> </div>
{:else} {:else}