refac
This commit is contained in:
1 parent
b1c2536ed2
commit
15c7e37438
15 files changed
+2929
-394
No files matched your search
@@ -857,6 +857,16 @@ EXTERNAL_DOCUMENT_LOADER_URL = os.getenv('EXTERNAL_DOCUMENT_LOADER_URL', '')
|
|||||||
|
|
||||||
EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '')
|
EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '')
|
||||||
|
|
||||||
|
external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '')
|
||||||
|
try:
|
||||||
|
external_document_loader_headers = json.loads(external_document_loader_headers)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
external_document_loader_headers = {}
|
||||||
|
if not isinstance(external_document_loader_headers, dict):
|
||||||
|
external_document_loader_headers = {}
|
||||||
|
|
||||||
|
EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
|
||||||
|
|
||||||
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
|
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
|
||||||
|
|
||||||
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
|
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
|
||||||
@@ -2652,6 +2662,7 @@ DEFAULT_CONFIG = {
|
|||||||
'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS,
|
'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS,
|
||||||
'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL,
|
'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL,
|
||||||
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
|
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'rag.tika_server_url': TIKA_SERVER_URL,
|
'rag.tika_server_url': TIKA_SERVER_URL,
|
||||||
'rag.docling_server_url': DOCLING_SERVER_URL,
|
'rag.docling_server_url': DOCLING_SERVER_URL,
|
||||||
'rag.docling_api_key': DOCLING_API_KEY,
|
'rag.docling_api_key': DOCLING_API_KEY,
|
||||||
|
|||||||
@@ -286,6 +286,17 @@ class KnowledgeTable:
|
|||||||
elif view_option == 'shared':
|
elif view_option == 'shared':
|
||||||
stmt = stmt.filter(Knowledge.user_id != user_id)
|
stmt = stmt.filter(Knowledge.user_id != user_id)
|
||||||
|
|
||||||
|
source = filter.get('source')
|
||||||
|
if source == 'external':
|
||||||
|
stmt = stmt.filter(Knowledge.meta['source'].as_string() == 'external')
|
||||||
|
elif source == 'local':
|
||||||
|
stmt = stmt.filter(
|
||||||
|
or_(
|
||||||
|
Knowledge.meta.is_(None),
|
||||||
|
Knowledge.meta['source'].as_string() != 'external',
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
stmt = AccessGrants.has_permission_filter(
|
stmt = AccessGrants.has_permission_filter(
|
||||||
db=db,
|
db=db,
|
||||||
query=stmt,
|
query=stmt,
|
||||||
@@ -765,6 +776,25 @@ class KnowledgeTable:
|
|||||||
log.exception(e)
|
log.exception(e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def update_knowledge_meta_by_id(
|
||||||
|
self, id: str, meta: dict, db: Optional[AsyncSession] = None
|
||||||
|
) -> Optional[KnowledgeModel]:
|
||||||
|
try:
|
||||||
|
async with get_async_db_context(db) as db:
|
||||||
|
await db.execute(
|
||||||
|
update(Knowledge)
|
||||||
|
.filter_by(id=id)
|
||||||
|
.values(
|
||||||
|
meta=meta,
|
||||||
|
updated_at=int(time.time()),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return await self.get_knowledge_by_id(id=id, db=db)
|
||||||
|
except Exception as e:
|
||||||
|
log.exception(e)
|
||||||
|
return None
|
||||||
|
|
||||||
async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
|
async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
|
||||||
try:
|
try:
|
||||||
async with get_async_db_context(db) as db:
|
async with get_async_db_context(db) as db:
|
||||||
|
|||||||
@@ -0,0 +1,378 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from open_webui.models.config import Config
|
||||||
|
from open_webui.models.knowledge import KnowledgeModel
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
|
||||||
|
IDENTIFIER_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_external_connection(connection_id: str) -> Optional[dict]:
|
||||||
|
connections = await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
|
||||||
|
return next((connection for connection in connections if connection.get('id') == connection_id), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_path(data: Any, path: Optional[str], default=None):
|
||||||
|
if not path:
|
||||||
|
return default
|
||||||
|
value = data
|
||||||
|
for part in path.split('.'):
|
||||||
|
if isinstance(value, dict):
|
||||||
|
value = value.get(part, default)
|
||||||
|
else:
|
||||||
|
return default
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_result(result: dict, mapping: dict, knowledge: KnowledgeModel, distance: Optional[float] = None) -> dict:
|
||||||
|
content = _get_path(result, mapping.get('content_field', 'content'), '')
|
||||||
|
title = _get_path(result, mapping.get('title_field', 'title'), None)
|
||||||
|
source = _get_path(result, mapping.get('source_field', 'source'), None)
|
||||||
|
url = _get_path(result, mapping.get('url_field', 'url'), None)
|
||||||
|
document_id = _get_path(result, mapping.get('document_id_field', 'document_id'), None)
|
||||||
|
page = _get_path(result, mapping.get('page_field', 'page'), None)
|
||||||
|
metadata = _get_path(result, mapping.get('metadata_field', 'metadata'), {}) or {}
|
||||||
|
score = _get_path(result, mapping.get('score_field', 'score'), distance)
|
||||||
|
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
metadata = {'external_metadata': metadata}
|
||||||
|
|
||||||
|
source_name = source or title or metadata.get('source') or metadata.get('name') or knowledge.name
|
||||||
|
metadata.update(
|
||||||
|
{
|
||||||
|
'name': title or source_name,
|
||||||
|
'source': source_name,
|
||||||
|
'url': url,
|
||||||
|
'file_id': document_id or f'external-{knowledge.id}',
|
||||||
|
'knowledge_id': knowledge.id,
|
||||||
|
'knowledge_name': knowledge.name,
|
||||||
|
'external': True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if page is not None:
|
||||||
|
metadata['page'] = page
|
||||||
|
if document_id is not None:
|
||||||
|
metadata['document_id'] = document_id
|
||||||
|
|
||||||
|
return {
|
||||||
|
'content': content,
|
||||||
|
'metadata': metadata,
|
||||||
|
'distance': score,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _source_config(knowledge: KnowledgeModel) -> dict:
|
||||||
|
external = (knowledge.meta or {}).get('external', {})
|
||||||
|
source = external.get('source') or {}
|
||||||
|
return source.get('config') or {}
|
||||||
|
|
||||||
|
|
||||||
|
def _root_field(path: Optional[str]) -> Optional[str]:
|
||||||
|
if not path:
|
||||||
|
return None
|
||||||
|
return path.split('.')[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_identifier(value: str, label: str) -> str:
|
||||||
|
if not value or not IDENTIFIER_RE.match(value):
|
||||||
|
raise RuntimeError(f'Invalid {label}')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
async def _retrieve_qdrant(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
|
||||||
|
try:
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
except ImportError as exc:
|
||||||
|
raise RuntimeError('qdrant-client is not installed') from exc
|
||||||
|
|
||||||
|
if not embedding_function:
|
||||||
|
raise RuntimeError('Embedding function is not configured')
|
||||||
|
|
||||||
|
config = connection.get('config') or {}
|
||||||
|
external = (knowledge.meta or {}).get('external', {})
|
||||||
|
source = external.get('source') or {}
|
||||||
|
collection_name = source.get('name')
|
||||||
|
if not collection_name:
|
||||||
|
raise RuntimeError('External source collection is not configured')
|
||||||
|
source_config = _source_config(knowledge)
|
||||||
|
vector_field = source_config.get('vector_field') or None
|
||||||
|
|
||||||
|
vector = await embedding_function(query)
|
||||||
|
|
||||||
|
def _search():
|
||||||
|
client = QdrantClient(
|
||||||
|
url=connection.get('endpoint'),
|
||||||
|
api_key=(auth_config or {}).get('api_key'),
|
||||||
|
timeout=config.get('timeout') or 30,
|
||||||
|
)
|
||||||
|
return client.query_points(
|
||||||
|
collection_name=collection_name,
|
||||||
|
query=vector,
|
||||||
|
using=vector_field,
|
||||||
|
limit=count,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await asyncio.to_thread(_search)
|
||||||
|
mapping = {
|
||||||
|
'content_field': source_config.get('content_field') or 'payload.text',
|
||||||
|
'metadata_field': source_config.get('metadata_field') or 'payload.metadata',
|
||||||
|
'document_id_field': source_config.get('document_id_field') or 'id',
|
||||||
|
'score_field': 'score',
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized = []
|
||||||
|
for point in response.points:
|
||||||
|
normalized.append(_normalize_result(point.model_dump(), mapping, knowledge, distance=point.score))
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
async def _retrieve_milvus(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
|
||||||
|
try:
|
||||||
|
from pymilvus import MilvusClient
|
||||||
|
except ImportError as exc:
|
||||||
|
raise RuntimeError('pymilvus is not installed') from exc
|
||||||
|
|
||||||
|
if not embedding_function:
|
||||||
|
raise RuntimeError('Embedding function is not configured')
|
||||||
|
|
||||||
|
config = connection.get('config') or {}
|
||||||
|
external = (knowledge.meta or {}).get('external', {})
|
||||||
|
source = external.get('source') or {}
|
||||||
|
collection_name = source.get('name')
|
||||||
|
if not collection_name:
|
||||||
|
raise RuntimeError('Milvus collection is not configured')
|
||||||
|
source_config = _source_config(knowledge)
|
||||||
|
vector_field = source_config.get('vector_field') or 'vector'
|
||||||
|
content_field = source_config.get('content_field') or 'data.text'
|
||||||
|
metadata_field = source_config.get('metadata_field') or 'metadata'
|
||||||
|
|
||||||
|
vector = await embedding_function(query)
|
||||||
|
|
||||||
|
def _search():
|
||||||
|
client_kwargs = {
|
||||||
|
'uri': connection.get('endpoint'),
|
||||||
|
}
|
||||||
|
token = (auth_config or {}).get('api_key') or (auth_config or {}).get('token')
|
||||||
|
if token:
|
||||||
|
client_kwargs['token'] = token
|
||||||
|
if config.get('db_name'):
|
||||||
|
client_kwargs['db_name'] = config.get('db_name')
|
||||||
|
|
||||||
|
client = MilvusClient(**client_kwargs)
|
||||||
|
output_fields = {
|
||||||
|
field
|
||||||
|
for field in (
|
||||||
|
_root_field(content_field),
|
||||||
|
_root_field(metadata_field),
|
||||||
|
_root_field(source_config.get('document_id_field')),
|
||||||
|
)
|
||||||
|
if field and field != vector_field
|
||||||
|
}
|
||||||
|
kwargs = {
|
||||||
|
'collection_name': collection_name,
|
||||||
|
'data': [vector],
|
||||||
|
'anns_field': vector_field,
|
||||||
|
'limit': count,
|
||||||
|
'output_fields': list(output_fields),
|
||||||
|
}
|
||||||
|
return client.search(**kwargs)
|
||||||
|
|
||||||
|
response = await asyncio.to_thread(_search)
|
||||||
|
mapping = {
|
||||||
|
'content_field': content_field,
|
||||||
|
'metadata_field': metadata_field,
|
||||||
|
'document_id_field': source_config.get('document_id_field') or 'id',
|
||||||
|
'score_field': 'distance',
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized = []
|
||||||
|
for hit in (response[0] if response else []):
|
||||||
|
item = dict(hit)
|
||||||
|
entity = item.get('entity') or {}
|
||||||
|
result = {
|
||||||
|
**entity,
|
||||||
|
'id': item.get('id') or entity.get('id'),
|
||||||
|
'distance': item.get('distance'),
|
||||||
|
}
|
||||||
|
normalized.append(_normalize_result(result, mapping, knowledge, distance=item.get('distance')))
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
async def _retrieve_pgvector(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
|
||||||
|
try:
|
||||||
|
import psycopg
|
||||||
|
from pgvector.psycopg import register_vector
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
except ImportError as exc:
|
||||||
|
raise RuntimeError('psycopg and pgvector are required for pgvector retrieval') from exc
|
||||||
|
|
||||||
|
if not embedding_function:
|
||||||
|
raise RuntimeError('Embedding function is not configured')
|
||||||
|
|
||||||
|
config = connection.get('config') or {}
|
||||||
|
external = (knowledge.meta or {}).get('external', {})
|
||||||
|
source = external.get('source') or {}
|
||||||
|
collection_name = source.get('name')
|
||||||
|
if not collection_name:
|
||||||
|
raise RuntimeError('pgvector collection is not configured')
|
||||||
|
source_config = _source_config(knowledge)
|
||||||
|
table_name = source_config.get('table_name') or 'document_chunk'
|
||||||
|
collection_field = source_config.get('collection_field') or 'collection_name'
|
||||||
|
content_field = source_config.get('content_field') or 'text'
|
||||||
|
vector_field = source_config.get('vector_field') or 'vector'
|
||||||
|
metadata_field = source_config.get('metadata_field') or 'vmetadata'
|
||||||
|
document_id_field = source_config.get('document_id_field') or 'id'
|
||||||
|
|
||||||
|
vector = await embedding_function(query)
|
||||||
|
|
||||||
|
def _search():
|
||||||
|
from psycopg import sql
|
||||||
|
|
||||||
|
table_identifier = sql.SQL('.').join(
|
||||||
|
sql.Identifier(_safe_identifier(part, 'table name')) for part in table_name.split('.')
|
||||||
|
)
|
||||||
|
collection_identifier = sql.Identifier(_safe_identifier(collection_field, 'collection field'))
|
||||||
|
content_identifier = sql.Identifier(_safe_identifier(content_field, 'content field'))
|
||||||
|
vector_identifier = sql.Identifier(_safe_identifier(vector_field, 'vector field'))
|
||||||
|
document_id_identifier = sql.Identifier(_safe_identifier(document_id_field, 'document id field'))
|
||||||
|
metadata_sql = (
|
||||||
|
sql.Identifier(_safe_identifier(metadata_field, 'metadata field'))
|
||||||
|
if metadata_field
|
||||||
|
else sql.SQL("'{}'::jsonb")
|
||||||
|
)
|
||||||
|
|
||||||
|
with psycopg.connect(
|
||||||
|
connection.get('endpoint'),
|
||||||
|
row_factory=dict_row,
|
||||||
|
connect_timeout=config.get('timeout') or 30,
|
||||||
|
) as conn:
|
||||||
|
register_vector(conn)
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
sql.SQL(
|
||||||
|
"""
|
||||||
|
SELECT {document_id} AS id,
|
||||||
|
{content} AS content,
|
||||||
|
{metadata} AS metadata,
|
||||||
|
{vector_column} <=> %s AS distance
|
||||||
|
FROM {table_name}
|
||||||
|
WHERE {collection} = %s
|
||||||
|
ORDER BY distance ASC
|
||||||
|
LIMIT %s
|
||||||
|
"""
|
||||||
|
).format(
|
||||||
|
document_id=document_id_identifier,
|
||||||
|
content=content_identifier,
|
||||||
|
metadata=metadata_sql,
|
||||||
|
vector_column=vector_identifier,
|
||||||
|
table_name=table_identifier,
|
||||||
|
collection=collection_identifier,
|
||||||
|
),
|
||||||
|
(vector, collection_name, count),
|
||||||
|
)
|
||||||
|
return cur.fetchall()
|
||||||
|
|
||||||
|
rows = await asyncio.to_thread(_search)
|
||||||
|
mapping = {
|
||||||
|
'content_field': 'content',
|
||||||
|
'metadata_field': 'metadata',
|
||||||
|
'document_id_field': 'id',
|
||||||
|
'score_field': 'distance',
|
||||||
|
}
|
||||||
|
return [_normalize_result(row, mapping, knowledge, distance=row.get('distance')) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
async def retrieve_external_knowledge(
|
||||||
|
request,
|
||||||
|
knowledge: KnowledgeModel,
|
||||||
|
queries: list[str],
|
||||||
|
count: int,
|
||||||
|
user=None,
|
||||||
|
) -> dict:
|
||||||
|
external = (knowledge.meta or {}).get('external', {})
|
||||||
|
connection_id = external.get('connection_id')
|
||||||
|
if not connection_id:
|
||||||
|
raise RuntimeError('External knowledge connection is not configured')
|
||||||
|
|
||||||
|
connection = await _get_external_connection(connection_id)
|
||||||
|
if not connection:
|
||||||
|
raise RuntimeError('External knowledge connection not found')
|
||||||
|
|
||||||
|
return await retrieve_external_knowledge_for_connection(request, knowledge, connection, queries, count, user=user)
|
||||||
|
|
||||||
|
|
||||||
|
async def retrieve_external_knowledge_for_connection(
|
||||||
|
request,
|
||||||
|
knowledge: KnowledgeModel,
|
||||||
|
connection: dict,
|
||||||
|
queries: list[str],
|
||||||
|
count: int,
|
||||||
|
user=None,
|
||||||
|
) -> dict:
|
||||||
|
auth_config = connection.get('auth_config') or {}
|
||||||
|
if not connection.get('enabled', True):
|
||||||
|
raise RuntimeError('External knowledge connection is disabled')
|
||||||
|
|
||||||
|
started_at = time.monotonic()
|
||||||
|
chunks = []
|
||||||
|
provider = (connection.get('provider') or '').lower()
|
||||||
|
|
||||||
|
for query in queries:
|
||||||
|
if provider == 'qdrant':
|
||||||
|
chunks.extend(
|
||||||
|
await _retrieve_qdrant(
|
||||||
|
connection,
|
||||||
|
auth_config,
|
||||||
|
knowledge,
|
||||||
|
query,
|
||||||
|
count,
|
||||||
|
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif provider == 'milvus':
|
||||||
|
chunks.extend(
|
||||||
|
await _retrieve_milvus(
|
||||||
|
connection,
|
||||||
|
auth_config,
|
||||||
|
knowledge,
|
||||||
|
query,
|
||||||
|
count,
|
||||||
|
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif provider == 'pgvector':
|
||||||
|
chunks.extend(
|
||||||
|
await _retrieve_pgvector(
|
||||||
|
connection,
|
||||||
|
auth_config,
|
||||||
|
knowledge,
|
||||||
|
query,
|
||||||
|
count,
|
||||||
|
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f'Unsupported external knowledge provider: {connection.get("provider")}')
|
||||||
|
|
||||||
|
chunks = chunks[:count]
|
||||||
|
log.info(
|
||||||
|
'external_knowledge_retrieval knowledge_id=%s connection_id=%s provider=%s user_id=%s latency_ms=%s result_count=%s',
|
||||||
|
knowledge.id,
|
||||||
|
connection.get('id'),
|
||||||
|
connection.get('provider'),
|
||||||
|
getattr(user, 'id', None),
|
||||||
|
round((time.monotonic() - started_at) * 1000),
|
||||||
|
len(chunks),
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'documents': [[chunk['content'] for chunk in chunks]],
|
||||||
|
'metadatas': [[chunk['metadata'] for chunk in chunks]],
|
||||||
|
'distances': [[chunk['distance'] for chunk in chunks]],
|
||||||
|
}
|
||||||
@@ -43,6 +43,7 @@ from open_webui.models.config import Config
|
|||||||
from open_webui.models.users import UserModel
|
from open_webui.models.users import UserModel
|
||||||
from open_webui.retrieval.loaders.youtube import YoutubeLoader
|
from open_webui.retrieval.loaders.youtube import YoutubeLoader
|
||||||
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
|
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
|
||||||
|
from open_webui.retrieval.external import retrieve_external_knowledge
|
||||||
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
|
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
|
||||||
from open_webui.retrieval.vector.main import GetResult, SearchResult
|
from open_webui.retrieval.vector.main import GetResult, SearchResult
|
||||||
from open_webui.retrieval.web.utils import get_web_loader
|
from open_webui.retrieval.web.utils import get_web_loader
|
||||||
@@ -84,6 +85,7 @@ LOADER_CONFIG_KEYS = {
|
|||||||
'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
|
'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
|
||||||
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
|
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
|
||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
||||||
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
|
||||||
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
'TIKA_SERVER_URL': 'rag.tika_server_url',
|
||||||
'DOCLING_SERVER_URL': 'rag.docling_server_url',
|
'DOCLING_SERVER_URL': 'rag.docling_server_url',
|
||||||
'DOCLING_API_KEY': 'rag.docling_api_key',
|
'DOCLING_API_KEY': 'rag.docling_api_key',
|
||||||
@@ -130,18 +132,16 @@ def build_loader_from_config(request, config: dict):
|
|||||||
"""Build a Loader instance with the admin's configured extraction engine settings."""
|
"""Build a Loader instance with the admin's configured extraction engine settings."""
|
||||||
from open_webui.retrieval.loaders.main import Loader
|
from open_webui.retrieval.loaders.main import Loader
|
||||||
|
|
||||||
loader_config = {
|
loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
|
||||||
key: config.get(key)
|
|
||||||
for key in LOADER_CONFIG_KEYS
|
|
||||||
if key.isupper()
|
|
||||||
}
|
|
||||||
return Loader(
|
return Loader(
|
||||||
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
|
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
|
||||||
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
|
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _extract_text_from_binary_response(request, response: requests.Response, url: str, loader_config: dict) -> tuple[str, list]:
|
def _extract_text_from_binary_response(
|
||||||
|
request, response: requests.Response, url: str, loader_config: dict
|
||||||
|
) -> tuple[str, list]:
|
||||||
"""Download response body to a temp file and extract text using the Loader pipeline."""
|
"""Download response body to a temp file and extract text using the Loader pipeline."""
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -774,11 +774,7 @@ async def query_collection_with_hybrid_search(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
native_task_results = await asyncio.gather(
|
native_task_results = await asyncio.gather(
|
||||||
*[
|
*[process_native_query(collection_name, query) for collection_name in collection_names for query in queries]
|
||||||
process_native_query(collection_name, query)
|
|
||||||
for collection_name in collection_names
|
|
||||||
for query in queries
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
if native_task_results and all(result is not None for result in native_task_results):
|
if native_task_results and all(result is not None for result in native_task_results):
|
||||||
return merge_and_sort_query_results(native_task_results, k=k)
|
return merge_and_sort_query_results(native_task_results, k=k)
|
||||||
@@ -1485,50 +1481,61 @@ async def get_sources_from_items(
|
|||||||
permission='read',
|
permission='read',
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
if item.get('context') == 'full' or bypass_embedding_and_retrieval:
|
if (knowledge_base.meta or {}).get('source') == 'external':
|
||||||
if knowledge_base and (
|
query_result = await retrieve_external_knowledge(
|
||||||
user.role == 'admin'
|
request,
|
||||||
or knowledge_base.user_id == user.id
|
knowledge_base,
|
||||||
or await AccessGrants.has_access(
|
queries=queries,
|
||||||
user_id=user.id,
|
count=k,
|
||||||
resource_type='knowledge',
|
user=user,
|
||||||
resource_id=knowledge_base.id,
|
)
|
||||||
permission='read',
|
extracted_collections.append(knowledge_base.id)
|
||||||
)
|
|
||||||
):
|
|
||||||
files = await Knowledges.get_files_by_id(knowledge_base.id)
|
|
||||||
|
|
||||||
documents = []
|
|
||||||
metadatas = []
|
|
||||||
for file in files:
|
|
||||||
documents.append(file.data.get('content', ''))
|
|
||||||
metadatas.append(
|
|
||||||
{
|
|
||||||
'file_id': file.id,
|
|
||||||
'name': file.filename,
|
|
||||||
'source': file.filename,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
query_result = {
|
|
||||||
'documents': [documents],
|
|
||||||
'metadatas': [metadatas],
|
|
||||||
}
|
|
||||||
else:
|
else:
|
||||||
if item.get('legacy'):
|
if item.get('context') == 'full' or bypass_embedding_and_retrieval:
|
||||||
if BYPASS_RETRIEVAL_ACCESS_CONTROL:
|
if knowledge_base and (
|
||||||
collection_names = item.get('collection_names', [])
|
user.role == 'admin'
|
||||||
else:
|
or knowledge_base.user_id == user.id
|
||||||
# Legacy KB: item.collection_names is client-supplied.
|
or await AccessGrants.has_access(
|
||||||
# Validate against the KB's actual files to prevent
|
user_id=user.id,
|
||||||
# cross-tenant collection name substitution.
|
resource_type='knowledge',
|
||||||
|
resource_id=knowledge_base.id,
|
||||||
|
permission='read',
|
||||||
|
)
|
||||||
|
):
|
||||||
files = await Knowledges.get_files_by_id(knowledge_base.id)
|
files = await Knowledges.get_files_by_id(knowledge_base.id)
|
||||||
owned_names = {f'file-{f.id}' for f in files}
|
|
||||||
owned_names.add(knowledge_base.id)
|
documents = []
|
||||||
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
|
metadatas = []
|
||||||
collection_names = valid_names if valid_names else [knowledge_base.id]
|
for file in files:
|
||||||
|
documents.append(file.data.get('content', ''))
|
||||||
|
metadatas.append(
|
||||||
|
{
|
||||||
|
'file_id': file.id,
|
||||||
|
'name': file.filename,
|
||||||
|
'source': file.filename,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
query_result = {
|
||||||
|
'documents': [documents],
|
||||||
|
'metadatas': [metadatas],
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
collection_names.append(item['id'])
|
if item.get('legacy'):
|
||||||
|
if BYPASS_RETRIEVAL_ACCESS_CONTROL:
|
||||||
|
collection_names = item.get('collection_names', [])
|
||||||
|
else:
|
||||||
|
# Legacy KB: item.collection_names is client-supplied.
|
||||||
|
# Validate against the KB's actual files to prevent
|
||||||
|
# cross-tenant collection name substitution.
|
||||||
|
files = await Knowledges.get_files_by_id(knowledge_base.id)
|
||||||
|
owned_names = {f'file-{f.id}' for f in files}
|
||||||
|
owned_names.add(knowledge_base.id)
|
||||||
|
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
|
||||||
|
collection_names = valid_names if valid_names else [knowledge_base.id]
|
||||||
|
else:
|
||||||
|
collection_names.append(item['id'])
|
||||||
|
|
||||||
elif item.get('docs'):
|
elif item.get('docs'):
|
||||||
# BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
|
# BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
import zipfile
|
import zipfile
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
@@ -27,6 +29,7 @@ from open_webui.models.knowledge import (
|
|||||||
)
|
)
|
||||||
from open_webui.models.models import ModelForm, Models
|
from open_webui.models.models import ModelForm, Models
|
||||||
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
|
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
|
||||||
|
from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection
|
||||||
from open_webui.routers.retrieval import (
|
from open_webui.routers.retrieval import (
|
||||||
BatchProcessFilesForm,
|
BatchProcessFilesForm,
|
||||||
ProcessFileForm,
|
ProcessFileForm,
|
||||||
@@ -110,6 +113,17 @@ class KnowledgeAccessListResponse(BaseModel):
|
|||||||
total: int
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
def is_external_knowledge(knowledge) -> bool:
|
||||||
|
return (knowledge.meta or {}).get('source') == 'external'
|
||||||
|
|
||||||
|
|
||||||
|
def external_knowledge_error():
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail='External knowledge bases are read-only.',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get('/', response_model=KnowledgeAccessListResponse)
|
@router.get('/', response_model=KnowledgeAccessListResponse)
|
||||||
async def get_knowledge_bases(
|
async def get_knowledge_bases(
|
||||||
page: int | None = 1,
|
page: int | None = 1,
|
||||||
@@ -163,6 +177,7 @@ async def get_knowledge_bases(
|
|||||||
async def search_knowledge_bases(
|
async def search_knowledge_bases(
|
||||||
query: str | None = None,
|
query: str | None = None,
|
||||||
view_option: str | None = None,
|
view_option: str | None = None,
|
||||||
|
source: str | None = None,
|
||||||
page: int | None = 1,
|
page: int | None = 1,
|
||||||
user=Depends(get_verified_user),
|
user=Depends(get_verified_user),
|
||||||
db: AsyncSession = Depends(get_async_session),
|
db: AsyncSession = Depends(get_async_session),
|
||||||
@@ -176,6 +191,8 @@ async def search_knowledge_bases(
|
|||||||
filter['query'] = query
|
filter['query'] = query
|
||||||
if view_option:
|
if view_option:
|
||||||
filter['view_option'] = view_option
|
filter['view_option'] = view_option
|
||||||
|
if source in {'local', 'external'}:
|
||||||
|
filter['source'] = source
|
||||||
|
|
||||||
groups = await Groups.get_groups_by_member_id(user.id, db=db)
|
groups = await Groups.get_groups_by_member_id(user.id, db=db)
|
||||||
user_group_ids = {group.id for group in groups}
|
user_group_ids = {group.id for group in groups}
|
||||||
@@ -379,6 +396,574 @@ async def reindex_knowledge_base_metadata_embeddings(
|
|||||||
return {'total': len(knowledge_bases), 'success': success_count}
|
return {'total': len(knowledge_bases), 'success': success_count}
|
||||||
|
|
||||||
|
|
||||||
|
############################
|
||||||
|
# External Knowledge Sources
|
||||||
|
############################
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeSourceForm(BaseModel):
|
||||||
|
type: str = 'collection'
|
||||||
|
name: str
|
||||||
|
config: Optional[dict] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeCreateForm(BaseModel):
|
||||||
|
name: str
|
||||||
|
description: str = ''
|
||||||
|
connection_id: str
|
||||||
|
source: ExternalKnowledgeSourceForm
|
||||||
|
access_grants: Optional[list[dict]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeSourceCreateForm(BaseModel):
|
||||||
|
name: str
|
||||||
|
description: str = ''
|
||||||
|
connection: ExternalKnowledgeConnectionForm
|
||||||
|
source: ExternalKnowledgeSourceForm
|
||||||
|
access_grants: Optional[list[dict]] = None
|
||||||
|
test_query: str
|
||||||
|
test_count: int = 5
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeSourceUpdateForm(ExternalKnowledgeSourceCreateForm):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeSourceTestForm(BaseModel):
|
||||||
|
connection_id: Optional[str] = None
|
||||||
|
connection: ExternalKnowledgeConnectionForm
|
||||||
|
source: ExternalKnowledgeSourceForm
|
||||||
|
query: str
|
||||||
|
count: int = 5
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeRetrieveTestForm(BaseModel):
|
||||||
|
query: str
|
||||||
|
source: Optional[ExternalKnowledgeSourceForm] = None
|
||||||
|
count: int = 5
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeConnectionForm(BaseModel):
|
||||||
|
name: str
|
||||||
|
provider: str
|
||||||
|
endpoint: str
|
||||||
|
auth_config: Optional[dict] = None
|
||||||
|
config: Optional[dict] = None
|
||||||
|
capabilities: Optional[dict] = None
|
||||||
|
enabled: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeConnectionListResponse(BaseModel):
|
||||||
|
items: list[dict]
|
||||||
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
|
||||||
|
EXTERNAL_KNOWLEDGE_PROVIDERS = {'qdrant', 'milvus', 'pgvector'}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_external_connection_form(form_data: ExternalKnowledgeConnectionForm) -> tuple[str, dict]:
|
||||||
|
provider = form_data.provider.lower().strip()
|
||||||
|
if provider not in EXTERNAL_KNOWLEDGE_PROVIDERS:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail='Unsupported external knowledge provider.',
|
||||||
|
)
|
||||||
|
|
||||||
|
if not form_data.name.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source name is required.')
|
||||||
|
|
||||||
|
if not form_data.endpoint.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source endpoint is required.')
|
||||||
|
|
||||||
|
config = form_data.config or {}
|
||||||
|
allowed_config_keys = {'timeout'}
|
||||||
|
if provider == 'milvus':
|
||||||
|
allowed_config_keys.add('db_name')
|
||||||
|
|
||||||
|
return provider, {key: value for key, value in config.items() if key in allowed_config_keys}
|
||||||
|
|
||||||
|
|
||||||
|
def _external_auth_config(provider: str, incoming: Optional[dict], existing: Optional[dict] = None) -> dict:
|
||||||
|
if provider == 'pgvector':
|
||||||
|
return {}
|
||||||
|
return existing if incoming is None else incoming or {}
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_external_source(source: ExternalKnowledgeSourceForm, provider: str) -> ExternalKnowledgeSourceForm:
|
||||||
|
source.type = (source.type or 'collection').strip()
|
||||||
|
source.name = source.name.strip()
|
||||||
|
|
||||||
|
if source.type != 'collection':
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Only collection sources are supported.')
|
||||||
|
if not source.name:
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Collection name is required.')
|
||||||
|
|
||||||
|
config = source.config or {}
|
||||||
|
allowed_keys = {'content_field', 'metadata_field', 'document_id_field'}
|
||||||
|
if provider in {'qdrant', 'milvus'}:
|
||||||
|
allowed_keys.add('vector_field')
|
||||||
|
if provider == 'pgvector':
|
||||||
|
allowed_keys.update({'table_name', 'collection_field', 'vector_field'})
|
||||||
|
|
||||||
|
normalized_config = {
|
||||||
|
key: value.strip() if isinstance(value, str) else value
|
||||||
|
for key, value in config.items()
|
||||||
|
if key in allowed_keys and value is not None and (not isinstance(value, str) or value.strip())
|
||||||
|
}
|
||||||
|
|
||||||
|
if not normalized_config.get('content_field'):
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Content field is required.')
|
||||||
|
if provider in {'milvus', 'pgvector'} and not normalized_config.get('vector_field'):
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Vector field is required.')
|
||||||
|
|
||||||
|
source.config = normalized_config
|
||||||
|
return source
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitize_external_connection(connection: dict) -> dict:
|
||||||
|
sanitized = {**connection}
|
||||||
|
sanitized.pop('auth_config', None)
|
||||||
|
sanitized['auth_configured'] = bool(connection.get('auth_config'))
|
||||||
|
return sanitized
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_external_connections() -> list[dict]:
|
||||||
|
return await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
|
||||||
|
|
||||||
|
|
||||||
|
async def _set_external_connections(connections: list[dict]) -> None:
|
||||||
|
await Config.upsert({EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY: connections})
|
||||||
|
|
||||||
|
|
||||||
|
def _external_connection_dict(form_data: ExternalKnowledgeConnectionForm, user_id: str, id: Optional[str] = None) -> dict:
|
||||||
|
provider, config = _validate_external_connection_form(form_data)
|
||||||
|
now = int(time.time())
|
||||||
|
return {
|
||||||
|
'id': id or str(uuid.uuid4()),
|
||||||
|
'name': form_data.name.strip(),
|
||||||
|
'provider': provider,
|
||||||
|
'endpoint': form_data.endpoint.strip(),
|
||||||
|
'auth_config': _external_auth_config(provider, form_data.auth_config),
|
||||||
|
'config': config,
|
||||||
|
'capabilities': form_data.capabilities or {'retrieve': True},
|
||||||
|
'health': None,
|
||||||
|
'enabled': form_data.enabled,
|
||||||
|
'created_by': user_id,
|
||||||
|
'created_at': now,
|
||||||
|
'updated_at': now,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _external_connection_update_dict(
|
||||||
|
form_data: ExternalKnowledgeConnectionForm,
|
||||||
|
existing: dict,
|
||||||
|
) -> dict:
|
||||||
|
provider, config = _validate_external_connection_form(form_data)
|
||||||
|
return {
|
||||||
|
**existing,
|
||||||
|
'name': form_data.name.strip(),
|
||||||
|
'provider': provider,
|
||||||
|
'endpoint': form_data.endpoint.strip(),
|
||||||
|
'auth_config': _external_auth_config(provider, form_data.auth_config, existing.get('auth_config')) or {},
|
||||||
|
'config': config,
|
||||||
|
'capabilities': form_data.capabilities or {'retrieve': True},
|
||||||
|
'enabled': form_data.enabled,
|
||||||
|
'updated_at': int(time.time()),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_external_connection(id: str) -> Optional[dict]:
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
return next((connection for connection in connections if connection.get('id') == id), None)
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_external_connection_mappings(connection_id: str, db: Optional[AsyncSession] = None) -> int:
|
||||||
|
count = 0
|
||||||
|
for knowledge in await Knowledges.get_knowledge_bases(db=db):
|
||||||
|
if (knowledge.meta or {}).get('external', {}).get('connection_id') == connection_id:
|
||||||
|
count += 1
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/external/connections', response_model=ExternalKnowledgeConnectionListResponse)
|
||||||
|
async def get_external_knowledge_connections(
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connections = [_sanitize_external_connection(connection) for connection in await _get_external_connections()]
|
||||||
|
return ExternalKnowledgeConnectionListResponse(items=connections, total=len(connections))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/connections', response_model=dict)
|
||||||
|
async def create_external_knowledge_connection(
|
||||||
|
form_data: ExternalKnowledgeConnectionForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
):
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
connection = _external_connection_dict(form_data, user.id)
|
||||||
|
connections.append(connection)
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
return _sanitize_external_connection(connection)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/external/connections/{id}', response_model=dict)
|
||||||
|
async def get_external_knowledge_connection(
|
||||||
|
id: str,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connection = await _get_external_connection(id)
|
||||||
|
if not connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
return _sanitize_external_connection(connection)
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch('/external/connections/{id}', response_model=dict)
|
||||||
|
async def update_external_knowledge_connection(
|
||||||
|
id: str,
|
||||||
|
form_data: ExternalKnowledgeConnectionForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
):
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == id), None)
|
||||||
|
if idx is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
|
||||||
|
connection = _external_connection_update_dict(form_data, connections[idx])
|
||||||
|
connections[idx] = connection
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
return _sanitize_external_connection(connection)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete('/external/connections/{id}', response_model=bool)
|
||||||
|
async def delete_external_knowledge_connection(
|
||||||
|
id: str,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connection = await _get_external_connection(id)
|
||||||
|
if not connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
|
||||||
|
if await _count_external_connection_mappings(id, db=db) > 0:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail='External connection is still used by knowledge bases.',
|
||||||
|
)
|
||||||
|
|
||||||
|
connections = [connection for connection in await _get_external_connections() if connection.get('id') != id]
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/connections/{id}/test', response_model=dict)
|
||||||
|
async def test_external_knowledge_connection(
|
||||||
|
id: str,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connection = await _get_external_connection(id)
|
||||||
|
if not connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
|
||||||
|
health = {
|
||||||
|
'ok': bool(connection.get('enabled') and connection.get('endpoint')),
|
||||||
|
'provider': connection.get('provider'),
|
||||||
|
'checked_at': int(time.time()),
|
||||||
|
}
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
for item in connections:
|
||||||
|
if item.get('id') == id:
|
||||||
|
item['health'] = health
|
||||||
|
item['updated_at'] = int(time.time())
|
||||||
|
break
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
return health
|
||||||
|
|
||||||
|
|
||||||
|
async def _test_external_source_definition(
|
||||||
|
request: Request,
|
||||||
|
connection: dict,
|
||||||
|
source: ExternalKnowledgeSourceForm,
|
||||||
|
query: str,
|
||||||
|
count: int,
|
||||||
|
user,
|
||||||
|
) -> dict:
|
||||||
|
if not query.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query is required.')
|
||||||
|
|
||||||
|
source = _normalize_external_source(source, connection.get('provider'))
|
||||||
|
test_knowledge = KnowledgeResponse(
|
||||||
|
id='external-test',
|
||||||
|
user_id=user.id,
|
||||||
|
name=connection.get('name'),
|
||||||
|
description='',
|
||||||
|
meta={
|
||||||
|
'source': 'external',
|
||||||
|
'read_only': True,
|
||||||
|
'external': {
|
||||||
|
'connection_id': connection.get('id'),
|
||||||
|
'source': source.model_dump(),
|
||||||
|
'provider': connection.get('provider'),
|
||||||
|
'auth_mode': 'service_account',
|
||||||
|
'capabilities': {'retrieve': True},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
access_grants=[],
|
||||||
|
created_at=int(time.time()),
|
||||||
|
updated_at=int(time.time()),
|
||||||
|
)
|
||||||
|
result = await retrieve_external_knowledge_for_connection(
|
||||||
|
request,
|
||||||
|
test_knowledge,
|
||||||
|
connection,
|
||||||
|
[query.strip()],
|
||||||
|
count,
|
||||||
|
user=user,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
'documents': result.get('documents', [[]])[0],
|
||||||
|
'metadatas': result.get('metadatas', [[]])[0],
|
||||||
|
'distances': result.get('distances', [[]])[0],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/source/test', response_model=dict)
|
||||||
|
async def test_external_knowledge_source(
|
||||||
|
request: Request,
|
||||||
|
form_data: ExternalKnowledgeSourceTestForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
):
|
||||||
|
if form_data.connection_id:
|
||||||
|
existing_connection = await _get_external_connection(form_data.connection_id)
|
||||||
|
if not existing_connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
|
||||||
|
connection = _external_connection_update_dict(form_data.connection, existing_connection)
|
||||||
|
else:
|
||||||
|
connection = _external_connection_dict(form_data.connection, user.id, id='external-test')
|
||||||
|
|
||||||
|
return await _test_external_source_definition(
|
||||||
|
request,
|
||||||
|
connection,
|
||||||
|
form_data.source,
|
||||||
|
form_data.query,
|
||||||
|
form_data.count,
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/connections/{id}/retrieve-test', response_model=dict)
|
||||||
|
async def test_external_knowledge_retrieval(
|
||||||
|
request: Request,
|
||||||
|
id: str,
|
||||||
|
form_data: ExternalKnowledgeRetrieveTestForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connection = await _get_external_connection(id)
|
||||||
|
if not connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
|
||||||
|
source = form_data.source or ExternalKnowledgeSourceForm(name='test', config={'content_field': 'payload.text'})
|
||||||
|
return await _test_external_source_definition(request, connection, source, form_data.query, form_data.count, user)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/knowledge/create', response_model=KnowledgeResponse | None)
|
||||||
|
async def create_external_knowledge(
|
||||||
|
request: Request,
|
||||||
|
form_data: ExternalKnowledgeCreateForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
connection = await _get_external_connection(form_data.connection_id)
|
||||||
|
if not connection:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
if not form_data.name.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
|
||||||
|
source = _normalize_external_source(form_data.source, connection.get('provider'))
|
||||||
|
|
||||||
|
form_data.access_grants = await filter_allowed_access_grants(
|
||||||
|
await Config.get('user.permissions'),
|
||||||
|
user.id,
|
||||||
|
user.role,
|
||||||
|
form_data.access_grants,
|
||||||
|
'sharing.public_knowledge',
|
||||||
|
)
|
||||||
|
|
||||||
|
knowledge = await Knowledges.insert_new_knowledge(
|
||||||
|
user.id,
|
||||||
|
KnowledgeForm(
|
||||||
|
name=form_data.name.strip(),
|
||||||
|
description=form_data.description,
|
||||||
|
access_grants=form_data.access_grants,
|
||||||
|
),
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
if not knowledge:
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
'source': 'external',
|
||||||
|
'read_only': True,
|
||||||
|
'external': {
|
||||||
|
'connection_id': form_data.connection_id,
|
||||||
|
'source': source.model_dump(),
|
||||||
|
'provider': connection.get('provider'),
|
||||||
|
'auth_mode': 'service_account',
|
||||||
|
'capabilities': {'retrieve': True},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
|
||||||
|
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
|
||||||
|
return knowledge
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/external/source/create', response_model=KnowledgeResponse | None)
|
||||||
|
async def create_external_knowledge_source(
|
||||||
|
request: Request,
|
||||||
|
form_data: ExternalKnowledgeSourceCreateForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
if not form_data.name.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
|
||||||
|
|
||||||
|
connection = _external_connection_dict(form_data.connection, user.id)
|
||||||
|
source = _normalize_external_source(form_data.source, connection.get('provider'))
|
||||||
|
test_result = await _test_external_source_definition(
|
||||||
|
request,
|
||||||
|
connection,
|
||||||
|
source,
|
||||||
|
form_data.test_query,
|
||||||
|
form_data.test_count,
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
if not test_result.get('documents'):
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
|
||||||
|
|
||||||
|
form_data.access_grants = await filter_allowed_access_grants(
|
||||||
|
await Config.get('user.permissions'),
|
||||||
|
user.id,
|
||||||
|
user.role,
|
||||||
|
form_data.access_grants,
|
||||||
|
'sharing.public_knowledge',
|
||||||
|
)
|
||||||
|
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
connections.append(connection)
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
|
||||||
|
knowledge = await Knowledges.insert_new_knowledge(
|
||||||
|
user.id,
|
||||||
|
KnowledgeForm(
|
||||||
|
name=form_data.name.strip(),
|
||||||
|
description=form_data.description,
|
||||||
|
access_grants=form_data.access_grants,
|
||||||
|
),
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
if not knowledge:
|
||||||
|
connections = [item for item in await _get_external_connections() if item.get('id') != connection.get('id')]
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
'source': 'external',
|
||||||
|
'read_only': True,
|
||||||
|
'external': {
|
||||||
|
'connection_id': connection.get('id'),
|
||||||
|
'source': source.model_dump(),
|
||||||
|
'provider': connection.get('provider'),
|
||||||
|
'auth_mode': 'service_account',
|
||||||
|
'capabilities': {'retrieve': True},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
|
||||||
|
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
|
||||||
|
return knowledge
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch('/external/source/{id}', response_model=KnowledgeResponse | None)
|
||||||
|
async def update_external_knowledge_source(
|
||||||
|
request: Request,
|
||||||
|
id: str,
|
||||||
|
form_data: ExternalKnowledgeSourceUpdateForm,
|
||||||
|
user=Depends(get_admin_user),
|
||||||
|
db: AsyncSession = Depends(get_async_session),
|
||||||
|
):
|
||||||
|
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
|
||||||
|
if not knowledge or not is_external_knowledge(knowledge):
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
|
||||||
|
if not form_data.name.strip():
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
|
||||||
|
|
||||||
|
connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
|
||||||
|
connections = await _get_external_connections()
|
||||||
|
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == connection_id), None)
|
||||||
|
if idx is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
|
||||||
|
|
||||||
|
existing_connection = connections[idx]
|
||||||
|
connection = _external_connection_update_dict(form_data.connection, existing_connection)
|
||||||
|
source = _normalize_external_source(form_data.source, connection.get('provider'))
|
||||||
|
test_result = await _test_external_source_definition(
|
||||||
|
request,
|
||||||
|
connection,
|
||||||
|
source,
|
||||||
|
form_data.test_query,
|
||||||
|
form_data.test_count,
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
if not test_result.get('documents'):
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
|
||||||
|
|
||||||
|
form_data.access_grants = await filter_allowed_access_grants(
|
||||||
|
await Config.get('user.permissions'),
|
||||||
|
user.id,
|
||||||
|
user.role,
|
||||||
|
form_data.access_grants,
|
||||||
|
'sharing.public_knowledge',
|
||||||
|
)
|
||||||
|
|
||||||
|
connections[idx] = connection
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
|
||||||
|
updated = await Knowledges.update_knowledge_by_id(
|
||||||
|
id=id,
|
||||||
|
form_data=KnowledgeForm(
|
||||||
|
name=form_data.name.strip(),
|
||||||
|
description=form_data.description,
|
||||||
|
access_grants=form_data.access_grants,
|
||||||
|
),
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
if not updated:
|
||||||
|
connections[idx] = existing_connection
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
'source': 'external',
|
||||||
|
'read_only': True,
|
||||||
|
'external': {
|
||||||
|
'connection_id': connection.get('id'),
|
||||||
|
'source': source.model_dump(),
|
||||||
|
'provider': connection.get('provider'),
|
||||||
|
'auth_mode': 'service_account',
|
||||||
|
'capabilities': {'retrieve': True},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
updated = await Knowledges.update_knowledge_meta_by_id(id, meta, db=db)
|
||||||
|
if not updated:
|
||||||
|
connections[idx] = existing_connection
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
|
||||||
|
|
||||||
|
await embed_knowledge_base_metadata(request, id, updated.name, updated.description)
|
||||||
|
return updated
|
||||||
|
|
||||||
|
|
||||||
############################
|
############################
|
||||||
# GetKnowledgeById
|
# GetKnowledgeById
|
||||||
############################
|
############################
|
||||||
@@ -711,6 +1296,8 @@ async def add_file_to_knowledge_by_id(
|
|||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
@@ -798,6 +1385,8 @@ async def update_file_from_knowledge_by_id(
|
|||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
@@ -877,6 +1466,8 @@ async def remove_file_from_knowledge_by_id(
|
|||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
@@ -1004,11 +1595,21 @@ async def delete_knowledge_by_id(
|
|||||||
await Models.update_model_by_id(model.id, model_form, db=db)
|
await Models.update_model_by_id(model.id, model_form, db=db)
|
||||||
|
|
||||||
# Clean up vector DB
|
# Clean up vector DB
|
||||||
try:
|
if is_external_knowledge(knowledge):
|
||||||
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
|
connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
|
||||||
except Exception as e:
|
if connection_id:
|
||||||
log.debug(e)
|
connections = [
|
||||||
pass
|
connection
|
||||||
|
for connection in await _get_external_connections()
|
||||||
|
if connection.get('id') != connection_id
|
||||||
|
]
|
||||||
|
await _set_external_connections(connections)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(e)
|
||||||
|
pass
|
||||||
|
|
||||||
# Remove knowledge base embedding
|
# Remove knowledge base embedding
|
||||||
await remove_knowledge_base_metadata_embedding(id)
|
await remove_knowledge_base_metadata_embedding(id)
|
||||||
@@ -1035,6 +1636,8 @@ async def reset_knowledge_by_id(
|
|||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
@@ -1263,6 +1866,8 @@ async def add_files_to_knowledge_batch(
|
|||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
@@ -1380,6 +1985,8 @@ async def export_knowledge_by_id(id: str, user=Depends(get_admin_user), db: Asyn
|
|||||||
status_code=status.HTTP_404_NOT_FOUND,
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
|
|
||||||
files = await Knowledges.get_files_by_id(id, db=db)
|
files = await Knowledges.get_files_by_id(id, db=db)
|
||||||
|
|
||||||
@@ -1441,6 +2048,8 @@ async def _verify_knowledge_write_access(id: str, user, db: AsyncSession):
|
|||||||
status_code=status.HTTP_404_NOT_FOUND,
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||||
)
|
)
|
||||||
|
if is_external_knowledge(knowledge):
|
||||||
|
external_knowledge_error()
|
||||||
if (
|
if (
|
||||||
knowledge.user_id != user.id
|
knowledge.user_id != user.id
|
||||||
and not await AccessGrants.has_access(
|
and not await AccessGrants.has_access(
|
||||||
|
|||||||
@@ -288,6 +288,7 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||||||
'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
|
'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
|
||||||
'EXA_API_KEY': 'rag.web.search.exa_api_key',
|
'EXA_API_KEY': 'rag.web.search.exa_api_key',
|
||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
|
||||||
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
|
||||||
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
|
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
|
||||||
'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
|
'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
|
||||||
'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
|
'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
|
||||||
@@ -632,6 +633,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
|
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
|
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
||||||
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
||||||
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
||||||
@@ -846,6 +848,7 @@ class ConfigForm(BaseModel):
|
|||||||
|
|
||||||
EXTERNAL_DOCUMENT_LOADER_URL: str | None = None
|
EXTERNAL_DOCUMENT_LOADER_URL: str | None = None
|
||||||
EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None
|
EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None
|
||||||
|
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
|
||||||
|
|
||||||
TIKA_SERVER_URL: str | None = None
|
TIKA_SERVER_URL: str | None = None
|
||||||
DOCLING_SERVER_URL: str | None = None
|
DOCLING_SERVER_URL: str | None = None
|
||||||
@@ -1021,6 +1024,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None
|
if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None
|
||||||
else config.EXTERNAL_DOCUMENT_LOADER_API_KEY
|
else config.EXTERNAL_DOCUMENT_LOADER_API_KEY
|
||||||
)
|
)
|
||||||
|
config.EXTERNAL_DOCUMENT_LOADER_HEADERS = (
|
||||||
|
form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS
|
||||||
|
if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None
|
||||||
|
else config.EXTERNAL_DOCUMENT_LOADER_HEADERS
|
||||||
|
)
|
||||||
config.TIKA_SERVER_URL = (
|
config.TIKA_SERVER_URL = (
|
||||||
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
|
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
|
||||||
)
|
)
|
||||||
@@ -1336,6 +1344,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||||||
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
|
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
|
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
|
||||||
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||||
|
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
|
||||||
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
|
||||||
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
|
||||||
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
'DOCLING_API_KEY': config.DOCLING_API_KEY,
|
||||||
@@ -1843,6 +1852,11 @@ async def process_file(
|
|||||||
loader_config = await get_loader_config()
|
loader_config = await get_loader_config()
|
||||||
loader = build_loader_from_config(request, loader_config)
|
loader = build_loader_from_config(request, loader_config)
|
||||||
loader.user = user
|
loader.user = user
|
||||||
|
loader.metadata = {
|
||||||
|
'file_id': file.id,
|
||||||
|
'file_name': file.filename,
|
||||||
|
'file_content_type': file.meta.get('content_type'),
|
||||||
|
}
|
||||||
docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path)
|
docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path)
|
||||||
|
|
||||||
docs = [
|
docs = [
|
||||||
|
|||||||
@@ -514,21 +514,14 @@ async def execute_code(
|
|||||||
|
|
||||||
elif engine == 'jupyter':
|
elif engine == 'jupyter':
|
||||||
from open_webui.utils.code_interpreter import execute_code_jupyter
|
from open_webui.utils.code_interpreter import execute_code_jupyter
|
||||||
|
|
||||||
jupyter_auth = await Config.get('code_interpreter.jupyter.auth')
|
jupyter_auth = await Config.get('code_interpreter.jupyter.auth')
|
||||||
|
|
||||||
output = await execute_code_jupyter(
|
output = await execute_code_jupyter(
|
||||||
await Config.get('code_interpreter.jupyter.url'),
|
await Config.get('code_interpreter.jupyter.url'),
|
||||||
code,
|
code,
|
||||||
(
|
(await Config.get('code_interpreter.jupyter.auth_token') if jupyter_auth == 'token' else None),
|
||||||
await Config.get('code_interpreter.jupyter.auth_token')
|
(await Config.get('code_interpreter.jupyter.auth_password') if jupyter_auth == 'password' else None),
|
||||||
if jupyter_auth == 'token'
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
(
|
|
||||||
await Config.get('code_interpreter.jupyter.auth_password')
|
|
||||||
if jupyter_auth == 'password'
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
await Config.get('code_interpreter.jupyter.timeout'),
|
await Config.get('code_interpreter.jupyter.timeout'),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2385,6 +2378,7 @@ async def query_knowledge_files(
|
|||||||
from open_webui.models.files import Files
|
from open_webui.models.files import Files
|
||||||
from open_webui.models.knowledge import Knowledges
|
from open_webui.models.knowledge import Knowledges
|
||||||
from open_webui.models.notes import Notes
|
from open_webui.models.notes import Notes
|
||||||
|
from open_webui.retrieval.external import retrieve_external_knowledge
|
||||||
from open_webui.retrieval.utils import query_collection
|
from open_webui.retrieval.utils import query_collection
|
||||||
|
|
||||||
user_id = __user__.get('id')
|
user_id = __user__.get('id')
|
||||||
@@ -2396,6 +2390,7 @@ async def query_knowledge_files(
|
|||||||
return json.dumps({'error': 'Embedding function not configured'})
|
return json.dumps({'error': 'Embedding function not configured'})
|
||||||
|
|
||||||
collection_names = []
|
collection_names = []
|
||||||
|
external_knowledges = []
|
||||||
note_results = [] # Notes aren't vectorized, handle separately
|
note_results = [] # Notes aren't vectorized, handle separately
|
||||||
|
|
||||||
# If model has attached knowledge, use those
|
# If model has attached knowledge, use those
|
||||||
@@ -2418,7 +2413,10 @@ async def query_knowledge_files(
|
|||||||
user_group_ids=set(user_group_ids),
|
user_group_ids=set(user_group_ids),
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
collection_names.append(item_id)
|
if (knowledge.meta or {}).get('source') == 'external':
|
||||||
|
external_knowledges.append(knowledge)
|
||||||
|
else:
|
||||||
|
collection_names.append(item_id)
|
||||||
|
|
||||||
elif item_type == 'file':
|
elif item_type == 'file':
|
||||||
# Individual file - use file-{id} as collection name
|
# Individual file - use file-{id} as collection name
|
||||||
@@ -2464,7 +2462,10 @@ async def query_knowledge_files(
|
|||||||
user_group_ids=set(user_group_ids),
|
user_group_ids=set(user_group_ids),
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
collection_names.append(knowledge_id)
|
if (knowledge.meta or {}).get('source') == 'external':
|
||||||
|
external_knowledges.append(knowledge)
|
||||||
|
else:
|
||||||
|
collection_names.append(knowledge_id)
|
||||||
else:
|
else:
|
||||||
# No model knowledge and no specific IDs - search all accessible KBs
|
# No model knowledge and no specific IDs - search all accessible KBs
|
||||||
result = await Knowledges.search_knowledge_bases(
|
result = await Knowledges.search_knowledge_bases(
|
||||||
@@ -2477,7 +2478,11 @@ async def query_knowledge_files(
|
|||||||
skip=0,
|
skip=0,
|
||||||
limit=50,
|
limit=50,
|
||||||
)
|
)
|
||||||
collection_names = [knowledge_base.id for knowledge_base in result.items]
|
for knowledge_base in result.items:
|
||||||
|
if (knowledge_base.meta or {}).get('source') == 'external':
|
||||||
|
external_knowledges.append(knowledge_base)
|
||||||
|
else:
|
||||||
|
collection_names.append(knowledge_base.id)
|
||||||
|
|
||||||
chunks = []
|
chunks = []
|
||||||
|
|
||||||
@@ -2509,6 +2514,31 @@ async def query_knowledge_files(
|
|||||||
chunk_info['distance'] = distances[idx]
|
chunk_info['distance'] = distances[idx]
|
||||||
chunks.append(chunk_info)
|
chunks.append(chunk_info)
|
||||||
|
|
||||||
|
for knowledge in external_knowledges:
|
||||||
|
query_results = await retrieve_external_knowledge(
|
||||||
|
__request__,
|
||||||
|
knowledge,
|
||||||
|
queries=[query],
|
||||||
|
count=count,
|
||||||
|
user=type('UserContext', (), {'id': user_id, 'role': user_role})(),
|
||||||
|
)
|
||||||
|
documents = query_results.get('documents', [[]])[0]
|
||||||
|
metadatas = query_results.get('metadatas', [[]])[0]
|
||||||
|
distances = query_results.get('distances', [[]])[0]
|
||||||
|
|
||||||
|
for idx, doc in enumerate(documents):
|
||||||
|
metadata = metadatas[idx] if idx < len(metadatas) else {}
|
||||||
|
chunk_info = {
|
||||||
|
'content': doc,
|
||||||
|
'source': metadata.get('source', metadata.get('name', knowledge.name)),
|
||||||
|
'file_id': metadata.get('file_id', f'external-{knowledge.id}'),
|
||||||
|
'type': 'external',
|
||||||
|
'knowledge_id': knowledge.id,
|
||||||
|
}
|
||||||
|
if idx < len(distances):
|
||||||
|
chunk_info['distance'] = distances[idx]
|
||||||
|
chunks.append(chunk_info)
|
||||||
|
|
||||||
# Limit to requested count
|
# Limit to requested count
|
||||||
chunks = chunks[:count]
|
chunks = chunks[:count]
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,304 @@ export const createNewKnowledge = async (
|
|||||||
return res;
|
return res;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getExternalKnowledgeConnections = async (token: string) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createExternalKnowledgeConnection = async (token: string, connection: object) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(connection)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const updateExternalKnowledgeConnection = async (
|
||||||
|
token: string,
|
||||||
|
id: string,
|
||||||
|
connection: object
|
||||||
|
) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(connection)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deleteExternalKnowledgeConnection = async (token: string, id: string) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const testExternalKnowledgeConnection = async (token: string, id: string) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/test`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const testExternalKnowledgeRetrieval = async (
|
||||||
|
token: string,
|
||||||
|
id: string,
|
||||||
|
payload: object
|
||||||
|
) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(
|
||||||
|
`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/retrieve-test`,
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const testExternalKnowledgeSource = async (token: string, payload: object) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/test`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createExternalKnowledgeSource = async (token: string, payload: object) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/create`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const updateExternalKnowledgeSource = async (token: string, id: string, payload: object) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/${id}`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createExternalKnowledge = async (token: string, payload: object) => {
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/knowledge/create`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
authorization: `Bearer ${token}`
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
})
|
||||||
|
.then(async (res) => {
|
||||||
|
if (!res.ok) throw await res.json();
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
error = err.detail;
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
export const getKnowledgeBases = async (token: string = '', page: number | null = null) => {
|
export const getKnowledgeBases = async (token: string = '', page: number | null = null) => {
|
||||||
let error = null;
|
let error = null;
|
||||||
|
|
||||||
@@ -76,13 +374,15 @@ export const searchKnowledgeBases = async (
|
|||||||
token: string = '',
|
token: string = '',
|
||||||
query: string | null = null,
|
query: string | null = null,
|
||||||
viewOption: string | null = null,
|
viewOption: string | null = null,
|
||||||
page: number | null = null
|
page: number | null = null,
|
||||||
|
source: string | null = null
|
||||||
) => {
|
) => {
|
||||||
let error = null;
|
let error = null;
|
||||||
|
|
||||||
const searchParams = new URLSearchParams();
|
const searchParams = new URLSearchParams();
|
||||||
if (query) searchParams.append('query', query);
|
if (query) searchParams.append('query', query);
|
||||||
if (viewOption) searchParams.append('view_option', viewOption);
|
if (viewOption) searchParams.append('view_option', viewOption);
|
||||||
|
if (source) searchParams.append('source', source);
|
||||||
if (page) searchParams.append('page', page.toString());
|
if (page) searchParams.append('page', page.toString());
|
||||||
|
|
||||||
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, {
|
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, {
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ type RAGConfigForm = {
|
|||||||
PDF_EXTRACT_IMAGES?: boolean;
|
PDF_EXTRACT_IMAGES?: boolean;
|
||||||
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
|
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
|
||||||
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
|
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
|
||||||
|
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
|
||||||
chunk?: ChunkConfigForm;
|
chunk?: ChunkConfigForm;
|
||||||
content_extraction?: ContentExtractConfigForm;
|
content_extraction?: ContentExtractConfigForm;
|
||||||
web_loader_ssl_verification?: boolean;
|
web_loader_ssl_verification?: boolean;
|
||||||
|
|||||||
@@ -155,7 +155,21 @@
|
|||||||
id: 'integrations',
|
id: 'integrations',
|
||||||
title: 'Integrations',
|
title: 'Integrations',
|
||||||
route: '/admin/settings/integrations',
|
route: '/admin/settings/integrations',
|
||||||
keywords: ['tools', 'integrations', 'plugins', 'extensions', 'functions', 'openapi', 'server']
|
keywords: [
|
||||||
|
'tools',
|
||||||
|
'integrations',
|
||||||
|
'plugins',
|
||||||
|
'extensions',
|
||||||
|
'functions',
|
||||||
|
'openapi',
|
||||||
|
'server',
|
||||||
|
'knowledge',
|
||||||
|
'vector db',
|
||||||
|
'qdrant',
|
||||||
|
'rag',
|
||||||
|
'retrieval',
|
||||||
|
'sources'
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'documents',
|
id: 'documents',
|
||||||
|
|||||||
@@ -37,6 +37,7 @@
|
|||||||
let showResetConfirm = false;
|
let showResetConfirm = false;
|
||||||
let showResetUploadDirConfirm = false;
|
let showResetUploadDirConfirm = false;
|
||||||
let showReindexConfirm = false;
|
let showReindexConfirm = false;
|
||||||
|
let showExternalDocumentLoaderHeadersHint = false;
|
||||||
|
|
||||||
let RAG_EMBEDDING_ENGINE = '';
|
let RAG_EMBEDDING_ENGINE = '';
|
||||||
let RAG_EMBEDDING_MODEL = '';
|
let RAG_EMBEDDING_MODEL = '';
|
||||||
@@ -149,6 +150,21 @@
|
|||||||
toast.error($i18n.t('External Document Loader URL required.'));
|
toast.error($i18n.t('External Document Loader URL required.'));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external' &&
|
||||||
|
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const headers = JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS);
|
||||||
|
if (headers === null || typeof headers !== 'object' || Array.isArray(headers)) {
|
||||||
|
throw new Error('Headers must be a valid JSON object');
|
||||||
|
}
|
||||||
|
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS = JSON.stringify(headers, null, 2);
|
||||||
|
} catch (error) {
|
||||||
|
toast.error($i18n.t('Headers must be a valid JSON object'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') {
|
if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') {
|
||||||
toast.error($i18n.t('Tika Server URL required.'));
|
toast.error($i18n.t('Tika Server URL required.'));
|
||||||
return;
|
return;
|
||||||
@@ -241,6 +257,11 @@
|
|||||||
typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== ''
|
typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== ''
|
||||||
? JSON.parse(RAGConfig.DOCLING_PARAMS)
|
? JSON.parse(RAGConfig.DOCLING_PARAMS)
|
||||||
: {},
|
: {},
|
||||||
|
EXTERNAL_DOCUMENT_LOADER_HEADERS:
|
||||||
|
typeof RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'string' &&
|
||||||
|
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS.trim() !== ''
|
||||||
|
? JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS)
|
||||||
|
: {},
|
||||||
MINERU_PARAMS:
|
MINERU_PARAMS:
|
||||||
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
|
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
|
||||||
? JSON.parse(RAGConfig.MINERU_PARAMS)
|
? JSON.parse(RAGConfig.MINERU_PARAMS)
|
||||||
@@ -289,6 +310,13 @@
|
|||||||
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
|
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
|
||||||
: config.MINERU_PARAMS;
|
: config.MINERU_PARAMS;
|
||||||
|
|
||||||
|
config.EXTERNAL_DOCUMENT_LOADER_HEADERS =
|
||||||
|
typeof config.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'object'
|
||||||
|
? Object.keys(config.EXTERNAL_DOCUMENT_LOADER_HEADERS ?? {}).length > 0
|
||||||
|
? JSON.stringify(config.EXTERNAL_DOCUMENT_LOADER_HEADERS, null, 2)
|
||||||
|
: ''
|
||||||
|
: config.EXTERNAL_DOCUMENT_LOADER_HEADERS;
|
||||||
|
|
||||||
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
|
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
|
||||||
|
|
||||||
RAGConfig = config;
|
RAGConfig = config;
|
||||||
@@ -581,17 +609,78 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
|
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
|
||||||
<div class="my-0.5 flex gap-2 pr-2">
|
<div class="my-0.5 flex flex-col gap-2 pr-2">
|
||||||
<input
|
<div class="flex gap-2">
|
||||||
class="flex-1 w-full text-sm bg-transparent outline-hidden"
|
<input
|
||||||
placeholder={$i18n.t('Enter External Document Loader URL')}
|
class="flex-1 w-full text-sm bg-transparent outline-hidden"
|
||||||
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL}
|
placeholder={$i18n.t('Enter External Document Loader URL')}
|
||||||
/>
|
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL}
|
||||||
<SensitiveInput
|
/>
|
||||||
placeholder={$i18n.t('Enter External Document Loader API Key')}
|
<SensitiveInput
|
||||||
required={false}
|
placeholder={$i18n.t('Enter External Document Loader API Key')}
|
||||||
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY}
|
required={false}
|
||||||
/>
|
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-col">
|
||||||
|
<div class="mb-0.5 text-xs text-gray-500">{$i18n.t('Headers')}</div>
|
||||||
|
<Tooltip
|
||||||
|
content={$i18n.t(
|
||||||
|
'Enter additional headers in JSON format (e.g. {"X-Custom-Header": "value"}'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<Textarea
|
||||||
|
className="w-full text-sm outline-hidden"
|
||||||
|
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS}
|
||||||
|
placeholder={$i18n.t('Enter additional headers in JSON format')}
|
||||||
|
required={false}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="mt-1 flex items-center gap-1 text-xs text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 transition w-fit"
|
||||||
|
on:click={() =>
|
||||||
|
(showExternalDocumentLoaderHeadersHint =
|
||||||
|
!showExternalDocumentLoaderHeadersHint)}
|
||||||
|
>
|
||||||
|
<svg
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 20 20"
|
||||||
|
fill="currentColor"
|
||||||
|
class="w-3 h-3 transition-transform {showExternalDocumentLoaderHeadersHint
|
||||||
|
? 'rotate-90'
|
||||||
|
: ''}"
|
||||||
|
>
|
||||||
|
<path
|
||||||
|
fill-rule="evenodd"
|
||||||
|
d="M7.21 14.77a.75.75 0 01.02-1.06L11.168 10 7.23 6.29a.75.75 0 111.04-1.08l4.5 4.25a.75.75 0 010 1.08l-4.5 4.25a.75.75 0 01-1.06-.02z"
|
||||||
|
clip-rule="evenodd"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
{$i18n.t('Header variables')}
|
||||||
|
</button>
|
||||||
|
{#if showExternalDocumentLoaderHeadersHint}
|
||||||
|
<div class="mt-1 text-xs text-gray-500 dark:text-gray-400 leading-5">
|
||||||
|
<div>
|
||||||
|
{$i18n.t('No additional headers are sent unless configured.')}
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
{$i18n.t('Example')}:
|
||||||
|
<code class="text-gray-700 dark:text-gray-300"
|
||||||
|
>{'{"X-OpenWebUI-File-Id": "{{FILE_ID}}"}'}</code
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
{$i18n.t('Available variables')}:
|
||||||
|
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_ID}}'}</code>,
|
||||||
|
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_NAME}}'}</code>,
|
||||||
|
<code class="text-gray-700 dark:text-gray-300"
|
||||||
|
>{'{{FILE_CONTENT_TYPE}}'}</code
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
|
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
|
||||||
<div class="flex w-full mt-1">
|
<div class="flex w-full mt-1">
|
||||||
|
|||||||
@@ -0,0 +1,883 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { getContext, onMount } from 'svelte';
|
||||||
|
import type { Writable } from 'svelte/store';
|
||||||
|
import type { i18n as i18nType } from 'i18next';
|
||||||
|
import { toast } from 'svelte-sonner';
|
||||||
|
|
||||||
|
import AccessControl from '$lib/components/workspace/common/AccessControl.svelte';
|
||||||
|
import Modal from '$lib/components/common/Modal.svelte';
|
||||||
|
import Spinner from '$lib/components/common/Spinner.svelte';
|
||||||
|
import Switch from '$lib/components/common/Switch.svelte';
|
||||||
|
import Tooltip from '$lib/components/common/Tooltip.svelte';
|
||||||
|
import Cog6 from '$lib/components/icons/Cog6.svelte';
|
||||||
|
import DatabaseSettings from '$lib/components/icons/DatabaseSettings.svelte';
|
||||||
|
import Plus from '$lib/components/icons/Plus.svelte';
|
||||||
|
import XMark from '$lib/components/icons/XMark.svelte';
|
||||||
|
import {
|
||||||
|
createExternalKnowledgeSource,
|
||||||
|
getExternalKnowledgeConnections,
|
||||||
|
searchKnowledgeBases,
|
||||||
|
testExternalKnowledgeSource,
|
||||||
|
updateExternalKnowledgeConnection,
|
||||||
|
updateExternalKnowledgeSource
|
||||||
|
} from '$lib/apis/knowledge';
|
||||||
|
|
||||||
|
const i18n = getContext<Writable<i18nType>>('i18n');
|
||||||
|
|
||||||
|
type ExternalKnowledgeConnection = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
provider: string;
|
||||||
|
endpoint: string;
|
||||||
|
config?: Record<string, any>;
|
||||||
|
capabilities?: Record<string, any>;
|
||||||
|
enabled?: boolean;
|
||||||
|
auth_configured?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ExternalKnowledgeItem = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
access_grants?: any[];
|
||||||
|
meta?: {
|
||||||
|
external?: {
|
||||||
|
connection_id?: string;
|
||||||
|
provider?: string;
|
||||||
|
source?: {
|
||||||
|
name?: string;
|
||||||
|
config?: Record<string, any>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let loading = false;
|
||||||
|
let testing = false;
|
||||||
|
let creating = false;
|
||||||
|
let showSourceModal = false;
|
||||||
|
let connections: ExternalKnowledgeConnection[] = [];
|
||||||
|
let items: ExternalKnowledgeItem[] = [];
|
||||||
|
let editingItem: ExternalKnowledgeItem | null = null;
|
||||||
|
let editingConnection: ExternalKnowledgeConnection | null = null;
|
||||||
|
let accessGrants: any[] = [];
|
||||||
|
let testResult: {
|
||||||
|
documents?: string[];
|
||||||
|
metadatas?: Record<string, any>[];
|
||||||
|
distances?: number[];
|
||||||
|
} | null = null;
|
||||||
|
|
||||||
|
let sourceForm = {
|
||||||
|
name: '',
|
||||||
|
description: '',
|
||||||
|
provider: 'qdrant',
|
||||||
|
endpoint: '',
|
||||||
|
apiKey: '',
|
||||||
|
timeout: 30,
|
||||||
|
dbName: '',
|
||||||
|
sourceName: '',
|
||||||
|
contentField: 'payload.text',
|
||||||
|
vectorField: '',
|
||||||
|
metadataField: 'payload.metadata',
|
||||||
|
documentIdField: 'id',
|
||||||
|
tableName: 'document_chunk',
|
||||||
|
collectionField: 'collection_name',
|
||||||
|
testQuery: ''
|
||||||
|
};
|
||||||
|
|
||||||
|
const schemaDefaults = (provider = sourceForm.provider) => {
|
||||||
|
if (provider === 'milvus') {
|
||||||
|
return {
|
||||||
|
contentField: 'data.text',
|
||||||
|
vectorField: 'vector',
|
||||||
|
metadataField: 'metadata',
|
||||||
|
documentIdField: 'id',
|
||||||
|
tableName: 'document_chunk',
|
||||||
|
collectionField: 'collection_name'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (provider === 'pgvector') {
|
||||||
|
return {
|
||||||
|
contentField: 'text',
|
||||||
|
vectorField: 'vector',
|
||||||
|
metadataField: 'vmetadata',
|
||||||
|
documentIdField: 'id',
|
||||||
|
tableName: 'document_chunk',
|
||||||
|
collectionField: 'collection_name'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
contentField: 'payload.text',
|
||||||
|
vectorField: '',
|
||||||
|
metadataField: 'payload.metadata',
|
||||||
|
documentIdField: 'id',
|
||||||
|
tableName: 'document_chunk',
|
||||||
|
collectionField: 'collection_name'
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const resetForm = () => {
|
||||||
|
sourceForm = {
|
||||||
|
name: '',
|
||||||
|
description: '',
|
||||||
|
provider: 'qdrant',
|
||||||
|
endpoint: '',
|
||||||
|
apiKey: '',
|
||||||
|
timeout: 30,
|
||||||
|
dbName: '',
|
||||||
|
sourceName: '',
|
||||||
|
testQuery: '',
|
||||||
|
...schemaDefaults('qdrant')
|
||||||
|
};
|
||||||
|
accessGrants = [];
|
||||||
|
testResult = null;
|
||||||
|
editingItem = null;
|
||||||
|
editingConnection = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const openCreateSource = () => {
|
||||||
|
resetForm();
|
||||||
|
showSourceModal = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const openEditSource = (item: ExternalKnowledgeItem) => {
|
||||||
|
const connection = connectionForItem(item);
|
||||||
|
if (!connection) {
|
||||||
|
toast.error($i18n.t('External connection not found.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
resetForm();
|
||||||
|
editingItem = item;
|
||||||
|
editingConnection = connection;
|
||||||
|
|
||||||
|
const source = item.meta?.external?.source;
|
||||||
|
const sourceConfig = source?.config ?? {};
|
||||||
|
const defaults = schemaDefaults(connection.provider);
|
||||||
|
sourceForm = {
|
||||||
|
name: item.name ?? '',
|
||||||
|
description: item.description ?? '',
|
||||||
|
provider: connection.provider ?? 'qdrant',
|
||||||
|
endpoint: connection.endpoint ?? '',
|
||||||
|
apiKey: '',
|
||||||
|
timeout: connection.config?.timeout ?? 30,
|
||||||
|
dbName: connection.config?.db_name ?? '',
|
||||||
|
sourceName: source?.name ?? '',
|
||||||
|
testQuery: '',
|
||||||
|
contentField: sourceConfig.content_field ?? defaults.contentField,
|
||||||
|
vectorField: sourceConfig.vector_field ?? defaults.vectorField,
|
||||||
|
metadataField: sourceConfig.metadata_field ?? defaults.metadataField,
|
||||||
|
documentIdField: sourceConfig.document_id_field ?? defaults.documentIdField,
|
||||||
|
tableName: sourceConfig.table_name ?? defaults.tableName,
|
||||||
|
collectionField: sourceConfig.collection_field ?? defaults.collectionField
|
||||||
|
};
|
||||||
|
accessGrants = item.access_grants ?? [];
|
||||||
|
showSourceModal = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const markUntested = () => {
|
||||||
|
testResult = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const refresh = async () => {
|
||||||
|
loading = true;
|
||||||
|
const [connectionRes, knowledgeRes] = await Promise.all([
|
||||||
|
getExternalKnowledgeConnections(localStorage.token).catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
}),
|
||||||
|
searchKnowledgeBases(localStorage.token, null, null, 1, 'external').catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
connections = connectionRes?.items ?? [];
|
||||||
|
items = knowledgeRes?.items ?? [];
|
||||||
|
loading = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const endpointPlaceholder = () => {
|
||||||
|
if (sourceForm.provider === 'pgvector') {
|
||||||
|
return 'postgresql://user:password@host:5432/db';
|
||||||
|
}
|
||||||
|
if (sourceForm.provider === 'milvus') {
|
||||||
|
return 'http://milvus.example.com:19530';
|
||||||
|
}
|
||||||
|
return 'https://qdrant.example.com';
|
||||||
|
};
|
||||||
|
|
||||||
|
const connectionForItem = (item: ExternalKnowledgeItem) =>
|
||||||
|
connections.find((connection) => connection.id === item?.meta?.external?.connection_id);
|
||||||
|
|
||||||
|
const connectionPayload = () => ({
|
||||||
|
name:
|
||||||
|
sourceForm.name.trim() ||
|
||||||
|
sourceForm.sourceName.trim() ||
|
||||||
|
$i18n.t('External Knowledge Source'),
|
||||||
|
provider: sourceForm.provider,
|
||||||
|
endpoint: sourceForm.endpoint,
|
||||||
|
auth_config:
|
||||||
|
sourceForm.provider === 'pgvector'
|
||||||
|
? {}
|
||||||
|
: sourceForm.apiKey
|
||||||
|
? { type: 'bearer', api_key: sourceForm.apiKey }
|
||||||
|
: editingConnection
|
||||||
|
? null
|
||||||
|
: {},
|
||||||
|
config: {
|
||||||
|
timeout: Number(sourceForm.timeout) || 30,
|
||||||
|
...(sourceForm.provider === 'milvus' && sourceForm.dbName
|
||||||
|
? { db_name: sourceForm.dbName }
|
||||||
|
: {})
|
||||||
|
},
|
||||||
|
capabilities: { retrieve: true },
|
||||||
|
enabled: editingConnection?.enabled !== false
|
||||||
|
});
|
||||||
|
|
||||||
|
const sourcePayload = () => {
|
||||||
|
const config: Record<string, string> = {
|
||||||
|
content_field: sourceForm.contentField.trim(),
|
||||||
|
...(sourceForm.vectorField.trim() ? { vector_field: sourceForm.vectorField.trim() } : {}),
|
||||||
|
...(sourceForm.metadataField.trim()
|
||||||
|
? { metadata_field: sourceForm.metadataField.trim() }
|
||||||
|
: {}),
|
||||||
|
...(sourceForm.documentIdField.trim()
|
||||||
|
? { document_id_field: sourceForm.documentIdField.trim() }
|
||||||
|
: {})
|
||||||
|
};
|
||||||
|
|
||||||
|
if (sourceForm.provider === 'pgvector') {
|
||||||
|
config.table_name = sourceForm.tableName.trim();
|
||||||
|
config.collection_field = sourceForm.collectionField.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: 'collection',
|
||||||
|
name: sourceForm.sourceName.trim(),
|
||||||
|
config
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const testFormIsValid = () =>
|
||||||
|
!!sourceForm.endpoint.trim() &&
|
||||||
|
!!sourceForm.sourceName.trim() &&
|
||||||
|
!!sourceForm.contentField.trim() &&
|
||||||
|
(sourceForm.provider === 'qdrant' || !!sourceForm.vectorField.trim()) &&
|
||||||
|
(sourceForm.provider !== 'pgvector' ||
|
||||||
|
(!!sourceForm.tableName.trim() && !!sourceForm.collectionField.trim())) &&
|
||||||
|
!!sourceForm.testQuery.trim();
|
||||||
|
|
||||||
|
const sourceFormIsValid = () => !!sourceForm.name.trim() && testFormIsValid();
|
||||||
|
|
||||||
|
const testSource = async () => {
|
||||||
|
if (!testFormIsValid()) {
|
||||||
|
toast.error($i18n.t('Fill the source fields and test query first.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
testing = true;
|
||||||
|
testResult = null;
|
||||||
|
const res = await testExternalKnowledgeSource(localStorage.token, {
|
||||||
|
...(editingConnection?.id ? { connection_id: editingConnection.id } : {}),
|
||||||
|
connection: connectionPayload(),
|
||||||
|
source: sourcePayload(),
|
||||||
|
query: sourceForm.testQuery,
|
||||||
|
count: 5
|
||||||
|
}).catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res?.documents?.length) {
|
||||||
|
testResult = res;
|
||||||
|
toast.success($i18n.t('Test succeeded.'));
|
||||||
|
} else if (res) {
|
||||||
|
toast.error($i18n.t('Test returned no results.'));
|
||||||
|
}
|
||||||
|
testing = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createSource = async () => {
|
||||||
|
if (!sourceFormIsValid()) {
|
||||||
|
toast.error($i18n.t('Fill the required fields first.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!testResult?.documents?.length) {
|
||||||
|
toast.error($i18n.t('Test the source before creating it.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
creating = true;
|
||||||
|
const res = await createExternalKnowledgeSource(localStorage.token, {
|
||||||
|
name: sourceForm.name,
|
||||||
|
description: sourceForm.description,
|
||||||
|
connection: connectionPayload(),
|
||||||
|
source: sourcePayload(),
|
||||||
|
access_grants: accessGrants,
|
||||||
|
test_query: sourceForm.testQuery,
|
||||||
|
test_count: 5
|
||||||
|
}).catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res) {
|
||||||
|
toast.success($i18n.t('Knowledge source created.'));
|
||||||
|
showSourceModal = false;
|
||||||
|
resetForm();
|
||||||
|
await refresh();
|
||||||
|
}
|
||||||
|
creating = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateSource = async () => {
|
||||||
|
if (!editingItem) return;
|
||||||
|
|
||||||
|
if (!sourceFormIsValid()) {
|
||||||
|
toast.error($i18n.t('Fill the required fields first.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!testResult?.documents?.length) {
|
||||||
|
toast.error($i18n.t('Test the source before saving it.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
creating = true;
|
||||||
|
const res = await updateExternalKnowledgeSource(localStorage.token, editingItem.id, {
|
||||||
|
name: sourceForm.name,
|
||||||
|
description: sourceForm.description,
|
||||||
|
connection: connectionPayload(),
|
||||||
|
source: sourcePayload(),
|
||||||
|
access_grants: accessGrants,
|
||||||
|
test_query: sourceForm.testQuery,
|
||||||
|
test_count: 5
|
||||||
|
}).catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res) {
|
||||||
|
toast.success($i18n.t('Knowledge source updated.'));
|
||||||
|
showSourceModal = false;
|
||||||
|
resetForm();
|
||||||
|
await refresh();
|
||||||
|
}
|
||||||
|
creating = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const submitSource = async () => {
|
||||||
|
if (editingItem) {
|
||||||
|
await updateSource();
|
||||||
|
} else {
|
||||||
|
await createSource();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const toggleSource = async (item: ExternalKnowledgeItem) => {
|
||||||
|
const connection = connectionForItem(item);
|
||||||
|
if (!connection) return;
|
||||||
|
|
||||||
|
const res = await updateExternalKnowledgeConnection(localStorage.token, connection.id, {
|
||||||
|
name: connection.name,
|
||||||
|
provider: connection.provider,
|
||||||
|
endpoint: connection.endpoint,
|
||||||
|
auth_config: null,
|
||||||
|
config: connection.config ?? {},
|
||||||
|
capabilities: connection.capabilities ?? { retrieve: true },
|
||||||
|
enabled: !connection.enabled
|
||||||
|
}).catch((error) => {
|
||||||
|
toast.error(`${error}`);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res) {
|
||||||
|
await refresh();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateProvider = () => {
|
||||||
|
sourceForm = {
|
||||||
|
...sourceForm,
|
||||||
|
...schemaDefaults(sourceForm.provider)
|
||||||
|
};
|
||||||
|
markUntested();
|
||||||
|
};
|
||||||
|
|
||||||
|
onMount(refresh);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<Modal bind:show={showSourceModal} size="sm">
|
||||||
|
<div>
|
||||||
|
<div class="flex justify-between dark:text-gray-100 px-5 pt-4 pb-2">
|
||||||
|
<h1 class="text-lg font-medium self-center font-primary">
|
||||||
|
{editingItem ? $i18n.t('Edit Knowledge Connection') : $i18n.t('Add Knowledge Connection')}
|
||||||
|
</h1>
|
||||||
|
|
||||||
|
<button
|
||||||
|
class="self-center"
|
||||||
|
aria-label={$i18n.t('Close')}
|
||||||
|
type="button"
|
||||||
|
on:click={() => {
|
||||||
|
showSourceModal = false;
|
||||||
|
resetForm();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<XMark className="size-5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col md:flex-row w-full px-4 pb-4 md:space-x-4 dark:text-gray-200">
|
||||||
|
<div class="flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
|
||||||
|
<form class="flex flex-col w-full" on:submit|preventDefault={submitSource}>
|
||||||
|
<div class="px-1">
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-name"
|
||||||
|
>{$i18n.t('Name')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-name"
|
||||||
|
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.name}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={$i18n.t('Research Knowledge')}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-provider"
|
||||||
|
>{$i18n.t('Provider')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<select
|
||||||
|
id="external-source-provider"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden"
|
||||||
|
bind:value={sourceForm.provider}
|
||||||
|
on:change={updateProvider}
|
||||||
|
>
|
||||||
|
<option value="qdrant">Qdrant</option>
|
||||||
|
<option value="milvus">Milvus</option>
|
||||||
|
<option value="pgvector">pgvector</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col w-full mt-2">
|
||||||
|
<label class="text-xs text-gray-500 mb-0.5" for="external-source-description"
|
||||||
|
>{$i18n.t('Description')}</label
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
id="external-source-description"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden resize-none placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
rows="2"
|
||||||
|
bind:value={sourceForm.description}
|
||||||
|
></textarea>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-endpoint"
|
||||||
|
>{$i18n.t('Endpoint')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-endpoint"
|
||||||
|
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.endpoint}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={endpointPlaceholder()}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-timeout"
|
||||||
|
>{$i18n.t('Timeout')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-timeout"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden"
|
||||||
|
type="number"
|
||||||
|
bind:value={sourceForm.timeout}
|
||||||
|
on:input={markUntested}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if sourceForm.provider !== 'pgvector'}
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-api-key"
|
||||||
|
>{$i18n.t('API Key / Token')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-api-key"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
type="password"
|
||||||
|
bind:value={sourceForm.apiKey}
|
||||||
|
on:input={markUntested}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if sourceForm.provider === 'milvus'}
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col w-full">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-db-name"
|
||||||
|
>{$i18n.t('Database')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-db-name"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden"
|
||||||
|
bind:value={sourceForm.dbName}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={$i18n.t('Default')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
|
||||||
|
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<div class="flex flex-col w-full">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-collection"
|
||||||
|
>{$i18n.t('Collection')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-collection"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.sourceName}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder="research-docs"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if sourceForm.provider === 'pgvector'}
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-table"
|
||||||
|
>{$i18n.t('Table')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-table"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.tableName}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder="document_chunk"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-collection-field"
|
||||||
|
>{$i18n.t('Collection Field')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-collection-field"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.collectionField}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder="collection_name"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-content-field"
|
||||||
|
>{$i18n.t('Content Field')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-content-field"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.contentField}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={sourceForm.provider === 'pgvector' ? 'text' : 'payload.text'}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-vector-field">
|
||||||
|
{$i18n.t('Vector Field')}
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-vector-field"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.vectorField}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={sourceForm.provider === 'qdrant' ? $i18n.t('Default') : 'vector'}
|
||||||
|
required={sourceForm.provider !== 'qdrant'}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-metadata-field"
|
||||||
|
>{$i18n.t('Metadata Field')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-metadata-field"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.metadataField}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={sourceForm.provider === 'pgvector'
|
||||||
|
? 'vmetadata'
|
||||||
|
: 'payload.metadata'}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col flex-1">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-document-id-field"
|
||||||
|
>{$i18n.t('Document ID Field')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-document-id-field"
|
||||||
|
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.documentIdField}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder="id"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex gap-2 mt-2">
|
||||||
|
<div class="flex flex-col w-full">
|
||||||
|
<div class="flex justify-between mb-0.5">
|
||||||
|
<label class="text-xs text-gray-500" for="external-source-test-query"
|
||||||
|
>{$i18n.t('Test Query')}</label
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-1 items-center">
|
||||||
|
<input
|
||||||
|
id="external-source-test-query"
|
||||||
|
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
|
||||||
|
bind:value={sourceForm.testQuery}
|
||||||
|
on:input={markUntested}
|
||||||
|
placeholder={$i18n.t('Ask a test question')}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Tooltip
|
||||||
|
content={$i18n.t('Verify Connection')}
|
||||||
|
className="shrink-0 flex items-center mr-1"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition"
|
||||||
|
type="button"
|
||||||
|
on:click={testSource}
|
||||||
|
disabled={testing}
|
||||||
|
aria-label={$i18n.t('Verify Connection')}
|
||||||
|
>
|
||||||
|
{#if testing}
|
||||||
|
<Spinner />
|
||||||
|
{:else}
|
||||||
|
<svg
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 20 20"
|
||||||
|
fill="currentColor"
|
||||||
|
class="w-4 h-4"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<path
|
||||||
|
fill-rule="evenodd"
|
||||||
|
d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
|
||||||
|
clip-rule="evenodd"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</Tooltip>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="text-xs text-gray-500 mt-1">
|
||||||
|
{$i18n.t(
|
||||||
|
'External vectors must be generated with the same embedding model configured in Open WebUI.'
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
|
||||||
|
|
||||||
|
<AccessControl
|
||||||
|
bind:accessGrants
|
||||||
|
accessRoles={['read']}
|
||||||
|
share={true}
|
||||||
|
sharePublic={true}
|
||||||
|
shareUsers={true}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div class="flex justify-between items-center pt-3 text-sm font-medium">
|
||||||
|
<div></div>
|
||||||
|
|
||||||
|
<button
|
||||||
|
class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full flex items-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
type="submit"
|
||||||
|
disabled={creating || !sourceFormIsValid() || !testResult?.documents?.length}
|
||||||
|
>
|
||||||
|
{editingItem ? $i18n.t('Save') : $i18n.t('Create')}
|
||||||
|
{#if creating}<Spinner />{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Modal>
|
||||||
|
|
||||||
|
<div class="mb-2.5 flex flex-col w-full justify-between text-sm">
|
||||||
|
<div class="flex justify-between items-center mb-1">
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<div class="font-medium">{$i18n.t('External Knowledge Sources')}</div>
|
||||||
|
<span
|
||||||
|
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
|
||||||
|
>{$i18n.t('Experimental')}</span
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<Tooltip content={$i18n.t('Add Connection')}>
|
||||||
|
<button class="px-1" on:click={openCreateSource} type="button">
|
||||||
|
<Plus />
|
||||||
|
</button>
|
||||||
|
</Tooltip>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-col gap-1.5">
|
||||||
|
{#each items as item}
|
||||||
|
{@const connection = connectionForItem(item)}
|
||||||
|
<div class="flex w-full gap-2 items-center">
|
||||||
|
<Tooltip className="w-full relative" content={''} placement="top-start">
|
||||||
|
<div class="flex w-full">
|
||||||
|
<div
|
||||||
|
class="flex-1 relative flex gap-1.5 items-center min-w-0 {connection?.enabled ===
|
||||||
|
false
|
||||||
|
? 'opacity-50'
|
||||||
|
: ''}"
|
||||||
|
>
|
||||||
|
<Tooltip content={$i18n.t('Knowledge')}>
|
||||||
|
<DatabaseSettings className="size-4" strokeWidth="1.5" />
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<div class="outline-hidden w-full bg-transparent text-sm min-w-0 line-clamp-1">
|
||||||
|
<span>{item.name}</span>
|
||||||
|
{' '}
|
||||||
|
<span class="text-gray-500">
|
||||||
|
{item?.meta?.external?.provider ?? connection?.provider}
|
||||||
|
{item?.meta?.external?.source?.name ? `· ${item.meta.external.source.name}` : ''}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<div class="flex gap-1 items-center">
|
||||||
|
<Tooltip content={$i18n.t('Configure')}>
|
||||||
|
<button
|
||||||
|
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
type="button"
|
||||||
|
disabled={!connection}
|
||||||
|
aria-label={$i18n.t('Configure')}
|
||||||
|
on:click={() => {
|
||||||
|
openEditSource(item);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Cog6 />
|
||||||
|
</button>
|
||||||
|
</Tooltip>
|
||||||
|
|
||||||
|
<Tooltip
|
||||||
|
content={connection?.enabled !== false ? $i18n.t('Enabled') : $i18n.t('Disabled')}
|
||||||
|
>
|
||||||
|
<Switch
|
||||||
|
state={connection?.enabled !== false}
|
||||||
|
on:change={() => {
|
||||||
|
toggleSource(item);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if loading}
|
||||||
|
<div class="py-2">
|
||||||
|
<Spinner />
|
||||||
|
</div>
|
||||||
|
{:else if items.length === 0}
|
||||||
|
<div class="text-xs text-gray-400 dark:text-gray-500">
|
||||||
|
{$i18n.t('No external knowledge sources configured.')}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<div class="my-1.5">
|
||||||
|
<div class="text-xs text-gray-500">
|
||||||
|
{$i18n.t(
|
||||||
|
'Create one read-only Knowledge source per external collection. Test must pass before the source is created.'
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@@ -1,11 +1,12 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { toast } from 'svelte-sonner';
|
import { toast } from 'svelte-sonner';
|
||||||
import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
|
import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import { getModels as _getModels } from '$lib/apis';
|
import { getModels as _getModels } from '$lib/apis';
|
||||||
|
import type { Writable } from 'svelte/store';
|
||||||
|
import type { i18n as i18nType } from 'i18next';
|
||||||
|
|
||||||
const dispatch = createEventDispatcher();
|
const dispatch = createEventDispatcher();
|
||||||
const i18n = getContext('i18n');
|
const i18n = getContext<Writable<i18nType>>('i18n');
|
||||||
|
|
||||||
import { models, settings, user, terminalServers } from '$lib/stores';
|
import { models, settings, user, terminalServers } from '$lib/stores';
|
||||||
import { getTerminalServers } from '$lib/apis/terminal';
|
import { getTerminalServers } from '$lib/apis/terminal';
|
||||||
@@ -22,6 +23,7 @@
|
|||||||
|
|
||||||
import AddToolServerModal from '$lib/components/AddToolServerModal.svelte';
|
import AddToolServerModal from '$lib/components/AddToolServerModal.svelte';
|
||||||
import AddTerminalServerModal from '$lib/components/AddTerminalServerModal.svelte';
|
import AddTerminalServerModal from '$lib/components/AddTerminalServerModal.svelte';
|
||||||
|
import ExternalKnowledge from './ExternalKnowledge.svelte';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
getToolServerConnections,
|
getToolServerConnections,
|
||||||
@@ -32,16 +34,26 @@
|
|||||||
|
|
||||||
export let saveSettings: Function;
|
export let saveSettings: Function;
|
||||||
|
|
||||||
let servers = null;
|
type ToolServerConnection = any;
|
||||||
|
type TerminalConnection = {
|
||||||
|
id?: string;
|
||||||
|
url?: string;
|
||||||
|
name?: string;
|
||||||
|
key?: string;
|
||||||
|
enabled?: boolean;
|
||||||
|
[key: string]: any;
|
||||||
|
};
|
||||||
|
|
||||||
|
let servers: ToolServerConnection[] | null = null;
|
||||||
let showConnectionModal = false;
|
let showConnectionModal = false;
|
||||||
|
|
||||||
// Terminal server admin connections
|
// Terminal server admin connections
|
||||||
let terminalConnections = [];
|
let terminalConnections: TerminalConnection[] = [];
|
||||||
let showAddTerminalModal = false;
|
let showAddTerminalModal = false;
|
||||||
let editTerminalIdx: number | null = null;
|
let editTerminalIdx: number | null = null;
|
||||||
|
|
||||||
const addConnectionHandler = async (server) => {
|
const addConnectionHandler = async (server: ToolServerConnection) => {
|
||||||
servers = [...servers, server];
|
servers = [...(servers ?? []), server];
|
||||||
await updateHandler();
|
await updateHandler();
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -71,7 +83,9 @@
|
|||||||
|
|
||||||
// Refresh the terminalServers store so changes are reflected immediately
|
// Refresh the terminalServers store so changes are reflected immediately
|
||||||
// Preserve user direct terminals, refresh system terminals from backend
|
// Preserve user direct terminals, refresh system terminals from backend
|
||||||
const existingDirectTerminals = ($terminalServers ?? []).filter((t) => !t.id);
|
const existingDirectTerminals = (($terminalServers ?? []) as TerminalConnection[]).filter(
|
||||||
|
(t) => !t.id
|
||||||
|
);
|
||||||
const systemTerminals = await getTerminalServers(localStorage.token);
|
const systemTerminals = await getTerminalServers(localStorage.token);
|
||||||
const systemEntries = systemTerminals.map((t) => ({
|
const systemEntries = systemTerminals.map((t) => ({
|
||||||
id: t.id,
|
id: t.id,
|
||||||
@@ -79,16 +93,19 @@
|
|||||||
name: t.name,
|
name: t.name,
|
||||||
key: localStorage.token
|
key: localStorage.token
|
||||||
}));
|
}));
|
||||||
terminalServers.set([...existingDirectTerminals, ...systemEntries]);
|
terminalServers.set([...existingDirectTerminals, ...systemEntries] as any);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const addTerminalConnection = (server) => {
|
const addTerminalConnection = (server: TerminalConnection) => {
|
||||||
terminalConnections = [...terminalConnections, { ...server, id: server.id ?? uuidv4() }];
|
terminalConnections = [
|
||||||
|
...terminalConnections,
|
||||||
|
{ ...server, id: server.id ?? crypto.randomUUID() }
|
||||||
|
];
|
||||||
saveTerminalServers();
|
saveTerminalServers();
|
||||||
};
|
};
|
||||||
|
|
||||||
const updateTerminalConnection = (idx: number, updated) => {
|
const updateTerminalConnection = (idx: number, updated: TerminalConnection) => {
|
||||||
terminalConnections = terminalConnections.map((c, i) =>
|
terminalConnections = terminalConnections.map((c, i) =>
|
||||||
i === idx ? { ...c, ...updated, id: updated.id ?? c.id } : c
|
i === idx ? { ...c, ...updated, id: updated.id ?? c.id } : c
|
||||||
);
|
);
|
||||||
@@ -102,13 +119,13 @@
|
|||||||
|
|
||||||
onMount(async () => {
|
onMount(async () => {
|
||||||
const res = await getToolServerConnections(localStorage.token);
|
const res = await getToolServerConnections(localStorage.token);
|
||||||
servers = res.TOOL_SERVER_CONNECTIONS;
|
servers = res.TOOL_SERVER_CONNECTIONS as ToolServerConnection[];
|
||||||
|
|
||||||
// Load terminal server connections
|
// Load terminal server connections
|
||||||
try {
|
try {
|
||||||
const terminalRes = await getTerminalServerConnections(localStorage.token);
|
const terminalRes = await getTerminalServerConnections(localStorage.token);
|
||||||
if (terminalRes?.TERMINAL_SERVER_CONNECTIONS) {
|
if (terminalRes?.TERMINAL_SERVER_CONNECTIONS) {
|
||||||
terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS;
|
terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS as TerminalConnection[];
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Not configured yet
|
// Not configured yet
|
||||||
@@ -122,7 +139,7 @@
|
|||||||
bind:show={showAddTerminalModal}
|
bind:show={showAddTerminalModal}
|
||||||
edit={editTerminalIdx !== null}
|
edit={editTerminalIdx !== null}
|
||||||
connection={editTerminalIdx !== null ? terminalConnections[editTerminalIdx] : null}
|
connection={editTerminalIdx !== null ? terminalConnections[editTerminalIdx] : null}
|
||||||
onSubmit={(c) => {
|
onSubmit={(c: TerminalConnection) => {
|
||||||
if (editTerminalIdx !== null) {
|
if (editTerminalIdx !== null) {
|
||||||
updateTerminalConnection(editTerminalIdx, c);
|
updateTerminalConnection(editTerminalIdx, c);
|
||||||
editTerminalIdx = null;
|
editTerminalIdx = null;
|
||||||
@@ -148,13 +165,13 @@
|
|||||||
{#if servers !== null}
|
{#if servers !== null}
|
||||||
<div class="">
|
<div class="">
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('General')}</div>
|
<div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('Tools')}</div>
|
||||||
|
|
||||||
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
|
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
|
||||||
|
|
||||||
<div class="mb-2.5 flex flex-col w-full justify-between">
|
<div class="mb-2.5 flex flex-col w-full justify-between">
|
||||||
<div class="flex justify-between items-center mb-0.5">
|
<div class="flex justify-between items-center mb-0.5">
|
||||||
<div class="font-medium">{$i18n.t('Manage Tool Servers')}</div>
|
<div class="font-medium">{$i18n.t('External Tool Servers')}</div>
|
||||||
|
|
||||||
<Tooltip content={$i18n.t(`Add Connection`)}>
|
<Tooltip content={$i18n.t(`Add Connection`)}>
|
||||||
<button
|
<button
|
||||||
@@ -170,21 +187,21 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="flex flex-col gap-1">
|
<div class="flex flex-col gap-1">
|
||||||
{#each servers as server, idx}
|
{#each servers ?? [] as server, idx}
|
||||||
<Connection
|
<Connection
|
||||||
bind:connection={server}
|
bind:connection={server}
|
||||||
onSubmit={() => {
|
onSubmit={() => {
|
||||||
updateHandler();
|
updateHandler();
|
||||||
}}
|
}}
|
||||||
onDelete={() => {
|
onDelete={() => {
|
||||||
servers = servers.filter((_, i) => i !== idx);
|
servers = (servers ?? []).filter((_, i) => i !== idx);
|
||||||
updateHandler();
|
updateHandler();
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
{/each}
|
{/each}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{#if servers.length === 0}
|
{#if (servers ?? []).length === 0}
|
||||||
<div class="text-xs text-gray-400 dark:text-gray-500">
|
<div class="text-xs text-gray-400 dark:text-gray-500">
|
||||||
{$i18n.t('No tool server connections configured.')}
|
{$i18n.t('No tool server connections configured.')}
|
||||||
</div>
|
</div>
|
||||||
@@ -197,17 +214,9 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-4" />
|
<div class="mt-4 mb-2.5 flex flex-col w-full">
|
||||||
|
|
||||||
<div class="mb-2.5 flex flex-col w-full">
|
|
||||||
<div class="flex justify-between items-center mb-1">
|
<div class="flex justify-between items-center mb-1">
|
||||||
<div class="flex items-center gap-2">
|
<div class="font-medium">{$i18n.t('Open Terminal')}</div>
|
||||||
<div class="font-medium">{$i18n.t('Open Terminal')}</div>
|
|
||||||
<span
|
|
||||||
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
|
|
||||||
>{$i18n.t('Experimental')}</span
|
|
||||||
>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<Tooltip content={$i18n.t('Add Connection')}>
|
<Tooltip content={$i18n.t('Add Connection')}>
|
||||||
<button
|
<button
|
||||||
@@ -300,6 +309,12 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-8 mb-2.5 text-base font-medium">{$i18n.t('Knowledge')}</div>
|
||||||
|
|
||||||
|
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
|
||||||
|
|
||||||
|
<ExternalKnowledge />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{:else}
|
{:else}
|
||||||
|
|||||||
@@ -5,7 +5,10 @@
|
|||||||
|
|
||||||
import { toast } from 'svelte-sonner';
|
import { toast } from 'svelte-sonner';
|
||||||
import { onMount, getContext, tick, onDestroy } from 'svelte';
|
import { onMount, getContext, tick, onDestroy } from 'svelte';
|
||||||
const i18n = getContext('i18n');
|
import type { Writable } from 'svelte/store';
|
||||||
|
import type { i18n as i18nType } from 'i18next';
|
||||||
|
|
||||||
|
const i18n = getContext<Writable<i18nType>>('i18n');
|
||||||
|
|
||||||
import { WEBUI_NAME, knowledge, user } from '$lib/stores';
|
import { WEBUI_NAME, knowledge, user } from '$lib/stores';
|
||||||
import {
|
import {
|
||||||
@@ -28,19 +31,33 @@
|
|||||||
import ViewSelector from './common/ViewSelector.svelte';
|
import ViewSelector from './common/ViewSelector.svelte';
|
||||||
import Loader from '../common/Loader.svelte';
|
import Loader from '../common/Loader.svelte';
|
||||||
|
|
||||||
|
type KnowledgeListItem = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
updated_at: number;
|
||||||
|
write_access?: boolean;
|
||||||
|
meta?: any;
|
||||||
|
user?: {
|
||||||
|
name?: string;
|
||||||
|
email?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
let loaded = false;
|
let loaded = false;
|
||||||
let showDeleteConfirm = false;
|
let showDeleteConfirm = false;
|
||||||
let tagsContainerElement: HTMLDivElement;
|
let tagsContainerElement: HTMLDivElement;
|
||||||
|
|
||||||
let selectedItem = null;
|
let selectedItem: KnowledgeListItem | null = null;
|
||||||
|
|
||||||
let page = 1;
|
let page = 1;
|
||||||
let query = '';
|
let query = '';
|
||||||
let searchDebounceTimer: ReturnType<typeof setTimeout>;
|
let searchDebounceTimer: ReturnType<typeof setTimeout>;
|
||||||
let viewOption = '';
|
let viewOption = '';
|
||||||
|
let sourceOption = '';
|
||||||
|
|
||||||
let items = null;
|
let items: KnowledgeListItem[] | null = null;
|
||||||
let total = null;
|
let total: number | null = null;
|
||||||
|
|
||||||
let allItemsLoaded = false;
|
let allItemsLoaded = false;
|
||||||
let itemsLoading = false;
|
let itemsLoading = false;
|
||||||
@@ -56,7 +73,7 @@
|
|||||||
clearTimeout(searchDebounceTimer);
|
clearTimeout(searchDebounceTimer);
|
||||||
});
|
});
|
||||||
|
|
||||||
$: if (loaded && viewOption !== undefined) {
|
$: if (loaded && viewOption !== undefined && sourceOption !== undefined) {
|
||||||
init();
|
init();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,16 +100,20 @@
|
|||||||
|
|
||||||
const getItemsPage = async () => {
|
const getItemsPage = async () => {
|
||||||
itemsLoading = true;
|
itemsLoading = true;
|
||||||
const res = await searchKnowledgeBases(localStorage.token, query, viewOption, page).catch(
|
const res = await searchKnowledgeBases(
|
||||||
() => {
|
localStorage.token,
|
||||||
return [];
|
query,
|
||||||
}
|
viewOption,
|
||||||
);
|
page,
|
||||||
|
sourceOption
|
||||||
|
).catch(() => {
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
|
||||||
if (res) {
|
if (res) {
|
||||||
console.log(res);
|
console.log(res);
|
||||||
total = res.total;
|
total = res.total;
|
||||||
const pageItems = res.items;
|
const pageItems: KnowledgeListItem[] = res.items ?? [];
|
||||||
|
|
||||||
if ((pageItems ?? []).length === 0) {
|
if ((pageItems ?? []).length === 0) {
|
||||||
allItemsLoaded = true;
|
allItemsLoaded = true;
|
||||||
@@ -113,7 +134,9 @@
|
|||||||
return res;
|
return res;
|
||||||
};
|
};
|
||||||
|
|
||||||
const deleteHandler = async (item) => {
|
const deleteHandler = async (item: KnowledgeListItem | null) => {
|
||||||
|
if (!item) return;
|
||||||
|
|
||||||
const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => {
|
const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => {
|
||||||
toast.error(`${e}`);
|
toast.error(`${e}`);
|
||||||
});
|
});
|
||||||
@@ -124,7 +147,7 @@
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const exportHandler = async (item) => {
|
const exportHandler = async (item: KnowledgeListItem) => {
|
||||||
try {
|
try {
|
||||||
const blob = await exportKnowledgeById(localStorage.token, item.id);
|
const blob = await exportKnowledgeById(localStorage.token, item.id);
|
||||||
if (blob) {
|
if (blob) {
|
||||||
@@ -145,6 +168,7 @@
|
|||||||
|
|
||||||
onMount(async () => {
|
onMount(async () => {
|
||||||
viewOption = localStorage?.workspaceViewOption || '';
|
viewOption = localStorage?.workspaceViewOption || '';
|
||||||
|
sourceOption = localStorage?.workspaceKnowledgeSourceOption || '';
|
||||||
loaded = true;
|
loaded = true;
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
@@ -241,6 +265,19 @@
|
|||||||
await tick();
|
await tick();
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
<select
|
||||||
|
class="relative w-full flex items-center gap-0.5 px-2.5 py-1.5 bg-gray-50 dark:bg-gray-850 rounded-xl outline-hidden"
|
||||||
|
bind:value={sourceOption}
|
||||||
|
on:change={async () => {
|
||||||
|
localStorage.workspaceKnowledgeSourceOption = sourceOption;
|
||||||
|
await tick();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<option value="">{$i18n.t('All Sources')}</option>
|
||||||
|
<option value="local">{$i18n.t('Local')}</option>
|
||||||
|
<option value="external">{$i18n.t('Connected')}</option>
|
||||||
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -267,11 +304,23 @@
|
|||||||
<div class=" self-center flex-1 justify-between">
|
<div class=" self-center flex-1 justify-between">
|
||||||
<div class="flex items-center justify-between -my-1 h-8">
|
<div class="flex items-center justify-between -my-1 h-8">
|
||||||
<div class=" flex gap-2 items-center justify-between w-full">
|
<div class=" flex gap-2 items-center justify-between w-full">
|
||||||
<div>
|
{#if item?.meta?.source === 'external'}
|
||||||
<Badge type="success" content={$i18n.t('Collection')} />
|
<div>
|
||||||
</div>
|
<Badge
|
||||||
|
type="muted"
|
||||||
|
content={item?.meta?.external?.provider ?? $i18n.t('Connected')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Badge type="muted" content={$i18n.t('Read Only')} />
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<div>
|
||||||
|
<Badge type="success" content={$i18n.t('Collection')} />
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
{#if !item?.write_access}
|
{#if !item?.write_access && item?.meta?.source !== 'external'}
|
||||||
<div>
|
<div>
|
||||||
<Badge type="muted" content={$i18n.t('Read Only')} />
|
<Badge type="muted" content={$i18n.t('Read Only')} />
|
||||||
</div>
|
</div>
|
||||||
@@ -282,7 +331,7 @@
|
|||||||
<div class="flex items-center gap-2">
|
<div class="flex items-center gap-2">
|
||||||
<div class=" flex self-center">
|
<div class=" flex self-center">
|
||||||
<ItemMenu
|
<ItemMenu
|
||||||
onExport={$user.role === 'admin'
|
onExport={$user?.role === 'admin'
|
||||||
? () => {
|
? () => {
|
||||||
exportHandler(item);
|
exportHandler(item);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,10 @@
|
|||||||
import { PaneGroup, Pane, PaneResizer } from 'paneforge';
|
import { PaneGroup, Pane, PaneResizer } from 'paneforge';
|
||||||
|
|
||||||
import { onMount, getContext, onDestroy, tick } from 'svelte';
|
import { onMount, getContext, onDestroy, tick } from 'svelte';
|
||||||
const i18n = getContext('i18n');
|
import type { Writable } from 'svelte/store';
|
||||||
|
import type { i18n as i18nType } from 'i18next';
|
||||||
|
|
||||||
|
const i18n = getContext<Writable<i18nType>>('i18n');
|
||||||
|
|
||||||
import { goto } from '$app/navigation';
|
import { goto } from '$app/navigation';
|
||||||
import { page } from '$app/stores';
|
import { page } from '$app/stores';
|
||||||
@@ -40,7 +43,8 @@
|
|||||||
deleteKnowledgeDirectory,
|
deleteKnowledgeDirectory,
|
||||||
moveFileInKnowledge,
|
moveFileInKnowledge,
|
||||||
syncKnowledgeDiff,
|
syncKnowledgeDiff,
|
||||||
syncKnowledgeCleanup
|
syncKnowledgeCleanup,
|
||||||
|
testExternalKnowledgeRetrieval
|
||||||
} from '$lib/apis/knowledge';
|
} from '$lib/apis/knowledge';
|
||||||
import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
|
import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
|
||||||
|
|
||||||
@@ -98,11 +102,13 @@
|
|||||||
files: any[];
|
files: any[];
|
||||||
access_grants?: any[];
|
access_grants?: any[];
|
||||||
write_access?: boolean;
|
write_access?: boolean;
|
||||||
|
meta?: any;
|
||||||
};
|
};
|
||||||
|
|
||||||
let id = null;
|
let id = null;
|
||||||
let knowledge: Knowledge | null = null;
|
let knowledge: Knowledge | null = null;
|
||||||
let knowledgeId = null;
|
let knowledgeId = null;
|
||||||
|
let isExternalKnowledge = false;
|
||||||
|
|
||||||
let selectedFileId = null;
|
let selectedFileId = null;
|
||||||
let selectedFile = null;
|
let selectedFile = null;
|
||||||
@@ -132,6 +138,14 @@
|
|||||||
let deleteDirectoryContents = true;
|
let deleteDirectoryContents = true;
|
||||||
|
|
||||||
let pendingPollTimer: ReturnType<typeof setInterval> | null = null;
|
let pendingPollTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
let externalTestQuery = '';
|
||||||
|
let externalTestResult: {
|
||||||
|
documents?: string[];
|
||||||
|
metadatas?: Record<string, any>[];
|
||||||
|
distances?: number[];
|
||||||
|
} | null = null;
|
||||||
|
|
||||||
|
$: isExternalKnowledge = knowledge?.meta?.source === 'external';
|
||||||
|
|
||||||
const reset = () => {
|
const reset = () => {
|
||||||
currentPage = 1;
|
currentPage = 1;
|
||||||
@@ -243,6 +257,24 @@
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const externalTestHandler = async () => {
|
||||||
|
if (!isExternalKnowledge || !externalTestQuery.trim()) return;
|
||||||
|
|
||||||
|
const external = knowledge?.meta?.external ?? {};
|
||||||
|
const res = await testExternalKnowledgeRetrieval(localStorage.token, external.connection_id, {
|
||||||
|
query: externalTestQuery,
|
||||||
|
source: external.source,
|
||||||
|
count: 5
|
||||||
|
}).catch((e) => {
|
||||||
|
toast.error(`${e}`);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res) {
|
||||||
|
externalTestResult = res;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const createFileFromText = (name, content) => {
|
const createFileFromText = (name, content) => {
|
||||||
const blob = new Blob([content], { type: 'text/plain' });
|
const blob = new Blob([content], { type: 'text/plain' });
|
||||||
const file = blobToFile(blob, `${name}.txt`);
|
const file = blobToFile(blob, `${name}.txt`);
|
||||||
@@ -1299,284 +1331,357 @@
|
|||||||
<div
|
<div
|
||||||
class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1"
|
class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1"
|
||||||
>
|
>
|
||||||
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
|
{#if isExternalKnowledge}
|
||||||
<div class="flex flex-1 items-center">
|
<div class="p-5 flex flex-col gap-4">
|
||||||
<div class=" self-center ml-1 mr-3">
|
<div class="flex flex-wrap gap-2 text-xs">
|
||||||
<Search className="size-3.5" />
|
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
|
||||||
</div>
|
{$i18n.t('Connected')}
|
||||||
<input
|
|
||||||
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
|
|
||||||
bind:value={query}
|
|
||||||
on:input={handleSearchInput}
|
|
||||||
aria-label={$i18n.t('Search Collection')}
|
|
||||||
placeholder={$i18n.t('Search Collection')}
|
|
||||||
on:focus={() => {
|
|
||||||
selectedFileId = null;
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<Dropdown align="end">
|
|
||||||
<button
|
|
||||||
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
|
|
||||||
type="button"
|
|
||||||
>
|
|
||||||
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
|
|
||||||
</button>
|
|
||||||
|
|
||||||
<div slot="content">
|
|
||||||
<div
|
|
||||||
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
|
|
||||||
>
|
|
||||||
<button
|
|
||||||
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
|
|
||||||
type="button"
|
|
||||||
on:click={() => {
|
|
||||||
includeContent = !includeContent;
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Checkbox
|
|
||||||
state={includeContent ? 'checked' : 'unchecked'}
|
|
||||||
on:change={(e) => {
|
|
||||||
includeContent = e.detail === 'checked';
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
{$i18n.t('File content')}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</Dropdown>
|
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
|
||||||
|
{$i18n.t('Read Only')}
|
||||||
|
</div>
|
||||||
|
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
|
||||||
|
{knowledge?.meta?.external?.provider ?? $i18n.t('Provider')}
|
||||||
|
</div>
|
||||||
|
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
|
||||||
|
{$i18n.t('Service Account')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{#if knowledge?.write_access}
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-3 text-sm">
|
||||||
<div>
|
<div>
|
||||||
<AddContentMenu
|
<div class="text-xs text-gray-500 mb-1">{$i18n.t('Mapped Source')}</div>
|
||||||
onUpload={(data) => {
|
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
|
||||||
if (data.type === 'directory') {
|
{knowledge?.meta?.external?.source?.name ?? $i18n.t('Not configured')}
|
||||||
uploadDirectoryHandler();
|
</div>
|
||||||
} else if (data.type === 'new_directory') {
|
|
||||||
showNewDirectoryModal = true;
|
|
||||||
} else if (data.type === 'web') {
|
|
||||||
showAddWebpageModal = true;
|
|
||||||
} else if (data.type === 'text') {
|
|
||||||
showAddTextContentModal = true;
|
|
||||||
} else {
|
|
||||||
document.getElementById('files-input').click();
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
onSync={async () => {
|
|
||||||
pendingSyncFiles = await collectDirectoryFiles();
|
|
||||||
if (pendingSyncFiles?.length) {
|
|
||||||
showSyncConfirmModal = true;
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
onReset={() => {
|
|
||||||
showResetConfirm = true;
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
</div>
|
</div>
|
||||||
{/if}
|
<div>
|
||||||
</div>
|
<div class="text-xs text-gray-500 mb-1">{$i18n.t('Auth Mode')}</div>
|
||||||
</div>
|
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
|
||||||
|
{$i18n.t('Admin-managed service account')}
|
||||||
<div class="px-3 flex justify-between">
|
|
||||||
<div
|
|
||||||
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
|
|
||||||
on:wheel={(e) => {
|
|
||||||
if (e.deltaY !== 0) {
|
|
||||||
e.preventDefault();
|
|
||||||
e.currentTarget.scrollLeft += e.deltaY;
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<div
|
|
||||||
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
|
|
||||||
>
|
|
||||||
<DropdownOptions
|
|
||||||
align="start"
|
|
||||||
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
|
|
||||||
bind:value={viewOption}
|
|
||||||
items={[
|
|
||||||
{ value: null, label: $i18n.t('All') },
|
|
||||||
{ value: 'created', label: $i18n.t('Created by you') },
|
|
||||||
{ value: 'shared', label: $i18n.t('Shared with you') }
|
|
||||||
]}
|
|
||||||
onChange={(value) => {
|
|
||||||
if (value) {
|
|
||||||
localStorage.workspaceViewOption = value;
|
|
||||||
} else {
|
|
||||||
delete localStorage.workspaceViewOption;
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<DropdownOptions
|
|
||||||
align="start"
|
|
||||||
bind:value={sortKey}
|
|
||||||
placeholder={$i18n.t('Sort')}
|
|
||||||
items={[
|
|
||||||
{ value: 'name', label: $i18n.t('Name') },
|
|
||||||
{ value: 'created_at', label: $i18n.t('Created At') },
|
|
||||||
{ value: 'updated_at', label: $i18n.t('Updated At') }
|
|
||||||
]}
|
|
||||||
/>
|
|
||||||
|
|
||||||
{#if sortKey}
|
|
||||||
<DropdownOptions
|
|
||||||
align="start"
|
|
||||||
bind:value={direction}
|
|
||||||
items={[
|
|
||||||
{ value: 'asc', label: $i18n.t('Asc') },
|
|
||||||
{ value: null, label: $i18n.t('Desc') }
|
|
||||||
]}
|
|
||||||
/>
|
|
||||||
{/if}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{#if currentDirectoryId !== null}
|
|
||||||
<div class="px-5 mt-2">
|
|
||||||
<KnowledgeBreadcrumbs
|
|
||||||
rootLabel={knowledge.name}
|
|
||||||
{breadcrumbs}
|
|
||||||
onNavigate={(dirId) => navigateToDirectory(dirId)}
|
|
||||||
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
|
|
||||||
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
|
|
||||||
{#if syncing}
|
|
||||||
<div class="mx-2.5 mt-2.5 -mb-0.5">
|
|
||||||
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
|
|
||||||
<Spinner className="size-3.5 shrink-0" />
|
|
||||||
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
|
|
||||||
{syncing}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
|
|
||||||
{#if fileItems !== null && fileItemsTotal !== null}
|
|
||||||
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
|
|
||||||
<div class="flex-1 flex">
|
|
||||||
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
|
|
||||||
<div class="w-full h-full flex flex-col min-h-full">
|
|
||||||
{#if fileItems.length > 0 || directoryItems.length > 0}
|
|
||||||
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
|
|
||||||
<Files
|
|
||||||
files={fileItems}
|
|
||||||
directories={directoryItems}
|
|
||||||
{knowledge}
|
|
||||||
{selectedFileId}
|
|
||||||
onClick={(fileId) => {
|
|
||||||
selectedFileId = fileId;
|
|
||||||
|
|
||||||
if (fileItems) {
|
|
||||||
const file = fileItems.find((file) => file.id === selectedFileId);
|
|
||||||
if (file) {
|
|
||||||
fileSelectHandler(file);
|
|
||||||
} else {
|
|
||||||
selectedFile = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
onDelete={(fileId) => {
|
|
||||||
selectedFileId = null;
|
|
||||||
selectedFile = null;
|
|
||||||
|
|
||||||
deleteFileHandler(fileId);
|
|
||||||
}}
|
|
||||||
onRename={(fileId, name) => renameFileHandler(fileId, name)}
|
|
||||||
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
|
|
||||||
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
|
|
||||||
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
|
|
||||||
onMoveFileToDirectory={(fileId, dirId) =>
|
|
||||||
moveFileToDirectoryHandler(fileId, dirId)}
|
|
||||||
onMoveDirectoryToDirectory={(dirId, targetId) =>
|
|
||||||
moveDirectoryHandler(dirId, targetId)}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{#if fileItemsTotal > 30}
|
|
||||||
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
|
|
||||||
{/if}
|
|
||||||
{:else}
|
|
||||||
<div class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs">
|
|
||||||
<div>
|
|
||||||
{$i18n.t('No content found')}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{#if selectedFileId !== null}
|
<div class="text-xs text-gray-500">
|
||||||
<Drawer
|
{$i18n.t(
|
||||||
className="h-full"
|
'This knowledge base retrieves from a connected source. Open WebUI can query it, but cannot upload, sync, edit, delete, reset, or reindex its source data.'
|
||||||
show={selectedFileId !== null}
|
)}
|
||||||
onClose={() => {
|
</div>
|
||||||
selectedFileId = null;
|
|
||||||
selectedFile = null;
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<div class="flex flex-col justify-start h-full max-h-full">
|
|
||||||
<div class=" flex flex-col w-full h-full max-h-full">
|
|
||||||
<div class="shrink-0 flex items-center p-2">
|
|
||||||
<div class="mr-2">
|
|
||||||
<button
|
|
||||||
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
|
|
||||||
aria-label={$i18n.t('Close')}
|
|
||||||
on:click={() => {
|
|
||||||
selectedFileId = null;
|
|
||||||
selectedFile = null;
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<ChevronLeft strokeWidth="2.5" />
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
<div class=" flex-1 text-lg line-clamp-1">
|
|
||||||
{selectedFile?.meta?.name}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{#if knowledge?.write_access}
|
<div class="flex flex-col gap-2">
|
||||||
<div>
|
<div class="font-medium text-sm">{$i18n.t('Test Query')}</div>
|
||||||
<button
|
<div class="flex gap-2">
|
||||||
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
|
<input
|
||||||
disabled={isSaving}
|
class="w-full text-sm rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2 outline-hidden"
|
||||||
on:click={() => {
|
bind:value={externalTestQuery}
|
||||||
updateFileContentHandler();
|
placeholder={$i18n.t('Ask this knowledge source a test question')}
|
||||||
}}
|
/>
|
||||||
>
|
<button
|
||||||
{$i18n.t('Save')}
|
class="px-3 py-2 rounded-xl bg-black text-white dark:bg-white dark:text-black text-sm"
|
||||||
{#if isSaving}
|
on:click={externalTestHandler}
|
||||||
<div class="ml-2 self-center">
|
>
|
||||||
<Spinner />
|
{$i18n.t('Test')}
|
||||||
</div>
|
</button>
|
||||||
{/if}
|
</div>
|
||||||
</button>
|
</div>
|
||||||
</div>
|
|
||||||
{/if}
|
{#if externalTestResult}
|
||||||
|
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 p-3 text-xs">
|
||||||
|
<div class="font-medium mb-2">{$i18n.t('Preview')}</div>
|
||||||
|
{#each externalTestResult.documents ?? [] as document, idx}
|
||||||
|
<div class="border-t border-gray-100 dark:border-gray-800 py-2">
|
||||||
|
<div class="line-clamp-4">{document}</div>
|
||||||
|
<div class="text-gray-500 mt-1">
|
||||||
|
{externalTestResult.metadatas?.[idx]?.source ?? ''}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{#key selectedFile.id}
|
|
||||||
<textarea
|
|
||||||
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
|
|
||||||
bind:value={selectedFileContent}
|
|
||||||
disabled={!knowledge?.write_access}
|
|
||||||
aria-label={$i18n.t('File content')}
|
|
||||||
placeholder={$i18n.t('Add content here')}
|
|
||||||
/>
|
|
||||||
{/key}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
{/each}
|
||||||
</Drawer>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
{:else}
|
{:else}
|
||||||
<div class="my-10">
|
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
|
||||||
<Spinner className="size-4" />
|
<div class="flex flex-1 items-center">
|
||||||
|
<div class=" self-center ml-1 mr-3">
|
||||||
|
<Search className="size-3.5" />
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
|
||||||
|
bind:value={query}
|
||||||
|
on:input={handleSearchInput}
|
||||||
|
aria-label={$i18n.t('Search Collection')}
|
||||||
|
placeholder={$i18n.t('Search Collection')}
|
||||||
|
on:focus={() => {
|
||||||
|
selectedFileId = null;
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Dropdown align="end">
|
||||||
|
<button
|
||||||
|
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<div slot="content">
|
||||||
|
<div
|
||||||
|
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
|
||||||
|
type="button"
|
||||||
|
on:click={() => {
|
||||||
|
includeContent = !includeContent;
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Checkbox
|
||||||
|
state={includeContent ? 'checked' : 'unchecked'}
|
||||||
|
on:change={(e) => {
|
||||||
|
includeContent = e.detail === 'checked';
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
{$i18n.t('File content')}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Dropdown>
|
||||||
|
|
||||||
|
{#if knowledge?.write_access}
|
||||||
|
<div>
|
||||||
|
<AddContentMenu
|
||||||
|
onUpload={(data) => {
|
||||||
|
if (data.type === 'directory') {
|
||||||
|
uploadDirectoryHandler();
|
||||||
|
} else if (data.type === 'new_directory') {
|
||||||
|
showNewDirectoryModal = true;
|
||||||
|
} else if (data.type === 'web') {
|
||||||
|
showAddWebpageModal = true;
|
||||||
|
} else if (data.type === 'text') {
|
||||||
|
showAddTextContentModal = true;
|
||||||
|
} else {
|
||||||
|
document.getElementById('files-input').click();
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onSync={async () => {
|
||||||
|
pendingSyncFiles = await collectDirectoryFiles();
|
||||||
|
if (pendingSyncFiles?.length) {
|
||||||
|
showSyncConfirmModal = true;
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onReset={() => {
|
||||||
|
showResetConfirm = true;
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="px-3 flex justify-between">
|
||||||
|
<div
|
||||||
|
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
|
||||||
|
on:wheel={(e) => {
|
||||||
|
if (e.deltaY !== 0) {
|
||||||
|
e.preventDefault();
|
||||||
|
e.currentTarget.scrollLeft += e.deltaY;
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
|
||||||
|
>
|
||||||
|
<DropdownOptions
|
||||||
|
align="start"
|
||||||
|
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
|
||||||
|
bind:value={viewOption}
|
||||||
|
items={[
|
||||||
|
{ value: null, label: $i18n.t('All') },
|
||||||
|
{ value: 'created', label: $i18n.t('Created by you') },
|
||||||
|
{ value: 'shared', label: $i18n.t('Shared with you') }
|
||||||
|
]}
|
||||||
|
onChange={(value) => {
|
||||||
|
if (value) {
|
||||||
|
localStorage.workspaceViewOption = value;
|
||||||
|
} else {
|
||||||
|
delete localStorage.workspaceViewOption;
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<DropdownOptions
|
||||||
|
align="start"
|
||||||
|
bind:value={sortKey}
|
||||||
|
placeholder={$i18n.t('Sort')}
|
||||||
|
items={[
|
||||||
|
{ value: 'name', label: $i18n.t('Name') },
|
||||||
|
{ value: 'created_at', label: $i18n.t('Created At') },
|
||||||
|
{ value: 'updated_at', label: $i18n.t('Updated At') }
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{#if sortKey}
|
||||||
|
<DropdownOptions
|
||||||
|
align="start"
|
||||||
|
bind:value={direction}
|
||||||
|
items={[
|
||||||
|
{ value: 'asc', label: $i18n.t('Asc') },
|
||||||
|
{ value: null, label: $i18n.t('Desc') }
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if currentDirectoryId !== null}
|
||||||
|
<div class="px-5 mt-2">
|
||||||
|
<KnowledgeBreadcrumbs
|
||||||
|
rootLabel={knowledge.name}
|
||||||
|
{breadcrumbs}
|
||||||
|
onNavigate={(dirId) => navigateToDirectory(dirId)}
|
||||||
|
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
|
||||||
|
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if syncing}
|
||||||
|
<div class="mx-2.5 mt-2.5 -mb-0.5">
|
||||||
|
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
|
||||||
|
<Spinner className="size-3.5 shrink-0" />
|
||||||
|
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
|
||||||
|
{syncing}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if fileItems !== null && fileItemsTotal !== null}
|
||||||
|
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
|
||||||
|
<div class="flex-1 flex">
|
||||||
|
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
|
||||||
|
<div class="w-full h-full flex flex-col min-h-full">
|
||||||
|
{#if fileItems.length > 0 || directoryItems.length > 0}
|
||||||
|
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
|
||||||
|
<Files
|
||||||
|
files={fileItems}
|
||||||
|
directories={directoryItems}
|
||||||
|
{knowledge}
|
||||||
|
{selectedFileId}
|
||||||
|
onClick={(fileId) => {
|
||||||
|
selectedFileId = fileId;
|
||||||
|
|
||||||
|
if (fileItems) {
|
||||||
|
const file = fileItems.find((file) => file.id === selectedFileId);
|
||||||
|
if (file) {
|
||||||
|
fileSelectHandler(file);
|
||||||
|
} else {
|
||||||
|
selectedFile = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onDelete={(fileId) => {
|
||||||
|
selectedFileId = null;
|
||||||
|
selectedFile = null;
|
||||||
|
|
||||||
|
deleteFileHandler(fileId);
|
||||||
|
}}
|
||||||
|
onRename={(fileId, name) => renameFileHandler(fileId, name)}
|
||||||
|
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
|
||||||
|
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
|
||||||
|
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
|
||||||
|
onMoveFileToDirectory={(fileId, dirId) =>
|
||||||
|
moveFileToDirectoryHandler(fileId, dirId)}
|
||||||
|
onMoveDirectoryToDirectory={(dirId, targetId) =>
|
||||||
|
moveDirectoryHandler(dirId, targetId)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if fileItemsTotal > 30}
|
||||||
|
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
|
||||||
|
{/if}
|
||||||
|
{:else}
|
||||||
|
<div
|
||||||
|
class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
{$i18n.t('No content found')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if selectedFileId !== null}
|
||||||
|
<Drawer
|
||||||
|
className="h-full"
|
||||||
|
show={selectedFileId !== null}
|
||||||
|
onClose={() => {
|
||||||
|
selectedFileId = null;
|
||||||
|
selectedFile = null;
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div class="flex flex-col justify-start h-full max-h-full">
|
||||||
|
<div class=" flex flex-col w-full h-full max-h-full">
|
||||||
|
<div class="shrink-0 flex items-center p-2">
|
||||||
|
<div class="mr-2">
|
||||||
|
<button
|
||||||
|
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
|
||||||
|
aria-label={$i18n.t('Close')}
|
||||||
|
on:click={() => {
|
||||||
|
selectedFileId = null;
|
||||||
|
selectedFile = null;
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<ChevronLeft strokeWidth="2.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class=" flex-1 text-lg line-clamp-1">
|
||||||
|
{selectedFile?.meta?.name}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if knowledge?.write_access}
|
||||||
|
<div>
|
||||||
|
<button
|
||||||
|
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
disabled={isSaving}
|
||||||
|
on:click={() => {
|
||||||
|
updateFileContentHandler();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{$i18n.t('Save')}
|
||||||
|
{#if isSaving}
|
||||||
|
<div class="ml-2 self-center">
|
||||||
|
<Spinner />
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#key selectedFile.id}
|
||||||
|
<textarea
|
||||||
|
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
|
||||||
|
bind:value={selectedFileContent}
|
||||||
|
disabled={!knowledge?.write_access}
|
||||||
|
aria-label={$i18n.t('File content')}
|
||||||
|
placeholder={$i18n.t('Add content here')}
|
||||||
|
/>
|
||||||
|
{/key}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Drawer>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<div class="my-10">
|
||||||
|
<Spinner className="size-4" />
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
{:else}
|
{:else}
|
||||||
|
|||||||
Reference in new issue
Block a user