bhetherman/open-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-05-28 17:24:33 -05:00
parent e61bcc8500
commit 591e0aafa1
4 changed files with 141 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/models/knowledge.py
+39 -58
View File
@@ -360,18 +360,21 @@ class KnowledgeTable:
)
# Apply filename / content search
# Use ->> (as_string) instead of CAST(-> AS TEXT) to avoid
# PostgreSQL "invalid memory alloc request size" on large
# extracted-content rows (#24670).
content_text = File.data['content'].as_string()
search_filter = None
if filter:
q = filter.get('query')
if q:
search_filter = or_(
File.filename.ilike(f'%{q}%'),
content_text.ilike(f'%{q}%'),
)
if filter.get('include_content'):
# Use ->> (as_string) instead of CAST(-> AS TEXT)
# to avoid PostgreSQL "invalid memory alloc request
# size" on large extracted-content rows (#24670).
content_text = File.data['content'].as_string()
search_filter = or_(
File.filename.ilike(f'%{q}%'),
content_text.ilike(f'%{q}%'),
)
else:
search_filter = File.filename.ilike(f'%{q}%')
stmt = stmt.filter(search_filter)
# Order by file changes
@@ -546,16 +549,19 @@ class KnowledgeTable:
if filter:
query_key = filter.get('query')
if query_key:
# Use ->> (as_string) instead of CAST(-> AS TEXT) to
# avoid PostgreSQL memory allocation failures on large
# content (#24670).
content_text = File.data['content'].as_string()
stmt = stmt.filter(
or_(
File.filename.ilike(f'%{query_key}%'),
content_text.ilike(f'%{query_key}%'),
if filter.get('include_content'):
# Use ->> (as_string) instead of CAST(-> AS TEXT)
# to avoid PostgreSQL memory allocation failures on
# large content (#24670).
content_text = File.data['content'].as_string()
stmt = stmt.filter(
or_(
File.filename.ilike(f'%{query_key}%'),
content_text.ilike(f'%{query_key}%'),
)
)
)
else:
stmt = stmt.filter(File.filename.ilike(f'%{query_key}%'))
view_option = filter.get('view_option')
if view_option == 'created':
@@ -692,7 +698,9 @@ class KnowledgeTable:
except Exception:
return False
async def reset_knowledge_by_id(self, id: str, include_directories: bool = True, db: Optional[AsyncSession] = None) -> Optional[KnowledgeModel]:
async def reset_knowledge_by_id(
self, id: str, include_directories: bool = True, db: Optional[AsyncSession] = None
) -> Optional[KnowledgeModel]:
try:
async with get_async_db_context(db) as db:
# Delete all knowledge_file entries for this knowledge_id
@@ -819,9 +827,7 @@ class KnowledgeTable:
) -> list[KnowledgeDirectoryModel]:
"""List directories at a given level (parent_id=None for root)."""
async with get_async_db_context(db) as db:
stmt = select(KnowledgeDirectory).filter(
KnowledgeDirectory.knowledge_id == knowledge_id
)
stmt = select(KnowledgeDirectory).filter(KnowledgeDirectory.knowledge_id == knowledge_id)
if parent_id:
stmt = stmt.filter(KnowledgeDirectory.parent_id == parent_id)
else:
@@ -859,10 +865,7 @@ class KnowledgeTable:
.join(KnowledgeFile, File.id == KnowledgeFile.file_id)
.filter(KnowledgeFile.knowledge_id == knowledge_id)
)
return [
(FileModel.model_validate(file), dir_id)
for file, dir_id in result.all()
]
return [(FileModel.model_validate(file), dir_id) for file, dir_id in result.all()]
except Exception:
return []
@@ -870,9 +873,7 @@ class KnowledgeTable:
self, directory_id: str, db: Optional[AsyncSession] = None
) -> Optional[KnowledgeDirectoryModel]:
async with get_async_db_context(db) as db:
result = await db.execute(
select(KnowledgeDirectory).filter_by(id=directory_id)
)
result = await db.execute(select(KnowledgeDirectory).filter_by(id=directory_id))
directory = result.scalars().first()
return KnowledgeDirectoryModel.model_validate(directory) if directory else None
@@ -892,9 +893,7 @@ class KnowledgeTable:
while current_id and current_id not in seen:
seen.add(current_id)
result = await db.execute(
select(KnowledgeDirectory).filter_by(id=current_id)
)
result = await db.execute(select(KnowledgeDirectory).filter_by(id=current_id))
directory = result.scalars().first()
if not directory:
break
@@ -913,9 +912,7 @@ class KnowledgeTable:
async with get_async_db_context(db) as db:
try:
await db.execute(
update(KnowledgeDirectory)
.filter_by(id=directory_id)
.values(name=name, updated_at=int(time.time()))
update(KnowledgeDirectory).filter_by(id=directory_id).values(name=name, updated_at=int(time.time()))
)
await db.commit()
return await self.get_directory_by_id(directory_id, db=db)
@@ -941,9 +938,7 @@ class KnowledgeTable:
if current == directory_id:
return None # Would create a cycle
seen.add(current)
result = await db.execute(
select(KnowledgeDirectory.parent_id).filter_by(id=current)
)
result = await db.execute(select(KnowledgeDirectory.parent_id).filter_by(id=current))
row = result.first()
current = row[0] if row else None
@@ -991,9 +986,7 @@ class KnowledgeTable:
async with get_async_db_context(db) as db:
try:
# Get the directory to find its parent
result = await db.execute(
select(KnowledgeDirectory).filter_by(id=directory_id)
)
result = await db.execute(select(KnowledgeDirectory).filter_by(id=directory_id))
directory = result.scalars().first()
if not directory:
return False
@@ -1003,9 +996,7 @@ class KnowledgeTable:
if move_files_to_parent:
# Move files in this directory to its parent (or root)
await db.execute(
update(KnowledgeFile)
.filter_by(directory_id=directory_id)
.values(directory_id=parent_id)
update(KnowledgeFile).filter_by(directory_id=directory_id).values(directory_id=parent_id)
)
# Recursively move files from all subdirectories too
await self._move_files_from_subtree(directory_id, parent_id, db=db)
@@ -1014,9 +1005,7 @@ class KnowledgeTable:
await self._delete_files_in_subtree(directory_id, db=db)
# CASCADE on parent_id will handle deleting subdirectories
await db.execute(
delete(KnowledgeDirectory).filter_by(id=directory_id)
)
await db.execute(delete(KnowledgeDirectory).filter_by(id=directory_id))
await db.commit()
return True
except Exception as e:
@@ -1030,16 +1019,12 @@ class KnowledgeTable:
db: AsyncSession,
) -> None:
"""Recursively move all files from a directory subtree to the target."""
result = await db.execute(
select(KnowledgeDirectory.id).filter_by(parent_id=directory_id)
)
result = await db.execute(select(KnowledgeDirectory.id).filter_by(parent_id=directory_id))
child_ids = [row[0] for row in result.all()]
for child_id in child_ids:
await db.execute(
update(KnowledgeFile)
.filter_by(directory_id=child_id)
.values(directory_id=target_directory_id)
update(KnowledgeFile).filter_by(directory_id=child_id).values(directory_id=target_directory_id)
)
await self._move_files_from_subtree(child_id, target_directory_id, db=db)
@@ -1049,12 +1034,8 @@ class KnowledgeTable:
db: AsyncSession,
) -> None:
"""Recursively delete all files from a directory subtree."""
await db.execute(
delete(KnowledgeFile).filter_by(directory_id=directory_id)
)
result = await db.execute(
select(KnowledgeDirectory.id).filter_by(parent_id=directory_id)
)
await db.execute(delete(KnowledgeFile).filter_by(directory_id=directory_id))
result = await db.execute(select(KnowledgeDirectory.id).filter_by(parent_id=directory_id))
child_ids = [row[0] for row in result.all()]
for child_id in child_ids:
await self._delete_files_in_subtree(child_id, db=db)
backend/open_webui/routers/knowledge.py
+29 -31
View File
@@ -217,6 +217,7 @@ async def search_knowledge_bases(
@router.get('/search/files', response_model=KnowledgeFileListResponse)
async def search_knowledge_files(
query: str | None = None,
include_content: bool = Query(False, description='Include file content in search (expensive).'),
page: int | None = 1,
user=Depends(get_verified_user),
db: AsyncSession = Depends(get_async_session),
@@ -228,6 +229,8 @@ async def search_knowledge_files(
filter = {}
if query:
filter['query'] = query
if include_content:
filter['include_content'] = True
groups = await Groups.get_groups_by_member_id(user.id, db=db)
if groups:
@@ -558,6 +561,7 @@ async def update_knowledge_access_by_id(
async def get_knowledge_files_by_id(
id: str,
query: str | None = None,
include_content: bool = Query(False, description='Include file content in search (expensive).'),
view_option: str | None = None,
order_by: str | None = None,
direction: str | None = None,
@@ -602,6 +606,8 @@ async def get_knowledge_files_by_id(
filter = {}
if query:
filter['query'] = query
if include_content:
filter['include_content'] = True
if view_option:
filter['view_option'] = view_option
if order_by:
@@ -996,9 +1002,9 @@ async def reset_knowledge_by_id(
class FileManifestEntry(BaseModel):
filename: str # basename: "readme.md"
path: str # relative dir: "docs/api" or "" for root
checksum: str # SHA-256 of raw bytes
filename: str # basename: "readme.md"
path: str # relative dir: "docs/api" or "" for root
checksum: str # SHA-256 of raw bytes
size: int
@@ -1007,13 +1013,13 @@ class SyncDiffForm(BaseModel):
class SyncDiffResponse(BaseModel):
added: list[dict] # [{filename, path}] — new files
modified: list[dict] # [{filename, path, stale_file_id}] — changed files
deleted: list[dict] # [{file_id, filename}] — files to remove
mkdir: list[str] # directory paths to create
rmdir: list[str] # directory IDs to remove
added: list[dict] # [{filename, path}] — new files
modified: list[dict] # [{filename, path, stale_file_id}] — changed files
deleted: list[dict] # [{file_id, filename}] — files to remove
mkdir: list[str] # directory paths to create
rmdir: list[str] # directory IDs to remove
unmodified_count: int
directory_map: dict[str, str] # existing path → directory ID
directory_map: dict[str, str] # existing path → directory ID
@router.post('/{id}/sync/diff', response_model=SyncDiffResponse)
@@ -1073,11 +1079,13 @@ async def sync_knowledge_diff(
if key not in indexed_files:
added.append({'filename': entry.filename, 'path': entry.path})
elif indexed_files[key]['checksum'] != entry.checksum:
modified.append({
'filename': entry.filename,
'path': entry.path,
'stale_file_id': indexed_files[key]['file_id'],
})
modified.append(
{
'filename': entry.filename,
'path': entry.path,
'stale_file_id': indexed_files[key]['file_id'],
}
)
else:
unmodified_count += 1
@@ -1091,12 +1099,9 @@ async def sync_knowledge_diff(
if entry.path:
segments = entry.path.split('/')
for depth in range(len(segments)):
required_directory_paths.add('/'.join(segments[:depth + 1]))
required_directory_paths.add('/'.join(segments[: depth + 1]))
mkdir = sorted(
[p for p in required_directory_paths if p not in directory_id_by_path],
key=lambda p: p.count('/')
)
mkdir = sorted([p for p in required_directory_paths if p not in directory_id_by_path], key=lambda p: p.count('/'))
orphaned_directory_paths = set(directory_id_by_path) - required_directory_paths
rmdir = [directory_id_by_path[p] for p in orphaned_directory_paths]
@@ -1118,8 +1123,8 @@ async def sync_knowledge_diff(
class SyncCleanupForm(BaseModel):
file_ids: list[str] # file IDs to delete
dir_ids: list[str] = [] # directory IDs to rmdir
file_ids: list[str] # file IDs to delete
dir_ids: list[str] = [] # directory IDs to rmdir
@router.post('/{id}/sync/cleanup')
@@ -1144,12 +1149,8 @@ async def sync_knowledge_cleanup(
await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db)
try:
await ASYNC_VECTOR_DB_CLIENT.delete(
collection_name=id, filter={'file_id': file_id}
)
await ASYNC_VECTOR_DB_CLIENT.delete(
collection_name=id, filter={'hash': file.hash}
)
await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=id, filter={'file_id': file_id})
await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=id, filter={'hash': file.hash})
except Exception:
pass
@@ -1342,9 +1343,7 @@ class KnowledgeFileMoveForm(BaseModel):
directory_id: Optional[str] = None
async def _verify_knowledge_write_access(
id: str, user, db: AsyncSession
):
async def _verify_knowledge_write_access(id: str, user, db: AsyncSession):
"""Verify the user has write access to the knowledge base. Returns the knowledge model."""
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
if not knowledge:
@@ -1494,4 +1493,3 @@ async def move_file_in_knowledge(
detail='Failed to move file.',
)
return {'status': True}
src/lib/apis/knowledge/index.ts
+14 -11
View File
@@ -115,11 +115,12 @@ export const searchKnowledgeBases = async (
export const searchKnowledgeFiles = async (
token: string,
query?: string | null = null,
viewOption?: string | null = null,
orderBy?: string | null = null,
direction?: string | null = null,
page: number = 1
query?: string | null,
viewOption?: string | null,
orderBy?: string | null,
direction?: string | null,
page: number = 1,
includeContent: boolean = false
) => {
let error = null;
@@ -129,6 +130,7 @@ export const searchKnowledgeFiles = async (
if (orderBy) searchParams.append('order_by', orderBy);
if (direction) searchParams.append('direction', direction);
searchParams.append('page', page.toString());
if (includeContent) searchParams.append('include_content', 'true');
const res = await fetch(
`${WEBUI_API_BASE_URL}/knowledge/search/files?${searchParams.toString()}`,
@@ -197,12 +199,13 @@ export const getKnowledgeById = async (token: string, id: string) => {
export const searchKnowledgeFilesById = async (
token: string,
id: string,
query?: string | null = null,
viewOption?: string | null = null,
orderBy?: string | null = null,
direction?: string | null = null,
query?: string | null,
viewOption?: string | null,
orderBy?: string | null,
direction?: string | null,
page: number = 1,
directoryId?: string | null = undefined
directoryId?: string | null,
includeContent: boolean = false
) => {
let error = null;
@@ -216,6 +219,7 @@ export const searchKnowledgeFilesById = async (
if (directoryId !== undefined) {
searchParams.append('directory_id', directoryId ?? '');
}
if (includeContent) searchParams.append('include_content', 'true');
const res = await fetch(
`${WEBUI_API_BASE_URL}/knowledge/${id}/files?${searchParams.toString()}`,
@@ -546,7 +550,6 @@ export const syncKnowledgeCleanup = async (
export const deleteKnowledgeById = async (token: string, id: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, {
method: 'DELETE',
headers: {
src/lib/components/workspace/Knowledge/KnowledgeBase.svelte
+59 -20
View File
@@ -65,6 +65,9 @@
import Search from '$lib/components/icons/Search.svelte';
import FilesOverlay from '$lib/components/chat/MessageInput/FilesOverlay.svelte';
import DropdownOptions from '$lib/components/common/DropdownOptions.svelte';
import Dropdown from '$lib/components/common/Dropdown.svelte';
import Checkbox from '$lib/components/common/Checkbox.svelte';
import AdjustmentsHorizontal from '$lib/components/icons/AdjustmentsHorizontal.svelte';
import Pagination from '$lib/components/common/Pagination.svelte';
import AttachWebpageModal from '$lib/components/chat/MessageInput/AttachWebpageModal.svelte';
@@ -107,6 +110,7 @@
let inputFiles = null;
let query = '';
let includeContent = false;
let searchDebounceTimer: ReturnType<typeof setTimeout>;
let viewOption = null;
@@ -150,7 +154,8 @@
viewOption !== undefined &&
sortKey !== undefined &&
direction !== undefined &&
currentPage !== undefined
currentPage !== undefined &&
includeContent !== undefined
) {
getItemsPage();
}
@@ -182,7 +187,8 @@
sortKey,
direction,
currentPage,
currentDirectoryId
currentDirectoryId,
includeContent
).catch(() => {
return null;
});
@@ -1272,6 +1278,37 @@
}}
/>
<Dropdown align="end">
<button
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
type="button"
>
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
</button>
<div slot="content">
<div
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
>
<button
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
type="button"
on:click={() => {
includeContent = !includeContent;
}}
>
<Checkbox
state={includeContent ? 'checked' : 'unchecked'}
on:change={(e) => {
includeContent = e.detail === 'checked';
}}
/>
{$i18n.t('File content')}
</button>
</div>
</div>
</Dropdown>
{#if knowledge?.write_access}
<div>
<AddContentMenu
@@ -1293,28 +1330,16 @@
if (pendingSyncFiles?.length) {
showSyncConfirmModal = true;
}
}}
onReset={() => {
showResetConfirm = true;
}}
/>
}}
onReset={() => {
showResetConfirm = true;
}}
/>
</div>
{/if}
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 -mt-1 pb-2">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
@@ -1371,6 +1396,18 @@
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 mt-1 pb-1">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
{#if syncing}
<div class="mx-2.5 mt-2.5 -mb-0.5">
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
@@ -1543,6 +1580,8 @@
}}
>
<div class="text-sm text-gray-700 dark:text-gray-300 flex-1 line-clamp-3">
{$i18n.t('This will remove all files and directories from this knowledge base. This action cannot be undone.')}
{$i18n.t(
'This will remove all files and directories from this knowledge base. This action cannot be undone.'
)}
</div>
</ConfirmDialog>