From 591e0aafa1d5e21dcd9fa1279dada2076968b9cb Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Thu, 28 May 2026 17:24:33 -0500 Subject: [PATCH] refac --- backend/open_webui/models/knowledge.py | 97 ++++++++----------- backend/open_webui/routers/knowledge.py | 60 ++++++------ src/lib/apis/knowledge/index.ts | 25 ++--- .../workspace/Knowledge/KnowledgeBase.svelte | 79 +++++++++++---- 4 files changed, 141 insertions(+), 120 deletions(-) diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index fdedf51ea7..84cf4b7ae8 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -360,18 +360,21 @@ class KnowledgeTable: ) # Apply filename / content search - # Use ->> (as_string) instead of CAST(-> AS TEXT) to avoid - # PostgreSQL "invalid memory alloc request size" on large - # extracted-content rows (#24670). - content_text = File.data['content'].as_string() search_filter = None if filter: q = filter.get('query') if q: - search_filter = or_( - File.filename.ilike(f'%{q}%'), - content_text.ilike(f'%{q}%'), - ) + if filter.get('include_content'): + # Use ->> (as_string) instead of CAST(-> AS TEXT) + # to avoid PostgreSQL "invalid memory alloc request + # size" on large extracted-content rows (#24670). + content_text = File.data['content'].as_string() + search_filter = or_( + File.filename.ilike(f'%{q}%'), + content_text.ilike(f'%{q}%'), + ) + else: + search_filter = File.filename.ilike(f'%{q}%') stmt = stmt.filter(search_filter) # Order by file changes @@ -546,16 +549,19 @@ class KnowledgeTable: if filter: query_key = filter.get('query') if query_key: - # Use ->> (as_string) instead of CAST(-> AS TEXT) to - # avoid PostgreSQL memory allocation failures on large - # content (#24670). - content_text = File.data['content'].as_string() - stmt = stmt.filter( - or_( - File.filename.ilike(f'%{query_key}%'), - content_text.ilike(f'%{query_key}%'), + if filter.get('include_content'): + # Use ->> (as_string) instead of CAST(-> AS TEXT) + # to avoid PostgreSQL memory allocation failures on + # large content (#24670). + content_text = File.data['content'].as_string() + stmt = stmt.filter( + or_( + File.filename.ilike(f'%{query_key}%'), + content_text.ilike(f'%{query_key}%'), + ) ) - ) + else: + stmt = stmt.filter(File.filename.ilike(f'%{query_key}%')) view_option = filter.get('view_option') if view_option == 'created': @@ -692,7 +698,9 @@ class KnowledgeTable: except Exception: return False - async def reset_knowledge_by_id(self, id: str, include_directories: bool = True, db: Optional[AsyncSession] = None) -> Optional[KnowledgeModel]: + async def reset_knowledge_by_id( + self, id: str, include_directories: bool = True, db: Optional[AsyncSession] = None + ) -> Optional[KnowledgeModel]: try: async with get_async_db_context(db) as db: # Delete all knowledge_file entries for this knowledge_id @@ -819,9 +827,7 @@ class KnowledgeTable: ) -> list[KnowledgeDirectoryModel]: """List directories at a given level (parent_id=None for root).""" async with get_async_db_context(db) as db: - stmt = select(KnowledgeDirectory).filter( - KnowledgeDirectory.knowledge_id == knowledge_id - ) + stmt = select(KnowledgeDirectory).filter(KnowledgeDirectory.knowledge_id == knowledge_id) if parent_id: stmt = stmt.filter(KnowledgeDirectory.parent_id == parent_id) else: @@ -859,10 +865,7 @@ class KnowledgeTable: .join(KnowledgeFile, File.id == KnowledgeFile.file_id) .filter(KnowledgeFile.knowledge_id == knowledge_id) ) - return [ - (FileModel.model_validate(file), dir_id) - for file, dir_id in result.all() - ] + return [(FileModel.model_validate(file), dir_id) for file, dir_id in result.all()] except Exception: return [] @@ -870,9 +873,7 @@ class KnowledgeTable: self, directory_id: str, db: Optional[AsyncSession] = None ) -> Optional[KnowledgeDirectoryModel]: async with get_async_db_context(db) as db: - result = await db.execute( - select(KnowledgeDirectory).filter_by(id=directory_id) - ) + result = await db.execute(select(KnowledgeDirectory).filter_by(id=directory_id)) directory = result.scalars().first() return KnowledgeDirectoryModel.model_validate(directory) if directory else None @@ -892,9 +893,7 @@ class KnowledgeTable: while current_id and current_id not in seen: seen.add(current_id) - result = await db.execute( - select(KnowledgeDirectory).filter_by(id=current_id) - ) + result = await db.execute(select(KnowledgeDirectory).filter_by(id=current_id)) directory = result.scalars().first() if not directory: break @@ -913,9 +912,7 @@ class KnowledgeTable: async with get_async_db_context(db) as db: try: await db.execute( - update(KnowledgeDirectory) - .filter_by(id=directory_id) - .values(name=name, updated_at=int(time.time())) + update(KnowledgeDirectory).filter_by(id=directory_id).values(name=name, updated_at=int(time.time())) ) await db.commit() return await self.get_directory_by_id(directory_id, db=db) @@ -941,9 +938,7 @@ class KnowledgeTable: if current == directory_id: return None # Would create a cycle seen.add(current) - result = await db.execute( - select(KnowledgeDirectory.parent_id).filter_by(id=current) - ) + result = await db.execute(select(KnowledgeDirectory.parent_id).filter_by(id=current)) row = result.first() current = row[0] if row else None @@ -991,9 +986,7 @@ class KnowledgeTable: async with get_async_db_context(db) as db: try: # Get the directory to find its parent - result = await db.execute( - select(KnowledgeDirectory).filter_by(id=directory_id) - ) + result = await db.execute(select(KnowledgeDirectory).filter_by(id=directory_id)) directory = result.scalars().first() if not directory: return False @@ -1003,9 +996,7 @@ class KnowledgeTable: if move_files_to_parent: # Move files in this directory to its parent (or root) await db.execute( - update(KnowledgeFile) - .filter_by(directory_id=directory_id) - .values(directory_id=parent_id) + update(KnowledgeFile).filter_by(directory_id=directory_id).values(directory_id=parent_id) ) # Recursively move files from all subdirectories too await self._move_files_from_subtree(directory_id, parent_id, db=db) @@ -1014,9 +1005,7 @@ class KnowledgeTable: await self._delete_files_in_subtree(directory_id, db=db) # CASCADE on parent_id will handle deleting subdirectories - await db.execute( - delete(KnowledgeDirectory).filter_by(id=directory_id) - ) + await db.execute(delete(KnowledgeDirectory).filter_by(id=directory_id)) await db.commit() return True except Exception as e: @@ -1030,16 +1019,12 @@ class KnowledgeTable: db: AsyncSession, ) -> None: """Recursively move all files from a directory subtree to the target.""" - result = await db.execute( - select(KnowledgeDirectory.id).filter_by(parent_id=directory_id) - ) + result = await db.execute(select(KnowledgeDirectory.id).filter_by(parent_id=directory_id)) child_ids = [row[0] for row in result.all()] for child_id in child_ids: await db.execute( - update(KnowledgeFile) - .filter_by(directory_id=child_id) - .values(directory_id=target_directory_id) + update(KnowledgeFile).filter_by(directory_id=child_id).values(directory_id=target_directory_id) ) await self._move_files_from_subtree(child_id, target_directory_id, db=db) @@ -1049,12 +1034,8 @@ class KnowledgeTable: db: AsyncSession, ) -> None: """Recursively delete all files from a directory subtree.""" - await db.execute( - delete(KnowledgeFile).filter_by(directory_id=directory_id) - ) - result = await db.execute( - select(KnowledgeDirectory.id).filter_by(parent_id=directory_id) - ) + await db.execute(delete(KnowledgeFile).filter_by(directory_id=directory_id)) + result = await db.execute(select(KnowledgeDirectory.id).filter_by(parent_id=directory_id)) child_ids = [row[0] for row in result.all()] for child_id in child_ids: await self._delete_files_in_subtree(child_id, db=db) diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 95e9b96924..a9f867eb6e 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -217,6 +217,7 @@ async def search_knowledge_bases( @router.get('/search/files', response_model=KnowledgeFileListResponse) async def search_knowledge_files( query: str | None = None, + include_content: bool = Query(False, description='Include file content in search (expensive).'), page: int | None = 1, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session), @@ -228,6 +229,8 @@ async def search_knowledge_files( filter = {} if query: filter['query'] = query + if include_content: + filter['include_content'] = True groups = await Groups.get_groups_by_member_id(user.id, db=db) if groups: @@ -558,6 +561,7 @@ async def update_knowledge_access_by_id( async def get_knowledge_files_by_id( id: str, query: str | None = None, + include_content: bool = Query(False, description='Include file content in search (expensive).'), view_option: str | None = None, order_by: str | None = None, direction: str | None = None, @@ -602,6 +606,8 @@ async def get_knowledge_files_by_id( filter = {} if query: filter['query'] = query + if include_content: + filter['include_content'] = True if view_option: filter['view_option'] = view_option if order_by: @@ -996,9 +1002,9 @@ async def reset_knowledge_by_id( class FileManifestEntry(BaseModel): - filename: str # basename: "readme.md" - path: str # relative dir: "docs/api" or "" for root - checksum: str # SHA-256 of raw bytes + filename: str # basename: "readme.md" + path: str # relative dir: "docs/api" or "" for root + checksum: str # SHA-256 of raw bytes size: int @@ -1007,13 +1013,13 @@ class SyncDiffForm(BaseModel): class SyncDiffResponse(BaseModel): - added: list[dict] # [{filename, path}] — new files - modified: list[dict] # [{filename, path, stale_file_id}] — changed files - deleted: list[dict] # [{file_id, filename}] — files to remove - mkdir: list[str] # directory paths to create - rmdir: list[str] # directory IDs to remove + added: list[dict] # [{filename, path}] — new files + modified: list[dict] # [{filename, path, stale_file_id}] — changed files + deleted: list[dict] # [{file_id, filename}] — files to remove + mkdir: list[str] # directory paths to create + rmdir: list[str] # directory IDs to remove unmodified_count: int - directory_map: dict[str, str] # existing path → directory ID + directory_map: dict[str, str] # existing path → directory ID @router.post('/{id}/sync/diff', response_model=SyncDiffResponse) @@ -1073,11 +1079,13 @@ async def sync_knowledge_diff( if key not in indexed_files: added.append({'filename': entry.filename, 'path': entry.path}) elif indexed_files[key]['checksum'] != entry.checksum: - modified.append({ - 'filename': entry.filename, - 'path': entry.path, - 'stale_file_id': indexed_files[key]['file_id'], - }) + modified.append( + { + 'filename': entry.filename, + 'path': entry.path, + 'stale_file_id': indexed_files[key]['file_id'], + } + ) else: unmodified_count += 1 @@ -1091,12 +1099,9 @@ async def sync_knowledge_diff( if entry.path: segments = entry.path.split('/') for depth in range(len(segments)): - required_directory_paths.add('/'.join(segments[:depth + 1])) + required_directory_paths.add('/'.join(segments[: depth + 1])) - mkdir = sorted( - [p for p in required_directory_paths if p not in directory_id_by_path], - key=lambda p: p.count('/') - ) + mkdir = sorted([p for p in required_directory_paths if p not in directory_id_by_path], key=lambda p: p.count('/')) orphaned_directory_paths = set(directory_id_by_path) - required_directory_paths rmdir = [directory_id_by_path[p] for p in orphaned_directory_paths] @@ -1118,8 +1123,8 @@ async def sync_knowledge_diff( class SyncCleanupForm(BaseModel): - file_ids: list[str] # file IDs to delete - dir_ids: list[str] = [] # directory IDs to rmdir + file_ids: list[str] # file IDs to delete + dir_ids: list[str] = [] # directory IDs to rmdir @router.post('/{id}/sync/cleanup') @@ -1144,12 +1149,8 @@ async def sync_knowledge_cleanup( await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db) try: - await ASYNC_VECTOR_DB_CLIENT.delete( - collection_name=id, filter={'file_id': file_id} - ) - await ASYNC_VECTOR_DB_CLIENT.delete( - collection_name=id, filter={'hash': file.hash} - ) + await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=id, filter={'file_id': file_id}) + await ASYNC_VECTOR_DB_CLIENT.delete(collection_name=id, filter={'hash': file.hash}) except Exception: pass @@ -1342,9 +1343,7 @@ class KnowledgeFileMoveForm(BaseModel): directory_id: Optional[str] = None -async def _verify_knowledge_write_access( - id: str, user, db: AsyncSession -): +async def _verify_knowledge_write_access(id: str, user, db: AsyncSession): """Verify the user has write access to the knowledge base. Returns the knowledge model.""" knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db) if not knowledge: @@ -1494,4 +1493,3 @@ async def move_file_in_knowledge( detail='Failed to move file.', ) return {'status': True} - diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index f0c1750fd4..7e00160ffe 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -115,11 +115,12 @@ export const searchKnowledgeBases = async ( export const searchKnowledgeFiles = async ( token: string, - query?: string | null = null, - viewOption?: string | null = null, - orderBy?: string | null = null, - direction?: string | null = null, - page: number = 1 + query?: string | null, + viewOption?: string | null, + orderBy?: string | null, + direction?: string | null, + page: number = 1, + includeContent: boolean = false ) => { let error = null; @@ -129,6 +130,7 @@ export const searchKnowledgeFiles = async ( if (orderBy) searchParams.append('order_by', orderBy); if (direction) searchParams.append('direction', direction); searchParams.append('page', page.toString()); + if (includeContent) searchParams.append('include_content', 'true'); const res = await fetch( `${WEBUI_API_BASE_URL}/knowledge/search/files?${searchParams.toString()}`, @@ -197,12 +199,13 @@ export const getKnowledgeById = async (token: string, id: string) => { export const searchKnowledgeFilesById = async ( token: string, id: string, - query?: string | null = null, - viewOption?: string | null = null, - orderBy?: string | null = null, - direction?: string | null = null, + query?: string | null, + viewOption?: string | null, + orderBy?: string | null, + direction?: string | null, page: number = 1, - directoryId?: string | null = undefined + directoryId?: string | null, + includeContent: boolean = false ) => { let error = null; @@ -216,6 +219,7 @@ export const searchKnowledgeFilesById = async ( if (directoryId !== undefined) { searchParams.append('directory_id', directoryId ?? ''); } + if (includeContent) searchParams.append('include_content', 'true'); const res = await fetch( `${WEBUI_API_BASE_URL}/knowledge/${id}/files?${searchParams.toString()}`, @@ -546,7 +550,6 @@ export const syncKnowledgeCleanup = async ( export const deleteKnowledgeById = async (token: string, id: string) => { let error = null; - const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, { method: 'DELETE', headers: { diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index b82666aa56..a184b47f29 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -65,6 +65,9 @@ import Search from '$lib/components/icons/Search.svelte'; import FilesOverlay from '$lib/components/chat/MessageInput/FilesOverlay.svelte'; import DropdownOptions from '$lib/components/common/DropdownOptions.svelte'; + import Dropdown from '$lib/components/common/Dropdown.svelte'; + import Checkbox from '$lib/components/common/Checkbox.svelte'; + import AdjustmentsHorizontal from '$lib/components/icons/AdjustmentsHorizontal.svelte'; import Pagination from '$lib/components/common/Pagination.svelte'; import AttachWebpageModal from '$lib/components/chat/MessageInput/AttachWebpageModal.svelte'; @@ -107,6 +110,7 @@ let inputFiles = null; let query = ''; + let includeContent = false; let searchDebounceTimer: ReturnType; let viewOption = null; @@ -150,7 +154,8 @@ viewOption !== undefined && sortKey !== undefined && direction !== undefined && - currentPage !== undefined + currentPage !== undefined && + includeContent !== undefined ) { getItemsPage(); } @@ -182,7 +187,8 @@ sortKey, direction, currentPage, - currentDirectoryId + currentDirectoryId, + includeContent ).catch(() => { return null; }); @@ -1272,6 +1278,37 @@ }} /> + + + +
+
+ +
+
+ + {#if knowledge?.write_access}
{ - showResetConfirm = true; - }} - /> + }} + onReset={() => { + showResetConfirm = true; + }} + />
{/if} - {#if currentDirectoryId !== null} -
- navigateToDirectory(dirId)} - onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} - onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} - /> -
- {/if} -
+ {#if currentDirectoryId !== null} +
+ navigateToDirectory(dirId)} + onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} + onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} + /> +
+ {/if} + {#if syncing}
@@ -1543,6 +1580,8 @@ }} >
- {$i18n.t('This will remove all files and directories from this knowledge base. This action cannot be undone.')} + {$i18n.t( + 'This will remove all files and directories from this knowledge base. This action cannot be undone.' + )}