refac: async db

This commit is contained in:
Timothy Jaeryang Baek committed 2026-04-12 14:22:11 -05:00
1 parent b618d84065
commit 27169124f2
74 files changed
+4831 -4479

No files matched your search

+53 -53
View File
@@ -21,8 +21,8 @@ from fastapi import (
)
from fastapi.responses import FileResponse, StreamingResponse
from sqlalchemy.orm import Session
from open_webui.internal.db import get_session, SessionLocal
from sqlalchemy.ext.asyncio import AsyncSession
from open_webui.internal.db import get_async_session, SessionLocal
from open_webui.constants import ERROR_MESSAGES
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
@@ -88,16 +88,16 @@ def _is_text_file(file_path: str, chunk_size: int = 8192) -> bool:
return False
def process_uploaded_file(
async def process_uploaded_file(
request,
file,
file_path,
file_item,
file_metadata,
user,
db: Optional[Session] = None,
db: Optional[AsyncSession] = None,
):
def _process_handler(db_session):
async def _process_handler(db_session):
try:
content_type = file.content_type
@@ -141,7 +141,7 @@ def process_uploaded_file(
except Exception as e:
log.error(f'Error processing file: {file_item.id}')
Files.update_file_data_by_id(
await Files.update_file_data_by_id(
file_item.id,
{
'status': 'failed',
@@ -158,7 +158,7 @@ def process_uploaded_file(
@router.post('/', response_model=FileModelResponse)
def upload_file(
async def upload_file(
request: Request,
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
@@ -166,9 +166,9 @@ def upload_file(
process: bool = Query(True),
process_in_background: bool = Query(True),
user=Depends(get_verified_user),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
return upload_file_handler(
return await upload_file_handler(
request,
file=file,
metadata=metadata,
@@ -180,7 +180,7 @@ def upload_file(
)
def upload_file_handler(
async def upload_file_handler(
request: Request,
file: UploadFile = File(...),
metadata: Optional[dict | str] = Form(None),
@@ -188,7 +188,7 @@ def upload_file_handler(
process_in_background: bool = Query(True),
user=Depends(get_verified_user),
background_tasks: Optional[BackgroundTasks] = None,
db: Optional[Session] = None,
db: Optional[AsyncSession] = None,
):
log.info(f'file.content_type: {file.content_type} {process}')
@@ -236,7 +236,7 @@ def upload_file_handler(
},
)
file_item = Files.insert_new_file(
file_item = await Files.insert_new_file(
user.id,
FileForm(
**{
@@ -258,9 +258,9 @@ def upload_file_handler(
)
if 'channel_id' in file_metadata:
channel = Channels.get_channel_by_id_and_user_id(file_metadata['channel_id'], user.id, db=db)
channel = await Channels.get_channel_by_id_and_user_id(file_metadata['channel_id'], user.id, db=db)
if channel:
Channels.add_file_to_channel_by_id(channel.id, file_item.id, user.id, db=db)
await Channels.add_file_to_channel_by_id(channel.id, file_item.id, user.id, db=db)
if process:
if background_tasks and process_in_background:
@@ -275,7 +275,7 @@ def upload_file_handler(
)
return {'status': True, **file_item.model_dump()}
else:
process_uploaded_file(
await process_uploaded_file(
request,
file,
file_path,
@@ -317,12 +317,12 @@ async def list_files(
user=Depends(get_verified_user),
page: int = Query(1, ge=1, description='Page number (1-indexed)'),
content: bool = Query(True),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
skip = (page - 1) * PAGE_SIZE
user_id = None if (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL) else user.id
result = Files.get_file_list(user_id=user_id, skip=skip, limit=PAGE_SIZE, db=db)
result = await Files.get_file_list(user_id=user_id, skip=skip, limit=PAGE_SIZE, db=db)
if not content:
for file in result.items:
@@ -347,7 +347,7 @@ async def search_files(
skip: int = Query(0, ge=0, description='Number of files to skip'),
limit: int = Query(100, ge=1, le=1000, description='Maximum number of files to return'),
user=Depends(get_verified_user),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
"""
Search for files by filename with support for wildcard patterns.
@@ -357,7 +357,7 @@ async def search_files(
user_id = None if (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL) else user.id
# Use optimized database query with pagination
files = Files.search_files(
files = await Files.search_files(
user_id=user_id,
filename=filename,
skip=skip,
@@ -385,8 +385,8 @@ async def search_files(
@router.delete('/all')
async def delete_all_files(user=Depends(get_admin_user), db: Session = Depends(get_session)):
result = Files.delete_all_files(db=db)
async def delete_all_files(user=Depends(get_admin_user), db: AsyncSession = Depends(get_async_session)):
result = await Files.delete_all_files(db=db)
if result:
try:
Storage.delete_all_files()
@@ -412,8 +412,8 @@ async def delete_all_files(user=Depends(get_admin_user), db: Session = Depends(g
@router.get('/{id}', response_model=Optional[FileModel])
async def get_file_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
file = Files.get_file_by_id(id, db=db)
async def get_file_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -421,7 +421,7 @@ async def get_file_by_id(id: str, user=Depends(get_verified_user), db: Session =
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
return file
else:
raise HTTPException(
@@ -435,9 +435,9 @@ async def get_file_process_status(
id: str,
stream: bool = Query(False),
user=Depends(get_verified_user),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
file = Files.get_file_by_id(id, db=db)
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -445,7 +445,7 @@ async def get_file_process_status(
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
if stream:
MAX_FILE_PROCESSING_DURATION = 3600 * 2
@@ -454,7 +454,7 @@ async def get_file_process_status(
# Each poll creates its own short-lived session to avoid holding a
# connection for hours. A WebSocket push would be more efficient.
for _ in range(MAX_FILE_PROCESSING_DURATION):
file_item = Files.get_file_by_id(file_id) # Creates own session
file_item = await Files.get_file_by_id(file_id) # Creates own session
if file_item:
data = file_item.model_dump().get('data', {})
status = data.get('status')
@@ -495,8 +495,8 @@ async def get_file_process_status(
@router.get('/{id}/data/content')
async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
file = Files.get_file_by_id(id, db=db)
async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -504,7 +504,7 @@ async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user),
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
return {'content': file.data.get('content', '')}
else:
raise HTTPException(
@@ -523,14 +523,14 @@ class ContentForm(BaseModel):
@router.post('/{id}/data/content/update')
def update_file_data_content_by_id(
async def update_file_data_content_by_id(
request: Request,
id: str,
form_data: ContentForm,
user=Depends(get_verified_user),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
file = Files.get_file_by_id(id, db=db)
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -538,7 +538,7 @@ def update_file_data_content_by_id(
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'write', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db):
try:
process_file(
request,
@@ -546,7 +546,7 @@ def update_file_data_content_by_id(
user=user,
db=db,
)
file = Files.get_file_by_id(id=id, db=db)
file = await Files.get_file_by_id(id=id, db=db)
except Exception as e:
log.exception(e)
log.error(f'Error processing file: {file.id}')
@@ -554,7 +554,7 @@ def update_file_data_content_by_id(
# Propagate content change to all knowledge collections referencing
# this file. Without this the old embeddings remain in the knowledge
# collection and RAG returns both stale and current data (#20558).
knowledges = Knowledges.get_knowledges_by_file_id(id, db=db)
knowledges = await Knowledges.get_knowledges_by_file_id(id, db=db)
for knowledge in knowledges:
try:
# Remove old embeddings for this file from the KB collection
@@ -587,9 +587,9 @@ async def get_file_content_by_id(
id: str,
user=Depends(get_verified_user),
attachment: bool = Query(False),
db: Session = Depends(get_session),
db: AsyncSession = Depends(get_async_session),
):
file = Files.get_file_by_id(id, db=db)
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -597,7 +597,7 @@ async def get_file_content_by_id(
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
try:
file_path = Storage.get_file(file.path)
file_path = Path(file_path)
@@ -646,8 +646,8 @@ async def get_file_content_by_id(
@router.get('/{id}/content/html')
async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
file = Files.get_file_by_id(id, db=db)
async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -655,14 +655,14 @@ async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user),
detail=ERROR_MESSAGES.NOT_FOUND,
)
file_user = Users.get_user_by_id(file.user_id, db=db)
file_user = await Users.get_user_by_id(file.user_id, db=db)
if not file_user or file_user.role != 'admin':
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
try:
file_path = Storage.get_file(file.path)
file_path = Path(file_path)
@@ -693,8 +693,8 @@ async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user),
@router.get('/{id}/content/{file_name}')
async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
file = Files.get_file_by_id(id, db=db)
async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -702,7 +702,7 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: S
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
file_path = file.path
# Handle Unicode filenames
@@ -749,8 +749,8 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: S
@router.delete('/{id}')
async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
file = Files.get_file_by_id(id, db=db)
async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
file = await Files.get_file_by_id(id, db=db)
if not file:
raise HTTPException(
@@ -758,12 +758,12 @@ async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Sessio
detail=ERROR_MESSAGES.NOT_FOUND,
)
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'write', user, db=db):
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db):
# Clean up KB associations and embeddings before deleting
knowledges = Knowledges.get_knowledges_by_file_id(id, db=db)
knowledges = await Knowledges.get_knowledges_by_file_id(id, db=db)
for knowledge in knowledges:
# Remove KB-file relationship
Knowledges.remove_file_from_knowledge_by_id(knowledge.id, id, db=db)
await Knowledges.remove_file_from_knowledge_by_id(knowledge.id, id, db=db)
# Clean KB embeddings (same logic as /knowledge/{id}/file/remove)
try:
VECTOR_DB_CLIENT.delete(collection_name=knowledge.id, filter={'file_id': id})
@@ -772,7 +772,7 @@ async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Sessio
except Exception as e:
log.debug(f'KB embedding cleanup for {knowledge.id}: {e}')
result = Files.delete_file_by_id(id, db=db)
result = await Files.delete_file_by_id(id, db=db)
if result:
try:
Storage.delete_file(file.path)