refac: async db
This commit is contained in:
1 parent
b618d84065
commit
27169124f2
74 files changed
+4831
-4479
No files matched your search
@@ -21,8 +21,8 @@ from fastapi import (
|
||||
)
|
||||
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from open_webui.internal.db import get_session, SessionLocal
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from open_webui.internal.db import get_async_session, SessionLocal
|
||||
|
||||
from open_webui.constants import ERROR_MESSAGES
|
||||
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
|
||||
@@ -88,16 +88,16 @@ def _is_text_file(file_path: str, chunk_size: int = 8192) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def process_uploaded_file(
|
||||
async def process_uploaded_file(
|
||||
request,
|
||||
file,
|
||||
file_path,
|
||||
file_item,
|
||||
file_metadata,
|
||||
user,
|
||||
db: Optional[Session] = None,
|
||||
db: Optional[AsyncSession] = None,
|
||||
):
|
||||
def _process_handler(db_session):
|
||||
async def _process_handler(db_session):
|
||||
try:
|
||||
content_type = file.content_type
|
||||
|
||||
@@ -141,7 +141,7 @@ def process_uploaded_file(
|
||||
|
||||
except Exception as e:
|
||||
log.error(f'Error processing file: {file_item.id}')
|
||||
Files.update_file_data_by_id(
|
||||
await Files.update_file_data_by_id(
|
||||
file_item.id,
|
||||
{
|
||||
'status': 'failed',
|
||||
@@ -158,7 +158,7 @@ def process_uploaded_file(
|
||||
|
||||
|
||||
@router.post('/', response_model=FileModelResponse)
|
||||
def upload_file(
|
||||
async def upload_file(
|
||||
request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
@@ -166,9 +166,9 @@ def upload_file(
|
||||
process: bool = Query(True),
|
||||
process_in_background: bool = Query(True),
|
||||
user=Depends(get_verified_user),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
return upload_file_handler(
|
||||
return await upload_file_handler(
|
||||
request,
|
||||
file=file,
|
||||
metadata=metadata,
|
||||
@@ -180,7 +180,7 @@ def upload_file(
|
||||
)
|
||||
|
||||
|
||||
def upload_file_handler(
|
||||
async def upload_file_handler(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
metadata: Optional[dict | str] = Form(None),
|
||||
@@ -188,7 +188,7 @@ def upload_file_handler(
|
||||
process_in_background: bool = Query(True),
|
||||
user=Depends(get_verified_user),
|
||||
background_tasks: Optional[BackgroundTasks] = None,
|
||||
db: Optional[Session] = None,
|
||||
db: Optional[AsyncSession] = None,
|
||||
):
|
||||
log.info(f'file.content_type: {file.content_type} {process}')
|
||||
|
||||
@@ -236,7 +236,7 @@ def upload_file_handler(
|
||||
},
|
||||
)
|
||||
|
||||
file_item = Files.insert_new_file(
|
||||
file_item = await Files.insert_new_file(
|
||||
user.id,
|
||||
FileForm(
|
||||
**{
|
||||
@@ -258,9 +258,9 @@ def upload_file_handler(
|
||||
)
|
||||
|
||||
if 'channel_id' in file_metadata:
|
||||
channel = Channels.get_channel_by_id_and_user_id(file_metadata['channel_id'], user.id, db=db)
|
||||
channel = await Channels.get_channel_by_id_and_user_id(file_metadata['channel_id'], user.id, db=db)
|
||||
if channel:
|
||||
Channels.add_file_to_channel_by_id(channel.id, file_item.id, user.id, db=db)
|
||||
await Channels.add_file_to_channel_by_id(channel.id, file_item.id, user.id, db=db)
|
||||
|
||||
if process:
|
||||
if background_tasks and process_in_background:
|
||||
@@ -275,7 +275,7 @@ def upload_file_handler(
|
||||
)
|
||||
return {'status': True, **file_item.model_dump()}
|
||||
else:
|
||||
process_uploaded_file(
|
||||
await process_uploaded_file(
|
||||
request,
|
||||
file,
|
||||
file_path,
|
||||
@@ -317,12 +317,12 @@ async def list_files(
|
||||
user=Depends(get_verified_user),
|
||||
page: int = Query(1, ge=1, description='Page number (1-indexed)'),
|
||||
content: bool = Query(True),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
skip = (page - 1) * PAGE_SIZE
|
||||
user_id = None if (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL) else user.id
|
||||
|
||||
result = Files.get_file_list(user_id=user_id, skip=skip, limit=PAGE_SIZE, db=db)
|
||||
result = await Files.get_file_list(user_id=user_id, skip=skip, limit=PAGE_SIZE, db=db)
|
||||
|
||||
if not content:
|
||||
for file in result.items:
|
||||
@@ -347,7 +347,7 @@ async def search_files(
|
||||
skip: int = Query(0, ge=0, description='Number of files to skip'),
|
||||
limit: int = Query(100, ge=1, le=1000, description='Maximum number of files to return'),
|
||||
user=Depends(get_verified_user),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
"""
|
||||
Search for files by filename with support for wildcard patterns.
|
||||
@@ -357,7 +357,7 @@ async def search_files(
|
||||
user_id = None if (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL) else user.id
|
||||
|
||||
# Use optimized database query with pagination
|
||||
files = Files.search_files(
|
||||
files = await Files.search_files(
|
||||
user_id=user_id,
|
||||
filename=filename,
|
||||
skip=skip,
|
||||
@@ -385,8 +385,8 @@ async def search_files(
|
||||
|
||||
|
||||
@router.delete('/all')
|
||||
async def delete_all_files(user=Depends(get_admin_user), db: Session = Depends(get_session)):
|
||||
result = Files.delete_all_files(db=db)
|
||||
async def delete_all_files(user=Depends(get_admin_user), db: AsyncSession = Depends(get_async_session)):
|
||||
result = await Files.delete_all_files(db=db)
|
||||
if result:
|
||||
try:
|
||||
Storage.delete_all_files()
|
||||
@@ -412,8 +412,8 @@ async def delete_all_files(user=Depends(get_admin_user), db: Session = Depends(g
|
||||
|
||||
|
||||
@router.get('/{id}', response_model=Optional[FileModel])
|
||||
async def get_file_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
async def get_file_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -421,7 +421,7 @@ async def get_file_by_id(id: str, user=Depends(get_verified_user), db: Session =
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
return file
|
||||
else:
|
||||
raise HTTPException(
|
||||
@@ -435,9 +435,9 @@ async def get_file_process_status(
|
||||
id: str,
|
||||
stream: bool = Query(False),
|
||||
user=Depends(get_verified_user),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -445,7 +445,7 @@ async def get_file_process_status(
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
if stream:
|
||||
MAX_FILE_PROCESSING_DURATION = 3600 * 2
|
||||
|
||||
@@ -454,7 +454,7 @@ async def get_file_process_status(
|
||||
# Each poll creates its own short-lived session to avoid holding a
|
||||
# connection for hours. A WebSocket push would be more efficient.
|
||||
for _ in range(MAX_FILE_PROCESSING_DURATION):
|
||||
file_item = Files.get_file_by_id(file_id) # Creates own session
|
||||
file_item = await Files.get_file_by_id(file_id) # Creates own session
|
||||
if file_item:
|
||||
data = file_item.model_dump().get('data', {})
|
||||
status = data.get('status')
|
||||
@@ -495,8 +495,8 @@ async def get_file_process_status(
|
||||
|
||||
|
||||
@router.get('/{id}/data/content')
|
||||
async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -504,7 +504,7 @@ async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user),
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
return {'content': file.data.get('content', '')}
|
||||
else:
|
||||
raise HTTPException(
|
||||
@@ -523,14 +523,14 @@ class ContentForm(BaseModel):
|
||||
|
||||
|
||||
@router.post('/{id}/data/content/update')
|
||||
def update_file_data_content_by_id(
|
||||
async def update_file_data_content_by_id(
|
||||
request: Request,
|
||||
id: str,
|
||||
form_data: ContentForm,
|
||||
user=Depends(get_verified_user),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -538,7 +538,7 @@ def update_file_data_content_by_id(
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'write', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db):
|
||||
try:
|
||||
process_file(
|
||||
request,
|
||||
@@ -546,7 +546,7 @@ def update_file_data_content_by_id(
|
||||
user=user,
|
||||
db=db,
|
||||
)
|
||||
file = Files.get_file_by_id(id=id, db=db)
|
||||
file = await Files.get_file_by_id(id=id, db=db)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
log.error(f'Error processing file: {file.id}')
|
||||
@@ -554,7 +554,7 @@ def update_file_data_content_by_id(
|
||||
# Propagate content change to all knowledge collections referencing
|
||||
# this file. Without this the old embeddings remain in the knowledge
|
||||
# collection and RAG returns both stale and current data (#20558).
|
||||
knowledges = Knowledges.get_knowledges_by_file_id(id, db=db)
|
||||
knowledges = await Knowledges.get_knowledges_by_file_id(id, db=db)
|
||||
for knowledge in knowledges:
|
||||
try:
|
||||
# Remove old embeddings for this file from the KB collection
|
||||
@@ -587,9 +587,9 @@ async def get_file_content_by_id(
|
||||
id: str,
|
||||
user=Depends(get_verified_user),
|
||||
attachment: bool = Query(False),
|
||||
db: Session = Depends(get_session),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -597,7 +597,7 @@ async def get_file_content_by_id(
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
try:
|
||||
file_path = Storage.get_file(file.path)
|
||||
file_path = Path(file_path)
|
||||
@@ -646,8 +646,8 @@ async def get_file_content_by_id(
|
||||
|
||||
|
||||
@router.get('/{id}/content/html')
|
||||
async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -655,14 +655,14 @@ async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user),
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
file_user = Users.get_user_by_id(file.user_id, db=db)
|
||||
file_user = await Users.get_user_by_id(file.user_id, db=db)
|
||||
if not file_user or file_user.role != 'admin':
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
try:
|
||||
file_path = Storage.get_file(file.path)
|
||||
file_path = Path(file_path)
|
||||
@@ -693,8 +693,8 @@ async def get_html_file_content_by_id(id: str, user=Depends(get_verified_user),
|
||||
|
||||
|
||||
@router.get('/{id}/content/{file_name}')
|
||||
async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -702,7 +702,7 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: S
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'read', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'read', user, db=db):
|
||||
file_path = file.path
|
||||
|
||||
# Handle Unicode filenames
|
||||
@@ -749,8 +749,8 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user), db: S
|
||||
|
||||
|
||||
@router.delete('/{id}')
|
||||
async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Session = Depends(get_session)):
|
||||
file = Files.get_file_by_id(id, db=db)
|
||||
async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session)):
|
||||
file = await Files.get_file_by_id(id, db=db)
|
||||
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
@@ -758,12 +758,12 @@ async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Sessio
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if file.user_id == user.id or user.role == 'admin' or has_access_to_file(id, 'write', user, db=db):
|
||||
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db):
|
||||
# Clean up KB associations and embeddings before deleting
|
||||
knowledges = Knowledges.get_knowledges_by_file_id(id, db=db)
|
||||
knowledges = await Knowledges.get_knowledges_by_file_id(id, db=db)
|
||||
for knowledge in knowledges:
|
||||
# Remove KB-file relationship
|
||||
Knowledges.remove_file_from_knowledge_by_id(knowledge.id, id, db=db)
|
||||
await Knowledges.remove_file_from_knowledge_by_id(knowledge.id, id, db=db)
|
||||
# Clean KB embeddings (same logic as /knowledge/{id}/file/remove)
|
||||
try:
|
||||
VECTOR_DB_CLIENT.delete(collection_name=knowledge.id, filter={'file_id': id})
|
||||
@@ -772,7 +772,7 @@ async def delete_file_by_id(id: str, user=Depends(get_verified_user), db: Sessio
|
||||
except Exception as e:
|
||||
log.debug(f'KB embedding cleanup for {knowledge.id}: {e}')
|
||||
|
||||
result = Files.delete_file_by_id(id, db=db)
|
||||
result = await Files.delete_file_by_id(id, db=db)
|
||||
if result:
|
||||
try:
|
||||
Storage.delete_file(file.path)
|
||||
|
||||
Reference in new issue
Block a user