perf: stop scanning every skill on each listing and chat turn (#28798)
Listing skills ran one database query per skill in the instance. A non-admin opening the list on a workspace with 500 skills issued over 500 queries, the paginated list re-resolved the caller's group membership once per row, and every chat message carrying a skill loaded every skill the user can read, full body and owner included, to use the two or three it actually referenced. Skills now arrive already filtered: the owner-or-grant check runs in the query as an EXISTS subquery, the same way prompts and the search endpoints already do it, the per-item write flag uses the existing batch grant lookup, and the chat path asks only for the skill ids the request names. Measured with 500 skills of which 3 are visible to the caller: 504 queries and ~300 ms before, 4 queries and ~2.6 ms after. The resulting set is unchanged for owner, public, direct-user, group and multi-grant entries, for both read and write.
This commit is contained in:
1 parent
9bdb072690
commit
dbf715cb63
3 files changed
+40
-58
No files matched your search
@@ -163,9 +163,30 @@ class SkillsTable:
|
|||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_skills(self, db: Optional[AsyncSession] = None) -> list[SkillUserModel]:
|
async def get_skills(
|
||||||
|
self,
|
||||||
|
user_id: str | None = None,
|
||||||
|
ids: list[str] | None = None,
|
||||||
|
db: AsyncSession | None = None,
|
||||||
|
) -> list[SkillUserModel]:
|
||||||
async with get_async_db_context(db) as db:
|
async with get_async_db_context(db) as db:
|
||||||
result = await db.execute(select(Skill).order_by(Skill.updated_at.desc()))
|
stmt = select(Skill).order_by(Skill.updated_at.desc())
|
||||||
|
|
||||||
|
if ids is not None:
|
||||||
|
stmt = stmt.filter(Skill.id.in_(ids))
|
||||||
|
|
||||||
|
if user_id is not None:
|
||||||
|
user_group_ids = {group.id for group in await Groups.get_groups_by_member_id(user_id, db=db)}
|
||||||
|
stmt = AccessGrants.has_permission_filter(
|
||||||
|
db=db,
|
||||||
|
query=stmt,
|
||||||
|
DocumentModel=Skill,
|
||||||
|
filter={'user_id': user_id, 'group_ids': user_group_ids},
|
||||||
|
resource_type='skill',
|
||||||
|
permission='read',
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await db.execute(stmt)
|
||||||
all_skills = result.scalars().all()
|
all_skills = result.scalars().all()
|
||||||
|
|
||||||
user_ids = list(set(skill.user_id for skill in all_skills))
|
user_ids = list(set(skill.user_id for skill in all_skills))
|
||||||
@@ -194,28 +215,6 @@ class SkillsTable:
|
|||||||
)
|
)
|
||||||
return skills
|
return skills
|
||||||
|
|
||||||
async def get_skills_by_user_id(
|
|
||||||
self, user_id: str, permission: str = 'write', db: Optional[AsyncSession] = None
|
|
||||||
) -> list[SkillUserModel]:
|
|
||||||
skills = await self.get_skills(db=db)
|
|
||||||
user_groups = await Groups.get_groups_by_member_id(user_id, db=db)
|
|
||||||
user_group_ids = {group.id for group in user_groups}
|
|
||||||
|
|
||||||
result = []
|
|
||||||
for skill in skills:
|
|
||||||
if skill.user_id == user_id:
|
|
||||||
result.append(skill)
|
|
||||||
elif await AccessGrants.has_access(
|
|
||||||
user_id=user_id,
|
|
||||||
resource_type='skill',
|
|
||||||
resource_id=skill.id,
|
|
||||||
permission=permission,
|
|
||||||
user_group_ids=user_group_ids,
|
|
||||||
db=db,
|
|
||||||
):
|
|
||||||
result.append(skill)
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def search_skills(
|
async def search_skills(
|
||||||
self,
|
self,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
|||||||
@@ -46,21 +46,7 @@ async def get_skills(
|
|||||||
if user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL:
|
if user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL:
|
||||||
skills = await Skills.get_skills(db=db)
|
skills = await Skills.get_skills(db=db)
|
||||||
else:
|
else:
|
||||||
user_group_ids = {group.id for group in await Groups.get_groups_by_member_id(user.id, db=db)}
|
skills = await Skills.get_skills(db=db, user_id=user.id)
|
||||||
all_skills = await Skills.get_skills(db=db)
|
|
||||||
skills = [
|
|
||||||
skill
|
|
||||||
for skill in all_skills
|
|
||||||
if skill.user_id == user.id
|
|
||||||
or await AccessGrants.has_access(
|
|
||||||
user_id=user.id,
|
|
||||||
resource_type='skill',
|
|
||||||
resource_id=skill.id,
|
|
||||||
permission='read',
|
|
||||||
user_group_ids=user_group_ids,
|
|
||||||
db=db,
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
if query:
|
if query:
|
||||||
q = query.casefold()
|
q = query.casefold()
|
||||||
@@ -99,30 +85,29 @@ async def get_skill_list(
|
|||||||
if direction:
|
if direction:
|
||||||
filter['direction'] = direction
|
filter['direction'] = direction
|
||||||
|
|
||||||
if not (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL):
|
is_bypass_admin = user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL
|
||||||
groups = await Groups.get_groups_by_member_id(user.id, db=db)
|
user_group_ids = {group.id for group in await Groups.get_groups_by_member_id(user.id, db=db)}
|
||||||
if groups:
|
|
||||||
filter['group_ids'] = [group.id for group in groups]
|
|
||||||
|
|
||||||
|
if not is_bypass_admin:
|
||||||
|
filter['group_ids'] = user_group_ids
|
||||||
filter['user_id'] = user.id
|
filter['user_id'] = user.id
|
||||||
|
|
||||||
result = await Skills.search_skills(user.id, filter=filter, skip=skip, limit=limit, db=db)
|
result = await Skills.search_skills(user.id, filter=filter, skip=skip, limit=limit, db=db)
|
||||||
|
|
||||||
|
writable_skill_ids = await AccessGrants.get_accessible_resource_ids(
|
||||||
|
user_id=user.id,
|
||||||
|
resource_type='skill',
|
||||||
|
resource_ids=[skill.id for skill in result.items],
|
||||||
|
permission='write',
|
||||||
|
user_group_ids=user_group_ids,
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
|
||||||
return SkillAccessListResponse(
|
return SkillAccessListResponse(
|
||||||
items=[
|
items=[
|
||||||
SkillAccessResponse(
|
SkillAccessResponse(
|
||||||
**skill.model_dump(),
|
**skill.model_dump(),
|
||||||
write_access=(
|
write_access=(is_bypass_admin or user.id == skill.user_id or skill.id in writable_skill_ids),
|
||||||
(user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL)
|
|
||||||
or user.id == skill.user_id
|
|
||||||
or await AccessGrants.has_access(
|
|
||||||
user_id=user.id,
|
|
||||||
resource_type='skill',
|
|
||||||
resource_id=skill.id,
|
|
||||||
permission='write',
|
|
||||||
db=db,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
for skill in result.items
|
for skill in result.items
|
||||||
],
|
],
|
||||||
@@ -155,7 +140,7 @@ async def export_skills(
|
|||||||
if user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL:
|
if user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL:
|
||||||
return await Skills.get_skills(db=db)
|
return await Skills.get_skills(db=db)
|
||||||
else:
|
else:
|
||||||
return await Skills.get_skills_by_user_id(user.id, 'read', db=db)
|
return await Skills.get_skills(db=db, user_id=user.id)
|
||||||
|
|
||||||
|
|
||||||
############################
|
############################
|
||||||
|
|||||||
@@ -2643,9 +2643,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||||||
if skill_ids:
|
if skill_ids:
|
||||||
from open_webui.models.skills import Skills as SkillsModel
|
from open_webui.models.skills import Skills as SkillsModel
|
||||||
|
|
||||||
# Reuse the rows from the access query instead of re-fetching each
|
accessible_skills = {s.id: s for s in await SkillsModel.get_skills(user_id=user.id, ids=skill_ids)}
|
||||||
# skill by id.
|
|
||||||
accessible_skills = {s.id: s for s in await SkillsModel.get_skills_by_user_id(user.id, 'read')}
|
|
||||||
for sid in skill_ids:
|
for sid in skill_ids:
|
||||||
s = accessible_skills.get(sid)
|
s = accessible_skills.get(sid)
|
||||||
if s and s.is_active:
|
if s and s.is_active:
|
||||||
|
|||||||
Reference in new issue
Block a user