fix: index the chat queries that make large SQLite instances unusable (#27663)
The timer scheduler polls once a second and cancels on every message send and chat open, the sidebar lists chats ordered by `updated_at`, and the folder badges count unread chats per folder. None of those could be served by an index, so each call read most of the `chat` table, and because `meta` sits after the chat payload column SQLite had to walk every row's overflow pages to get there. On a large history that stalls the sidebar, every chat switch and every send, and the idle poll alone burns about a quarter of a CPU core. Timers now keep their due time in a dedicated `chat.timer_at` column behind a partial index, and the chat list, unread and unfinished-reply queries each get an index matching their filter and ordering. Existing pending timers are backfilled from their meta by the migration. Dropping the `internal` and `type` checks also makes a forked timer chat inert, where a fork used to copy `meta` verbatim and become a second claim target that could fire a duplicate timer. Measured on SQLite, same rows returned: | query | before | after | |---|---|---| | idle timer poll (2000 chats, 0.43 GB) | 170 ms | 0.04 ms | | cancel on send and chat open (4000 chats, 377 MB) | 200 ms | 0.04 ms | | sidebar chat list (15000 chats, 1.26 GB) | 157 ms | 1.8 ms | | folder unread badges (15000 chats, 1.4 GB) | 54 ms | 0.2 ms | PostgreSQL 17 serves all of them as index-only scans with no sort node. Exercised through fresh install, upgrade with seeded data, downgrade and re-upgrade on SQLite and PostgreSQL 17. Fixes #27622
This commit is contained in:
1 parent
29e8d7db67
commit
16c2a9eda4
4 files changed
+103
-9
No files matched your search
@@ -152,6 +152,7 @@ async def create_timer(
|
||||
'cancel_on': selected_events,
|
||||
'run': run,
|
||||
},
|
||||
timer_at=due_at,
|
||||
)
|
||||
if not chat:
|
||||
return 'Error: failed to create timer.'
|
||||
@@ -171,20 +172,18 @@ async def claim_due_timers(now_ns: int, limit: int = 10) -> list[tuple[str, str]
|
||||
async with get_async_db() as db:
|
||||
stmt = (
|
||||
select(Chat)
|
||||
.where(Chat.meta['internal'].as_boolean().is_(True))
|
||||
.where(Chat.meta['type'].as_string() == 'timer')
|
||||
.where(Chat.timer_at <= now_ns)
|
||||
.where(Chat.meta['status'].as_string() == 'pending')
|
||||
.order_by(Chat.timer_at)
|
||||
.limit(limit)
|
||||
)
|
||||
if db.bind.dialect.name == 'postgresql':
|
||||
stmt = stmt.with_for_update(skip_locked=True)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = [row for row in result.scalars().all() if int((row.meta or {}).get('timer_at') or 0) <= now_ns]
|
||||
rows.sort(key=lambda row: int((row.meta or {}).get('timer_at') or 0))
|
||||
rows = rows[:limit]
|
||||
|
||||
claimed = []
|
||||
for row in rows:
|
||||
for row in result.scalars().all():
|
||||
claim_id = str(uuid4())
|
||||
row.meta = {
|
||||
**(row.meta or {}),
|
||||
@@ -192,6 +191,7 @@ async def claim_due_timers(now_ns: int, limit: int = 10) -> list[tuple[str, str]
|
||||
'timer_started_at': now_ns,
|
||||
'timer_claim_id': claim_id,
|
||||
}
|
||||
row.timer_at = None
|
||||
row.updated_at = int(time.time())
|
||||
claimed.append((row.id, claim_id))
|
||||
await db.commit()
|
||||
@@ -205,9 +205,8 @@ async def cancel_timers_for_chat(
|
||||
async with get_async_db() as db:
|
||||
result = await db.execute(
|
||||
select(Chat)
|
||||
.where(Chat.timer_at.isnot(None))
|
||||
.where(Chat.user_id == user_id)
|
||||
.where(Chat.meta['internal'].as_boolean().is_(True))
|
||||
.where(Chat.meta['type'].as_string() == 'timer')
|
||||
.where(Chat.meta['parent_chat_id'].as_string() == parent_chat_id)
|
||||
.where(Chat.meta['status'].as_string() == 'pending')
|
||||
)
|
||||
@@ -222,6 +221,7 @@ async def cancel_timers_for_chat(
|
||||
'timer_cancelled_at': now_ns,
|
||||
'timer_cancelled_by': event,
|
||||
}
|
||||
row.timer_at = None
|
||||
row.updated_at = int(time.time())
|
||||
await db.commit()
|
||||
|
||||
@@ -289,12 +289,14 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
|
||||
if await has_active_tasks(app.state.redis, parent_chat_id):
|
||||
timer_row = await db.get(Chat, timer_id)
|
||||
if timer_row:
|
||||
timer_meta = timer_row.meta or {}
|
||||
timer_row.meta = {
|
||||
**(timer_row.meta or {}),
|
||||
**timer_meta,
|
||||
'status': 'pending',
|
||||
'timer_claim_id': None,
|
||||
'timer_started_at': None,
|
||||
}
|
||||
timer_row.timer_at = timer_meta.get('timer_at')
|
||||
timer_row.updated_at = int(time.time())
|
||||
await db.commit()
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user