fix: index the chat queries that make large SQLite instances unusable (#27663)
The timer scheduler polls once a second and cancels on every message send and chat open, the sidebar lists chats ordered by `updated_at`, and the folder badges count unread chats per folder. None of those could be served by an index, so each call read most of the `chat` table, and because `meta` sits after the chat payload column SQLite had to walk every row's overflow pages to get there. On a large history that stalls the sidebar, every chat switch and every send, and the idle poll alone burns about a quarter of a CPU core. Timers now keep their due time in a dedicated `chat.timer_at` column behind a partial index, and the chat list, unread and unfinished-reply queries each get an index matching their filter and ordering. Existing pending timers are backfilled from their meta by the migration. Dropping the `internal` and `type` checks also makes a forked timer chat inert, where a fork used to copy `meta` verbatim and become a second claim target that could fire a duplicate timer. Measured on SQLite, same rows returned: | query | before | after | |---|---|---| | idle timer poll (2000 chats, 0.43 GB) | 170 ms | 0.04 ms | | cancel on send and chat open (4000 chats, 377 MB) | 200 ms | 0.04 ms | | sidebar chat list (15000 chats, 1.26 GB) | 157 ms | 1.8 ms | | folder unread badges (15000 chats, 1.4 GB) | 54 ms | 0.2 ms | PostgreSQL 17 serves all of them as index-only scans with no sort node. Exercised through fresh install, upgrade with seeded data, downgrade and re-upgrade on SQLite and PostgreSQL 17. Fixes #27622
This commit is contained in:
1 parent
29e8d7db67
commit
16c2a9eda4
4 files changed
+103
-9
No files matched your search
+70
@@ -0,0 +1,70 @@
|
||||
"""add chat timer_at and chat list, unread and timer indexes
|
||||
|
||||
Revision ID: d4c1a8e37b62
|
||||
Revises: 6d09d1bf1f23
|
||||
Create Date: 2026-08-23 18:05:12.441907
|
||||
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'd4c1a8e37b62'
|
||||
down_revision: str | None = '6d09d1bf1f23'
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('chat', sa.Column('timer_at', sa.BigInteger(), nullable=True))
|
||||
op.create_index(
|
||||
'timer_at_idx',
|
||||
'chat',
|
||||
['timer_at'],
|
||||
sqlite_where=sa.text('timer_at IS NOT NULL'),
|
||||
postgresql_where=sa.text('timer_at IS NOT NULL'),
|
||||
)
|
||||
|
||||
# Timers created before this migration carry their due time in meta only, and would never fire.
|
||||
chat = sa.table(
|
||||
'chat', sa.column('id', sa.String), sa.column('meta', sa.JSON), sa.column('timer_at', sa.BigInteger)
|
||||
)
|
||||
conn = op.get_bind()
|
||||
pending = conn.execute(
|
||||
sa.select(chat.c.id, chat.c.meta)
|
||||
.where(chat.c.meta['type'].as_string() == 'timer')
|
||||
.where(chat.c.meta['status'].as_string() == 'pending')
|
||||
).all()
|
||||
for chat_id, meta in pending:
|
||||
try: # imported chats can carry any meta, and a non-numeric due time must not abort the migration
|
||||
due_at = int(meta.get('timer_at'))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
conn.execute(chat.update().where(chat.c.id == chat_id).values(timer_at=due_at))
|
||||
|
||||
op.create_index('user_id_updated_at_id_idx', 'chat', ['user_id', sa.text('updated_at DESC'), 'id'])
|
||||
op.create_index(
|
||||
'user_id_timer_at_idx',
|
||||
'chat',
|
||||
['user_id', 'timer_at'],
|
||||
sqlite_where=sa.text('timer_at IS NOT NULL'),
|
||||
postgresql_where=sa.text('timer_at IS NOT NULL'),
|
||||
)
|
||||
op.create_index(
|
||||
'user_id_folder_unread_idx',
|
||||
'chat',
|
||||
['user_id', 'folder_id', 'archived', 'updated_at', 'last_read_at', 'id'],
|
||||
)
|
||||
op.create_index('chat_message_chat_role_done_idx', 'chat_message', ['chat_id', 'role', 'done'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('chat_message_chat_role_done_idx', table_name='chat_message')
|
||||
op.drop_index('user_id_folder_unread_idx', table_name='chat')
|
||||
op.drop_index('user_id_timer_at_idx', table_name='chat')
|
||||
op.drop_index('user_id_updated_at_id_idx', table_name='chat')
|
||||
op.drop_index('timer_at_idx', table_name='chat')
|
||||
op.drop_column('chat', 'timer_at')
|
||||
Reference in new issue
Block a user