This commit is contained in:
Timothy Jaeryang Baek committed 2026-07-16 00:39:31 -04:00
1 parent 92fbc77877
commit 771a024b40
3 files changed
+69 -41

No files matched your search

+15 -3
View File
@@ -11,6 +11,7 @@ The scheduler_worker_loop handles all time-based background work:
Environment: Environment:
SCHEDULER_POLL_INTERVAL – seconds between polls (default: 10) SCHEDULER_POLL_INTERVAL – seconds between polls (default: 10)
TIMER_POLL_INTERVAL – seconds between timer polls (default: 1)
CALENDAR_ALERT_LOOKAHEAD_MINUTES – default alert window (default: 5) CALENDAR_ALERT_LOOKAHEAD_MINUTES – default alert window (default: 5)
""" """
@@ -43,6 +44,7 @@ from starlette.datastructures import Headers
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
SCHEDULER_POLL_INTERVAL = int(os.getenv('SCHEDULER_POLL_INTERVAL', os.getenv('AUTOMATION_POLL_INTERVAL', '10'))) SCHEDULER_POLL_INTERVAL = int(os.getenv('SCHEDULER_POLL_INTERVAL', os.getenv('AUTOMATION_POLL_INTERVAL', '10')))
TIMER_POLL_INTERVAL = int(os.getenv('TIMER_POLL_INTERVAL', '1'))
CALENDAR_ALERT_LOOKAHEAD_MINUTES = int(os.getenv('CALENDAR_ALERT_LOOKAHEAD_MINUTES', '10')) CALENDAR_ALERT_LOOKAHEAD_MINUTES = int(os.getenv('CALENDAR_ALERT_LOOKAHEAD_MINUTES', '10'))
@@ -190,10 +192,15 @@ async def scheduler_worker_loop(app) -> None:
Runs on every instance. Poll interval is configurable via Runs on every instance. Poll interval is configurable via
SCHEDULER_POLL_INTERVAL env var (default: 10 seconds). SCHEDULER_POLL_INTERVAL env var (default: 10 seconds).
""" """
log.info(f'Scheduler worker started (poll interval: {SCHEDULER_POLL_INTERVAL}s)') log.info(
f'Scheduler worker started (timer poll interval: {TIMER_POLL_INTERVAL}s, '
f'scheduler poll interval: {SCHEDULER_POLL_INTERVAL}s)'
)
next_scheduler_poll = 0.0
while True: while True:
try: try:
now = time.monotonic()
# ── Timers ── # ── Timers ──
try: try:
from open_webui.utils.timers import claim_due_timers, execute_due_timer from open_webui.utils.timers import claim_due_timers, execute_due_timer
@@ -203,6 +210,12 @@ async def scheduler_worker_loop(app) -> None:
except Exception: except Exception:
log.exception('Scheduler: timer error') log.exception('Scheduler: timer error')
if now < next_scheduler_poll:
await asyncio.sleep(max(1, TIMER_POLL_INTERVAL))
continue
# Jitter to spread automation/calendar load across instances; timers keep a tight poll.
next_scheduler_poll = now + SCHEDULER_POLL_INTERVAL + random.uniform(0, 2)
# ── Automations ── # ── Automations ──
if await Config.get('automations.enable'): if await Config.get('automations.enable'):
try: try:
@@ -225,8 +238,7 @@ async def scheduler_worker_loop(app) -> None:
except Exception: except Exception:
log.exception('Scheduler worker error') log.exception('Scheduler worker error')
# Jitter to spread load across instances await asyncio.sleep(max(1, TIMER_POLL_INTERVAL))
await asyncio.sleep(SCHEDULER_POLL_INTERVAL + random.uniform(0, 2))
########################## ##########################
+35 -35
View File
@@ -9,13 +9,15 @@ from uuid import uuid4
from fastapi import Request from fastapi import Request
from fastapi.security import HTTPAuthorizationCredentials from fastapi.security import HTTPAuthorizationCredentials
from open_webui.internal.db import get_async_db
from open_webui.models.chat_messages import ChatMessages from open_webui.models.chat_messages import ChatMessages
from open_webui.models.chats import ChatForm, Chats from open_webui.models.chats import Chat, ChatForm, Chats
from open_webui.models.config import Config from open_webui.models.config import Config
from open_webui.models.users import UserModel, Users from open_webui.models.users import UserModel, Users
from open_webui.tasks import create_task, has_active_tasks from open_webui.tasks import create_task, has_active_tasks
from open_webui.utils.auth import create_token from open_webui.utils.auth import create_token
from open_webui.utils.misc import get_message_list from open_webui.utils.misc import get_message_list
from sqlalchemy import select
from starlette.datastructures import Headers from starlette.datastructures import Headers
DEFAULT_SUBAGENT_SYSTEM_PROMPT = """You are a sub-agent working on a specific task assigned by the lead agent. DEFAULT_SUBAGENT_SYSTEM_PROMPT = """You are a sub-agent working on a specific task assigned by the lead agent.
@@ -81,12 +83,20 @@ async def process_pending_internal_messages(
if await has_active_tasks(source_request.app.state.redis, parent_chat_id): if await has_active_tasks(source_request.app.state.redis, parent_chat_id):
return return
chat = await Chats.get_chat_by_id_and_user_id(parent_chat_id, user_id)
user = await Users.get_user_by_id(user_id) user = await Users.get_user_by_id(user_id)
if not chat or not user: if not user:
return return
history = copy.deepcopy(chat.chat.get('history') or {}) async with get_async_db() as db:
stmt = select(Chat).where(Chat.id == parent_chat_id, Chat.user_id == user_id)
if db.bind.dialect.name == 'postgresql':
stmt = stmt.with_for_update()
result = await db.execute(stmt)
chat = result.scalar_one_or_none()
if not chat:
return
history = copy.deepcopy((chat.chat or {}).get('history') or {})
messages = history.get('messages') or {} messages = history.get('messages') or {}
pending = [ pending = [
message message
@@ -113,14 +123,7 @@ async def process_pending_internal_messages(
run = {**run, **(((timer.meta or {}).get('run') if timer else None) or {})} run = {**run, **(((timer.meta or {}).get('run') if timer else None) or {})}
model_id = first.get('model') or run['model_id'] model_id = first.get('model') or run['model_id']
if kind == 'timer': if kind == 'timer':
batch = [ batch = [first]
message
for message in pending
if message.get('parentId') == parent_id
and (message.get('model') or model_id) == model_id
and (message.get('meta') or {}).get('internal') is True
and (message.get('meta') or {}).get('type') == 'timer'
]
else: else:
batch = [ batch = [
message message
@@ -131,7 +134,9 @@ async def process_pending_internal_messages(
] ]
combined_content = '\n\n'.join(message.get('content', '') for message in batch if message.get('content')) combined_content = '\n\n'.join(message.get('content', '') for message in batch if message.get('content'))
if kind == 'timer': if kind == 'timer':
timer_ids = [message['meta']['timer_id'] for message in batch if (message.get('meta') or {}).get('timer_id')] timer_ids = [
message['meta']['timer_id'] for message in batch if (message.get('meta') or {}).get('timer_id')
]
combined_meta = {'internal': True, 'type': 'timer'} combined_meta = {'internal': True, 'type': 'timer'}
if len(timer_ids) == 1: if len(timer_ids) == 1:
combined_meta['timer_id'] = timer_ids[0] combined_meta['timer_id'] = timer_ids[0]
@@ -161,20 +166,17 @@ async def process_pending_internal_messages(
pending_flag = 'timer_pending' if kind == 'timer' else 'async_subagent_pending' pending_flag = 'timer_pending' if kind == 'timer' else 'async_subagent_pending'
reuse_message = len(batch) == 1 and not (first.get('meta') or {}).get(pending_flag) reuse_message = len(batch) == 1 and not (first.get('meta') or {}).get(pending_flag)
user_message_id = first['id'] if reuse_message else str(uuid4()) user_message_id = first['id'] if reuse_message else str(uuid4())
removed_ids = set()
if not reuse_message: if not reuse_message:
removed_ids = {message['id'] for message in batch} removed_ids = {message['id'] for message in batch}
for message_id in removed_ids: for message_id in removed_ids:
messages.pop(message_id, None) messages.pop(message_id, None)
if parent_id and parent_id in messages: if parent_id and parent_id in messages:
messages[parent_id]['childrenIds'] = [ messages[parent_id]['childrenIds'] = [
child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id not in removed_ids child_id
for child_id in messages[parent_id].get('childrenIds', [])
if child_id not in removed_ids
] ]
history['messages'] = messages
history['currentId'] = parent_id
updated_chat = copy.deepcopy(chat.chat)
updated_chat['history'] = history
await Chats.update_chat_by_id(parent_chat_id, updated_chat)
await ChatMessages.delete_message_ids_by_chat_id(parent_chat_id, removed_ids)
assistant_message_id = str(uuid4()) assistant_message_id = str(uuid4())
message_list = get_message_list(messages, parent_id) message_list = get_message_list(messages, parent_id)
@@ -205,21 +207,19 @@ async def process_pending_internal_messages(
child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id != user_message_id child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id != user_message_id
] ]
parent_children.append(user_message_id) parent_children.append(user_message_id)
await Chats.upsert_message_to_chat_by_id_and_message_id( messages[parent_id]['childrenIds'] = parent_children
parent_chat_id, messages[user_message_id] = {**messages.get(user_message_id, {}), **user_message}
parent_id, messages[assistant_message_id] = assistant_message
{'childrenIds': parent_children}, history['messages'] = messages
) history['currentId'] = assistant_message_id
await Chats.upsert_message_to_chat_by_id_and_message_id( chat.chat = {**(chat.chat or {}), 'history': history}
parent_chat_id, chat.updated_at = int(time.time())
user_message_id, await db.commit()
user_message,
) if removed_ids:
await Chats.upsert_message_to_chat_by_id_and_message_id( await ChatMessages.delete_message_ids_by_chat_id(parent_chat_id, removed_ids)
parent_chat_id, await ChatMessages.upsert_message(user_message_id, parent_chat_id, user_id, user_message)
assistant_message_id, await ChatMessages.upsert_message(assistant_message_id, parent_chat_id, user_id, assistant_message)
assistant_message,
)
from open_webui.socket.main import sio from open_webui.socket.main import sio
+19 -3
View File
@@ -275,10 +275,16 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
user_message_id = str(uuid4()) user_message_id = str(uuid4())
parent_lock = _parent_locks.setdefault(parent_chat_id, asyncio.Lock()) parent_lock = _parent_locks.setdefault(parent_chat_id, asyncio.Lock())
async with parent_lock: async with parent_lock:
parent = await Chats.get_chat_by_id_and_user_id(parent_chat_id, timer.user_id) async with get_async_db() as db:
stmt = select(Chat).where(Chat.id == parent_chat_id, Chat.user_id == timer.user_id)
if db.bind.dialect.name == 'postgresql':
stmt = stmt.with_for_update()
result = await db.execute(stmt)
parent = result.scalar_one_or_none()
if not parent: if not parent:
await _set_timer_status(timer_id, 'error', timer_error='parent chat no longer exists') await _set_timer_status(timer_id, 'error', timer_error='parent chat no longer exists')
return return
parent_chat = copy.deepcopy(parent.chat or {}) parent_chat = copy.deepcopy(parent.chat or {})
history = parent_chat.setdefault('history', {}) history = parent_chat.setdefault('history', {})
messages = history.setdefault('messages', {}) messages = history.setdefault('messages', {})
@@ -311,9 +317,19 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
children = messages[parent_id].setdefault('childrenIds', []) children = messages[parent_id].setdefault('childrenIds', [])
if user_message_id not in children: if user_message_id not in children:
children.append(user_message_id) children.append(user_message_id)
await Chats.update_chat_by_id(parent_chat_id, parent_chat)
parent.chat = parent_chat
parent.updated_at = int(time.time())
timer_row = await db.get(Chat, timer_id)
if timer_row:
timer_row.meta = {
**(timer_row.meta or {}),
'timer_status': 'dispatched',
'timer_dispatched_at': int(time.time_ns()),
}
timer_row.updated_at = int(time.time())
await db.commit()
await ChatMessages.upsert_message(user_message_id, parent_chat_id, timer.user_id, user_message) await ChatMessages.upsert_message(user_message_id, parent_chat_id, timer.user_id, user_message)
await _set_timer_status(timer_id, 'dispatched', timer_dispatched_at=int(time.time_ns()))
if user_message['meta'].get('timer_pending') is True: if user_message['meta'].get('timer_pending') is True:
await sio.emit( await sio.emit(