refac
This commit is contained in:
1 parent
92fbc77877
commit
771a024b40
3 files changed
+69
-41
No files matched your search
@@ -11,6 +11,7 @@ The scheduler_worker_loop handles all time-based background work:
|
|||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
SCHEDULER_POLL_INTERVAL – seconds between polls (default: 10)
|
SCHEDULER_POLL_INTERVAL – seconds between polls (default: 10)
|
||||||
|
TIMER_POLL_INTERVAL – seconds between timer polls (default: 1)
|
||||||
CALENDAR_ALERT_LOOKAHEAD_MINUTES – default alert window (default: 5)
|
CALENDAR_ALERT_LOOKAHEAD_MINUTES – default alert window (default: 5)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -43,6 +44,7 @@ from starlette.datastructures import Headers
|
|||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
SCHEDULER_POLL_INTERVAL = int(os.getenv('SCHEDULER_POLL_INTERVAL', os.getenv('AUTOMATION_POLL_INTERVAL', '10')))
|
SCHEDULER_POLL_INTERVAL = int(os.getenv('SCHEDULER_POLL_INTERVAL', os.getenv('AUTOMATION_POLL_INTERVAL', '10')))
|
||||||
|
TIMER_POLL_INTERVAL = int(os.getenv('TIMER_POLL_INTERVAL', '1'))
|
||||||
CALENDAR_ALERT_LOOKAHEAD_MINUTES = int(os.getenv('CALENDAR_ALERT_LOOKAHEAD_MINUTES', '10'))
|
CALENDAR_ALERT_LOOKAHEAD_MINUTES = int(os.getenv('CALENDAR_ALERT_LOOKAHEAD_MINUTES', '10'))
|
||||||
|
|
||||||
|
|
||||||
@@ -190,10 +192,15 @@ async def scheduler_worker_loop(app) -> None:
|
|||||||
Runs on every instance. Poll interval is configurable via
|
Runs on every instance. Poll interval is configurable via
|
||||||
SCHEDULER_POLL_INTERVAL env var (default: 10 seconds).
|
SCHEDULER_POLL_INTERVAL env var (default: 10 seconds).
|
||||||
"""
|
"""
|
||||||
log.info(f'Scheduler worker started (poll interval: {SCHEDULER_POLL_INTERVAL}s)')
|
log.info(
|
||||||
|
f'Scheduler worker started (timer poll interval: {TIMER_POLL_INTERVAL}s, '
|
||||||
|
f'scheduler poll interval: {SCHEDULER_POLL_INTERVAL}s)'
|
||||||
|
)
|
||||||
|
next_scheduler_poll = 0.0
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
|
now = time.monotonic()
|
||||||
# ── Timers ──
|
# ── Timers ──
|
||||||
try:
|
try:
|
||||||
from open_webui.utils.timers import claim_due_timers, execute_due_timer
|
from open_webui.utils.timers import claim_due_timers, execute_due_timer
|
||||||
@@ -203,6 +210,12 @@ async def scheduler_worker_loop(app) -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
log.exception('Scheduler: timer error')
|
log.exception('Scheduler: timer error')
|
||||||
|
|
||||||
|
if now < next_scheduler_poll:
|
||||||
|
await asyncio.sleep(max(1, TIMER_POLL_INTERVAL))
|
||||||
|
continue
|
||||||
|
# Jitter to spread automation/calendar load across instances; timers keep a tight poll.
|
||||||
|
next_scheduler_poll = now + SCHEDULER_POLL_INTERVAL + random.uniform(0, 2)
|
||||||
|
|
||||||
# ── Automations ──
|
# ── Automations ──
|
||||||
if await Config.get('automations.enable'):
|
if await Config.get('automations.enable'):
|
||||||
try:
|
try:
|
||||||
@@ -225,8 +238,7 @@ async def scheduler_worker_loop(app) -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
log.exception('Scheduler worker error')
|
log.exception('Scheduler worker error')
|
||||||
|
|
||||||
# Jitter to spread load across instances
|
await asyncio.sleep(max(1, TIMER_POLL_INTERVAL))
|
||||||
await asyncio.sleep(SCHEDULER_POLL_INTERVAL + random.uniform(0, 2))
|
|
||||||
|
|
||||||
|
|
||||||
##########################
|
##########################
|
||||||
|
|||||||
@@ -9,13 +9,15 @@ from uuid import uuid4
|
|||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from fastapi.security import HTTPAuthorizationCredentials
|
from fastapi.security import HTTPAuthorizationCredentials
|
||||||
|
from open_webui.internal.db import get_async_db
|
||||||
from open_webui.models.chat_messages import ChatMessages
|
from open_webui.models.chat_messages import ChatMessages
|
||||||
from open_webui.models.chats import ChatForm, Chats
|
from open_webui.models.chats import Chat, ChatForm, Chats
|
||||||
from open_webui.models.config import Config
|
from open_webui.models.config import Config
|
||||||
from open_webui.models.users import UserModel, Users
|
from open_webui.models.users import UserModel, Users
|
||||||
from open_webui.tasks import create_task, has_active_tasks
|
from open_webui.tasks import create_task, has_active_tasks
|
||||||
from open_webui.utils.auth import create_token
|
from open_webui.utils.auth import create_token
|
||||||
from open_webui.utils.misc import get_message_list
|
from open_webui.utils.misc import get_message_list
|
||||||
|
from sqlalchemy import select
|
||||||
from starlette.datastructures import Headers
|
from starlette.datastructures import Headers
|
||||||
|
|
||||||
DEFAULT_SUBAGENT_SYSTEM_PROMPT = """You are a sub-agent working on a specific task assigned by the lead agent.
|
DEFAULT_SUBAGENT_SYSTEM_PROMPT = """You are a sub-agent working on a specific task assigned by the lead agent.
|
||||||
@@ -81,12 +83,20 @@ async def process_pending_internal_messages(
|
|||||||
if await has_active_tasks(source_request.app.state.redis, parent_chat_id):
|
if await has_active_tasks(source_request.app.state.redis, parent_chat_id):
|
||||||
return
|
return
|
||||||
|
|
||||||
chat = await Chats.get_chat_by_id_and_user_id(parent_chat_id, user_id)
|
|
||||||
user = await Users.get_user_by_id(user_id)
|
user = await Users.get_user_by_id(user_id)
|
||||||
if not chat or not user:
|
if not user:
|
||||||
return
|
return
|
||||||
|
|
||||||
history = copy.deepcopy(chat.chat.get('history') or {})
|
async with get_async_db() as db:
|
||||||
|
stmt = select(Chat).where(Chat.id == parent_chat_id, Chat.user_id == user_id)
|
||||||
|
if db.bind.dialect.name == 'postgresql':
|
||||||
|
stmt = stmt.with_for_update()
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
chat = result.scalar_one_or_none()
|
||||||
|
if not chat:
|
||||||
|
return
|
||||||
|
|
||||||
|
history = copy.deepcopy((chat.chat or {}).get('history') or {})
|
||||||
messages = history.get('messages') or {}
|
messages = history.get('messages') or {}
|
||||||
pending = [
|
pending = [
|
||||||
message
|
message
|
||||||
@@ -113,14 +123,7 @@ async def process_pending_internal_messages(
|
|||||||
run = {**run, **(((timer.meta or {}).get('run') if timer else None) or {})}
|
run = {**run, **(((timer.meta or {}).get('run') if timer else None) or {})}
|
||||||
model_id = first.get('model') or run['model_id']
|
model_id = first.get('model') or run['model_id']
|
||||||
if kind == 'timer':
|
if kind == 'timer':
|
||||||
batch = [
|
batch = [first]
|
||||||
message
|
|
||||||
for message in pending
|
|
||||||
if message.get('parentId') == parent_id
|
|
||||||
and (message.get('model') or model_id) == model_id
|
|
||||||
and (message.get('meta') or {}).get('internal') is True
|
|
||||||
and (message.get('meta') or {}).get('type') == 'timer'
|
|
||||||
]
|
|
||||||
else:
|
else:
|
||||||
batch = [
|
batch = [
|
||||||
message
|
message
|
||||||
@@ -131,7 +134,9 @@ async def process_pending_internal_messages(
|
|||||||
]
|
]
|
||||||
combined_content = '\n\n'.join(message.get('content', '') for message in batch if message.get('content'))
|
combined_content = '\n\n'.join(message.get('content', '') for message in batch if message.get('content'))
|
||||||
if kind == 'timer':
|
if kind == 'timer':
|
||||||
timer_ids = [message['meta']['timer_id'] for message in batch if (message.get('meta') or {}).get('timer_id')]
|
timer_ids = [
|
||||||
|
message['meta']['timer_id'] for message in batch if (message.get('meta') or {}).get('timer_id')
|
||||||
|
]
|
||||||
combined_meta = {'internal': True, 'type': 'timer'}
|
combined_meta = {'internal': True, 'type': 'timer'}
|
||||||
if len(timer_ids) == 1:
|
if len(timer_ids) == 1:
|
||||||
combined_meta['timer_id'] = timer_ids[0]
|
combined_meta['timer_id'] = timer_ids[0]
|
||||||
@@ -161,20 +166,17 @@ async def process_pending_internal_messages(
|
|||||||
pending_flag = 'timer_pending' if kind == 'timer' else 'async_subagent_pending'
|
pending_flag = 'timer_pending' if kind == 'timer' else 'async_subagent_pending'
|
||||||
reuse_message = len(batch) == 1 and not (first.get('meta') or {}).get(pending_flag)
|
reuse_message = len(batch) == 1 and not (first.get('meta') or {}).get(pending_flag)
|
||||||
user_message_id = first['id'] if reuse_message else str(uuid4())
|
user_message_id = first['id'] if reuse_message else str(uuid4())
|
||||||
|
removed_ids = set()
|
||||||
if not reuse_message:
|
if not reuse_message:
|
||||||
removed_ids = {message['id'] for message in batch}
|
removed_ids = {message['id'] for message in batch}
|
||||||
for message_id in removed_ids:
|
for message_id in removed_ids:
|
||||||
messages.pop(message_id, None)
|
messages.pop(message_id, None)
|
||||||
if parent_id and parent_id in messages:
|
if parent_id and parent_id in messages:
|
||||||
messages[parent_id]['childrenIds'] = [
|
messages[parent_id]['childrenIds'] = [
|
||||||
child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id not in removed_ids
|
child_id
|
||||||
|
for child_id in messages[parent_id].get('childrenIds', [])
|
||||||
|
if child_id not in removed_ids
|
||||||
]
|
]
|
||||||
history['messages'] = messages
|
|
||||||
history['currentId'] = parent_id
|
|
||||||
updated_chat = copy.deepcopy(chat.chat)
|
|
||||||
updated_chat['history'] = history
|
|
||||||
await Chats.update_chat_by_id(parent_chat_id, updated_chat)
|
|
||||||
await ChatMessages.delete_message_ids_by_chat_id(parent_chat_id, removed_ids)
|
|
||||||
|
|
||||||
assistant_message_id = str(uuid4())
|
assistant_message_id = str(uuid4())
|
||||||
message_list = get_message_list(messages, parent_id)
|
message_list = get_message_list(messages, parent_id)
|
||||||
@@ -205,21 +207,19 @@ async def process_pending_internal_messages(
|
|||||||
child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id != user_message_id
|
child_id for child_id in messages[parent_id].get('childrenIds', []) if child_id != user_message_id
|
||||||
]
|
]
|
||||||
parent_children.append(user_message_id)
|
parent_children.append(user_message_id)
|
||||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
messages[parent_id]['childrenIds'] = parent_children
|
||||||
parent_chat_id,
|
messages[user_message_id] = {**messages.get(user_message_id, {}), **user_message}
|
||||||
parent_id,
|
messages[assistant_message_id] = assistant_message
|
||||||
{'childrenIds': parent_children},
|
history['messages'] = messages
|
||||||
)
|
history['currentId'] = assistant_message_id
|
||||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
chat.chat = {**(chat.chat or {}), 'history': history}
|
||||||
parent_chat_id,
|
chat.updated_at = int(time.time())
|
||||||
user_message_id,
|
await db.commit()
|
||||||
user_message,
|
|
||||||
)
|
if removed_ids:
|
||||||
await Chats.upsert_message_to_chat_by_id_and_message_id(
|
await ChatMessages.delete_message_ids_by_chat_id(parent_chat_id, removed_ids)
|
||||||
parent_chat_id,
|
await ChatMessages.upsert_message(user_message_id, parent_chat_id, user_id, user_message)
|
||||||
assistant_message_id,
|
await ChatMessages.upsert_message(assistant_message_id, parent_chat_id, user_id, assistant_message)
|
||||||
assistant_message,
|
|
||||||
)
|
|
||||||
|
|
||||||
from open_webui.socket.main import sio
|
from open_webui.socket.main import sio
|
||||||
|
|
||||||
|
|||||||
@@ -275,10 +275,16 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
|
|||||||
user_message_id = str(uuid4())
|
user_message_id = str(uuid4())
|
||||||
parent_lock = _parent_locks.setdefault(parent_chat_id, asyncio.Lock())
|
parent_lock = _parent_locks.setdefault(parent_chat_id, asyncio.Lock())
|
||||||
async with parent_lock:
|
async with parent_lock:
|
||||||
parent = await Chats.get_chat_by_id_and_user_id(parent_chat_id, timer.user_id)
|
async with get_async_db() as db:
|
||||||
|
stmt = select(Chat).where(Chat.id == parent_chat_id, Chat.user_id == timer.user_id)
|
||||||
|
if db.bind.dialect.name == 'postgresql':
|
||||||
|
stmt = stmt.with_for_update()
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
parent = result.scalar_one_or_none()
|
||||||
if not parent:
|
if not parent:
|
||||||
await _set_timer_status(timer_id, 'error', timer_error='parent chat no longer exists')
|
await _set_timer_status(timer_id, 'error', timer_error='parent chat no longer exists')
|
||||||
return
|
return
|
||||||
|
|
||||||
parent_chat = copy.deepcopy(parent.chat or {})
|
parent_chat = copy.deepcopy(parent.chat or {})
|
||||||
history = parent_chat.setdefault('history', {})
|
history = parent_chat.setdefault('history', {})
|
||||||
messages = history.setdefault('messages', {})
|
messages = history.setdefault('messages', {})
|
||||||
@@ -311,9 +317,19 @@ async def execute_due_timer(app, timer_id: str, claim_id: str | None = None) ->
|
|||||||
children = messages[parent_id].setdefault('childrenIds', [])
|
children = messages[parent_id].setdefault('childrenIds', [])
|
||||||
if user_message_id not in children:
|
if user_message_id not in children:
|
||||||
children.append(user_message_id)
|
children.append(user_message_id)
|
||||||
await Chats.update_chat_by_id(parent_chat_id, parent_chat)
|
|
||||||
|
parent.chat = parent_chat
|
||||||
|
parent.updated_at = int(time.time())
|
||||||
|
timer_row = await db.get(Chat, timer_id)
|
||||||
|
if timer_row:
|
||||||
|
timer_row.meta = {
|
||||||
|
**(timer_row.meta or {}),
|
||||||
|
'timer_status': 'dispatched',
|
||||||
|
'timer_dispatched_at': int(time.time_ns()),
|
||||||
|
}
|
||||||
|
timer_row.updated_at = int(time.time())
|
||||||
|
await db.commit()
|
||||||
await ChatMessages.upsert_message(user_message_id, parent_chat_id, timer.user_id, user_message)
|
await ChatMessages.upsert_message(user_message_id, parent_chat_id, timer.user_id, user_message)
|
||||||
await _set_timer_status(timer_id, 'dispatched', timer_dispatched_at=int(time.time_ns()))
|
|
||||||
|
|
||||||
if user_message['meta'].get('timer_pending') is True:
|
if user_message['meta'].get('timer_pending') is True:
|
||||||
await sio.emit(
|
await sio.emit(
|
||||||
|
|||||||
Reference in new issue
Block a user