refac: clean up Redis sentinel utilities and import grouping

This commit is contained in:
Timothy Jaeryang Baek committed 2026-05-21 11:47:25 +04:00
1 parent 1527eb6e01
commit 154679200f
5 files changed
+321 -289

No files matched your search

+7
View File
@@ -193,6 +193,13 @@ class AppConfig:
super().__setattr__('_entries', {}) super().__setattr__('_entries', {})
super().__setattr__('_key_prefix', redis_key_prefix) super().__setattr__('_key_prefix', redis_key_prefix)
# If sentinels weren't explicitly provided, read from env.
if redis_sentinels is None:
from open_webui.env import REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT
from open_webui.utils.redis import get_sentinels_from_env
redis_sentinels = get_sentinels_from_env(REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT)
rc: Union[redis.Redis, redis.cluster.RedisCluster, None] = None rc: Union[redis.Redis, redis.cluster.RedisCluster, None] = None
if redis_url: if redis_url:
rc = get_redis_connection(redis_url, redis_sentinels or [], redis_cluster, decode_responses=True) rc = get_redis_connection(redis_url, redis_sentinels or [], redis_cluster, decode_responses=True)
+3 -10
View File
@@ -447,10 +447,9 @@ from open_webui.env import (
LICENSE_KEY, LICENSE_KEY,
LOG_FORMAT, LOG_FORMAT,
MAX_BODY_LOG_SIZE, MAX_BODY_LOG_SIZE,
# Redis
REDIS_CLUSTER, REDIS_CLUSTER,
REDIS_KEY_PREFIX, REDIS_KEY_PREFIX,
REDIS_SENTINEL_HOSTS,
REDIS_SENTINEL_PORT,
REDIS_URL, REDIS_URL,
RESET_CONFIG_ON_START, RESET_CONFIG_ON_START,
SAFE_MODE, SAFE_MODE,
@@ -579,7 +578,7 @@ from open_webui.utils.oauth import (
resolve_oauth_client_info, resolve_oauth_client_info,
) )
from open_webui.utils.plugin import install_tool_and_function_dependencies from open_webui.utils.plugin import install_tool_and_function_dependencies
from open_webui.utils.redis import get_redis_connection, get_sentinels_from_env from open_webui.utils.redis import get_redis_client, get_redis_connection
from open_webui.utils.security_headers import SecurityHeadersMiddleware from open_webui.utils.security_headers import SecurityHeadersMiddleware
from open_webui.utils.session_pool import get_session from open_webui.utils.session_pool import get_session
from open_webui.utils.tools import set_terminal_servers, set_tool_servers from open_webui.utils.tools import set_terminal_servers, set_tool_servers
@@ -652,12 +651,7 @@ async def lifespan(app: FastAPI):
log.info('Installing external dependencies of functions and tools...') log.info('Installing external dependencies of functions and tools...')
await install_tool_and_function_dependencies() await install_tool_and_function_dependencies()
app.state.redis = get_redis_connection( app.state.redis = get_redis_client(async_mode=True)
redis_url=REDIS_URL,
redis_sentinels=get_sentinels_from_env(REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT),
redis_cluster=REDIS_CLUSTER,
async_mode=True,
)
if app.state.redis is not None: if app.state.redis is not None:
app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app)) app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app))
@@ -764,7 +758,6 @@ app.state.oauth_client_manager = oauth_client_manager
app.state.instance_id = None app.state.instance_id = None
app.state.config = AppConfig( app.state.config = AppConfig(
redis_url=REDIS_URL, redis_url=REDIS_URL,
redis_sentinels=get_sentinels_from_env(REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT),
redis_cluster=REDIS_CLUSTER, redis_cluster=REDIS_CLUSTER,
redis_key_prefix=REDIS_KEY_PREFIX, redis_key_prefix=REDIS_KEY_PREFIX,
) )
+15 -16
View File
@@ -40,8 +40,8 @@ from open_webui.tasks import create_task, stop_item_tasks
from open_webui.utils.access_control import has_permission from open_webui.utils.access_control import has_permission
from open_webui.utils.auth import decode_token from open_webui.utils.auth import decode_token
from open_webui.utils.redis import ( from open_webui.utils.redis import (
build_sentinel_url,
get_redis_connection, get_redis_connection,
get_sentinel_url_from_env,
get_sentinels_from_env, get_sentinels_from_env,
) )
from redis import asyncio as aioredis from redis import asyncio as aioredis
@@ -58,20 +58,20 @@ REDIS = None
SOCKETIO_CORS_ORIGINS = '*' if CORS_ALLOW_ORIGIN == ['*'] else CORS_ALLOW_ORIGIN SOCKETIO_CORS_ORIGINS = '*' if CORS_ALLOW_ORIGIN == ['*'] else CORS_ALLOW_ORIGIN
if WEBSOCKET_MANAGER == 'redis': if WEBSOCKET_MANAGER == 'redis':
if WEBSOCKET_SENTINEL_HOSTS: sentinel_hosts = WEBSOCKET_SENTINEL_HOSTS or ''
mgr = socketio.AsyncRedisManager( ws_redis_url = (
get_sentinel_url_from_env(WEBSOCKET_REDIS_URL, WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT), build_sentinel_url(WEBSOCKET_REDIS_URL, sentinel_hosts, WEBSOCKET_SENTINEL_PORT)
redis_options=WEBSOCKET_REDIS_OPTIONS, if sentinel_hosts
) else WEBSOCKET_REDIS_URL
else: )
mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL, redis_options=WEBSOCKET_REDIS_OPTIONS) redis_manager = socketio.AsyncRedisManager(ws_redis_url, redis_options=WEBSOCKET_REDIS_OPTIONS)
sio = socketio.AsyncServer( sio = socketio.AsyncServer(
cors_allowed_origins=SOCKETIO_CORS_ORIGINS, cors_allowed_origins=SOCKETIO_CORS_ORIGINS,
async_mode='asgi', async_mode='asgi',
transports=(['websocket'] if ENABLE_WEBSOCKET_SUPPORT else ['polling']), transports=(['websocket'] if ENABLE_WEBSOCKET_SUPPORT else ['polling']),
allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, allow_upgrades=ENABLE_WEBSOCKET_SUPPORT,
always_connect=True, always_connect=True,
client_manager=mgr, client_manager=redis_manager,
logger=WEBSOCKET_SERVER_LOGGING, logger=WEBSOCKET_SERVER_LOGGING,
ping_interval=WEBSOCKET_SERVER_PING_INTERVAL, ping_interval=WEBSOCKET_SERVER_PING_INTERVAL,
ping_timeout=WEBSOCKET_SERVER_PING_TIMEOUT, ping_timeout=WEBSOCKET_SERVER_PING_TIMEOUT,
@@ -99,32 +99,31 @@ SESSION_POOL_TIMEOUT = 120 # seconds without heartbeat before session is reaped
if WEBSOCKET_MANAGER == 'redis': if WEBSOCKET_MANAGER == 'redis':
log.debug('Using Redis to manage websockets.') log.debug('Using Redis to manage websockets.')
ws_sentinels = get_sentinels_from_env(WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT)
REDIS = get_redis_connection( REDIS = get_redis_connection(
redis_url=WEBSOCKET_REDIS_URL, redis_url=WEBSOCKET_REDIS_URL,
redis_sentinels=get_sentinels_from_env(WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT), redis_sentinels=ws_sentinels,
redis_cluster=WEBSOCKET_REDIS_CLUSTER, redis_cluster=WEBSOCKET_REDIS_CLUSTER,
async_mode=True, async_mode=True,
) )
redis_sentinels = get_sentinels_from_env(WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT)
MODELS = RedisDict( MODELS = RedisDict(
f'{REDIS_KEY_PREFIX}:models', f'{REDIS_KEY_PREFIX}:models',
redis_url=WEBSOCKET_REDIS_URL, redis_url=WEBSOCKET_REDIS_URL,
redis_sentinels=redis_sentinels, redis_sentinels=ws_sentinels,
redis_cluster=WEBSOCKET_REDIS_CLUSTER, redis_cluster=WEBSOCKET_REDIS_CLUSTER,
) )
SESSION_POOL = RedisDict( SESSION_POOL = RedisDict(
f'{REDIS_KEY_PREFIX}:session_pool', f'{REDIS_KEY_PREFIX}:session_pool',
redis_url=WEBSOCKET_REDIS_URL, redis_url=WEBSOCKET_REDIS_URL,
redis_sentinels=redis_sentinels, redis_sentinels=ws_sentinels,
redis_cluster=WEBSOCKET_REDIS_CLUSTER, redis_cluster=WEBSOCKET_REDIS_CLUSTER,
) )
USAGE_POOL = RedisDict( USAGE_POOL = RedisDict(
f'{REDIS_KEY_PREFIX}:usage_pool', f'{REDIS_KEY_PREFIX}:usage_pool',
redis_url=WEBSOCKET_REDIS_URL, redis_url=WEBSOCKET_REDIS_URL,
redis_sentinels=redis_sentinels, redis_sentinels=ws_sentinels,
redis_cluster=WEBSOCKET_REDIS_CLUSTER, redis_cluster=WEBSOCKET_REDIS_CLUSTER,
) )
@@ -132,7 +131,7 @@ if WEBSOCKET_MANAGER == 'redis':
redis_url=WEBSOCKET_REDIS_URL, redis_url=WEBSOCKET_REDIS_URL,
lock_name=f'{REDIS_KEY_PREFIX}:usage_cleanup_lock', lock_name=f'{REDIS_KEY_PREFIX}:usage_cleanup_lock',
timeout_secs=WEBSOCKET_REDIS_LOCK_TIMEOUT, timeout_secs=WEBSOCKET_REDIS_LOCK_TIMEOUT,
redis_sentinels=redis_sentinels, redis_sentinels=ws_sentinels,
redis_cluster=WEBSOCKET_REDIS_CLUSTER, redis_cluster=WEBSOCKET_REDIS_CLUSTER,
) )
aquire_func = clean_up_lock.aquire_lock aquire_func = clean_up_lock.aquire_lock
+11 -6
View File
@@ -1,13 +1,18 @@
"""Redis-backed distributed data structures for WebSocket state management."""
from __future__ import annotations
import json import json
import uuid import uuid
from typing import List, Optional, Tuple
import pycrdt as Y import pycrdt as Y
from open_webui.env import REDIS_KEY_PREFIX
from open_webui.utils.redis import get_redis_connection from open_webui.utils.redis import get_redis_connection
from open_webui.env import REDIS_KEY_PREFIX
YDOC_KEY_PREFIX = f"{REDIS_KEY_PREFIX}:ydoc:documents"
class RedisLock: class RedisLock:
"""Distributed lock backed by a Redis SET with NX/EX semantics."""
def __init__( def __init__(
self, self,
redis_url, redis_url,
@@ -128,7 +133,7 @@ class YdocManager:
def __init__( def __init__(
self, self,
redis=None, redis=None,
redis_key_prefix: str = f'{REDIS_KEY_PREFIX}:ydoc:documents', redis_key_prefix: str = YDOC_KEY_PREFIX,
): ):
self._updates = {} self._updates = {}
self._users = {} self._users = {}
@@ -177,7 +182,7 @@ class YdocManager:
ydoc.apply_update(bytes(update)) ydoc.apply_update(bytes(update))
self._updates[document_id] = [ydoc.get_update()] + updates[mid:] self._updates[document_id] = [ydoc.get_update()] + updates[mid:]
async def get_updates(self, document_id: str) -> List[bytes]: async def get_updates(self, document_id: str) -> list[bytes]:
document_id = document_id.replace(':', '_') document_id = document_id.replace(':', '_')
if self._redis: if self._redis:
@@ -196,7 +201,7 @@ class YdocManager:
else: else:
return document_id in self._updates return document_id in self._updates
async def get_users(self, document_id: str) -> List[str]: async def get_users(self, document_id: str) -> list[str]:
document_id = document_id.replace(':', '_') document_id = document_id.replace(':', '_')
if self._redis: if self._redis:
+285 -257
View File
@@ -1,10 +1,20 @@
"""Redis connection utilities.
Provides connection factory functions for standalone, Sentinel, and Cluster
Redis deployments, with optional async support and automatic connection caching.
"""
from __future__ import annotations
import asyncio import asyncio
import inspect import inspect
import logging import logging
import time import time
from urllib.parse import urlparse from typing import Any
from urllib.parse import ParseResult, urlparse
import redis as _redis_sync
import redis
from open_webui.env import ( from open_webui.env import (
REDIS_CLUSTER, REDIS_CLUSTER,
REDIS_HEALTH_CHECK_INTERVAL, REDIS_HEALTH_CHECK_INTERVAL,
@@ -19,286 +29,304 @@ from open_webui.env import (
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
MAX_RETRY_COUNT = REDIS_SENTINEL_MAX_RETRY_COUNT _ACCEPTED_SCHEMES = frozenset({"redis", "rediss"})
_SENTINEL_RETRYABLE = (
_redis_sync.exceptions.ConnectionError,
_redis_sync.exceptions.ReadOnlyError,
)
_FACTORY_METHODS = frozenset({"pipeline", "pubsub", "monitor", "client", "transaction"})
_CONNECTION_POOL: dict[tuple, Any] = {}
# Let not our connections be timed out but deliver them from def parse_redis_url(url: str) -> dict[str, Any]:
# partition. For the cache and the socket and the uptime """Break a ``redis://`` URL into its parts: service, port, db, username, password."""
# belong to the one who first opened them, now and always. parts: ParseResult = urlparse(url)
_CONNECTION_CACHE = {} if parts.scheme not in _ACCEPTED_SCHEMES:
raise ValueError(
f"Invalid Redis URL scheme '{parts.scheme}'; expected 'redis' or 'rediss'."
class SentinelRedisProxy: )
def __init__(self, sentinel, service, *, async_mode: bool = True, **kw):
self._sentinel = sentinel
self._service = service
self._kw = kw
self._async_mode = async_mode
def _master(self):
return self._sentinel.master_for(self._service, **self._kw)
def __getattr__(self, item):
master = self._master()
orig_attr = getattr(master, item)
if not callable(orig_attr):
return orig_attr
FACTORY_METHODS = {'pipeline', 'pubsub', 'monitor', 'client', 'transaction'}
if item in FACTORY_METHODS:
return orig_attr
if self._async_mode:
if inspect.isasyncgenfunction(orig_attr):
def _wrapped_iter(*args, **kwargs):
async def _iter():
for i in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(self._master(), item)
async for value in method(*args, **kwargs):
yield value
return
except (
redis.exceptions.ConnectionError,
redis.exceptions.ReadOnlyError,
) as e:
if i < REDIS_SENTINEL_MAX_RETRY_COUNT - 1:
log.debug(
'Redis sentinel fail-over (%s). Retry %s/%s',
type(e).__name__,
i + 1,
REDIS_SENTINEL_MAX_RETRY_COUNT,
)
if REDIS_RECONNECT_DELAY:
time.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
log.error(
'Redis operation failed after %s retries: %s',
REDIS_SENTINEL_MAX_RETRY_COUNT,
e,
)
raise e from e
return _iter()
return _wrapped_iter
async def _wrapped(*args, **kwargs):
for i in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(self._master(), item)
result = method(*args, **kwargs)
if inspect.iscoroutine(result):
return await result
return result
except (
redis.exceptions.ConnectionError,
redis.exceptions.ReadOnlyError,
) as e:
if i < REDIS_SENTINEL_MAX_RETRY_COUNT - 1:
log.debug(
'Redis sentinel fail-over (%s). Retry %s/%s',
type(e).__name__,
i + 1,
REDIS_SENTINEL_MAX_RETRY_COUNT,
)
if REDIS_RECONNECT_DELAY:
await asyncio.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
log.error(
'Redis operation failed after %s retries: %s',
REDIS_SENTINEL_MAX_RETRY_COUNT,
e,
)
raise e from e
return _wrapped
else:
def _wrapped(*args, **kwargs):
for i in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(self._master(), item)
return method(*args, **kwargs)
except (
redis.exceptions.ConnectionError,
redis.exceptions.ReadOnlyError,
) as e:
if i < REDIS_SENTINEL_MAX_RETRY_COUNT - 1:
log.debug(
'Redis sentinel fail-over (%s). Retry %s/%s',
type(e).__name__,
i + 1,
REDIS_SENTINEL_MAX_RETRY_COUNT,
)
if REDIS_RECONNECT_DELAY:
time.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
log.error(
'Redis operation failed after %s retries: %s',
REDIS_SENTINEL_MAX_RETRY_COUNT,
e,
)
raise e from e
return _wrapped
def parse_redis_service_url(redis_url):
parsed_url = urlparse(redis_url)
if parsed_url.scheme != 'redis' and parsed_url.scheme != 'rediss':
raise ValueError("Invalid Redis URL scheme. Must be 'redis' or 'rediss'.")
return { return {
'username': parsed_url.username or None, "service": parts.hostname or "mymaster",
'password': parsed_url.password or None, "port": parts.port or 6379,
'service': parsed_url.hostname or 'mymaster', "db": int(parts.path.lstrip("/") or "0"),
'port': parsed_url.port or 6379, "username": parts.username or None,
'db': int(parsed_url.path.lstrip('/') or 0), "password": parts.password or None,
} }
parse_redis_service_url = parse_redis_url
def get_redis_client(async_mode=False):
def get_sentinels_from_env(
hosts_csv: str | None,
port: str | int | None,
) -> list[tuple[str, int]]:
"""Turn a comma-separated host string into ``[(host, port), …]``."""
if not hosts_csv:
return []
resolved_port = int(port) if port else 26379
return [
(host.strip(), resolved_port)
for host in hosts_csv.split(",")
if host.strip()
]
def build_sentinel_url(
base_url: str,
hosts_csv: str,
port: str | int,
) -> str:
"""Construct a ``redis+sentinel://`` connection string.
``base_url`` supplies credentials, db index, and master service name.
``hosts_csv`` is a comma-separated list of sentinel hostnames.
"""
cfg = parse_redis_url(base_url)
auth = ""
if cfg["username"] or cfg["password"]:
auth = f"{cfg['username'] or ''}:{cfg['password'] or ''}@"
nodes = ",".join(
f"{host.strip()}:{port}" for host in hosts_csv.split(",") if host.strip()
)
return f"redis+sentinel://{auth}{nodes}/{cfg['db']}/{cfg['service']}"
def get_redis_client(async_mode: bool = False) -> Any | None:
"""Create a Redis connection using settings from environment variables.
Returns ``None`` when Redis is not configured or the connection fails.
"""
sentinel_list = get_sentinels_from_env(REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT)
if not REDIS_URL and not sentinel_list:
return None
try: try:
return get_redis_connection( return get_redis_connection(
redis_url=REDIS_URL, REDIS_URL,
redis_sentinels=get_sentinels_from_env(REDIS_SENTINEL_HOSTS, REDIS_SENTINEL_PORT), redis_sentinels=sentinel_list,
redis_cluster=REDIS_CLUSTER, redis_cluster=REDIS_CLUSTER,
async_mode=async_mode, async_mode=async_mode,
) )
except Exception as e: except Exception:
log.debug(f'Failed to get Redis client: {e}') log.debug("Could not establish Redis connection", exc_info=True)
return None return None
# ---------------------------------------------------------------------------
# Sentinel proxy with automatic failover retry
# ---------------------------------------------------------------------------
class SentinelRedisProxy:
"""Transparent proxy that re-resolves the Sentinel master on connection errors.
Every call (sync or async) is wrapped with retry logic so that transient
Sentinel failovers are handled without caller intervention.
"""
def __init__(
self,
sentinel: Any,
service_name: str,
*,
async_mode: bool = True,
) -> None:
self._sentinel = sentinel
self._service_name = service_name
self._async_mode = async_mode
def __getattr__(self, name: str) -> Any:
"""Proxy attribute access with automatic Sentinel failover retry."""
current_master = self._sentinel.master_for(self._service_name)
original = getattr(current_master, name)
# Non-callable or factory attributes pass through without wrapping.
if not callable(original) or name in _FACTORY_METHODS:
return original
# Select the retry wrapper matching the execution mode.
if not self._async_mode:
return self._wrap_sync(name)
return self._wrap_async(name, original)
def _resolve_master(self) -> Any:
"""Ask Sentinel for the current master connection."""
return self._sentinel.master_for(self._service_name)
def _should_retry(self, attempt: int) -> bool:
return attempt < REDIS_SENTINEL_MAX_RETRY_COUNT - 1
def _log_retry(self, exc: Exception, attempt: int) -> None:
log.debug(
"Sentinel failover (%s) — retry %d/%d",
type(exc).__name__,
attempt + 1,
REDIS_SENTINEL_MAX_RETRY_COUNT,
)
def _log_exhausted(self, exc: Exception) -> None:
log.error(
"Redis operation failed after %d retries: %s",
REDIS_SENTINEL_MAX_RETRY_COUNT,
exc,
)
# -- async wrappers -----------------------------------------------------
def _wrap_async(self, name: str, attr: Any) -> Any:
if inspect.isasyncgenfunction(attr):
return self._wrap_async_gen(name)
return self._wrap_async_call(name)
def _wrap_async_gen(self, name: str) -> Any:
proxy = self
def wrapper(*args: Any, **kwargs: Any) -> Any:
async def _inner():
for attempt in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(proxy._resolve_master(), name)
async for value in method(*args, **kwargs):
yield value
return
except _SENTINEL_RETRYABLE as exc:
if proxy._should_retry(attempt):
proxy._log_retry(exc, attempt)
if REDIS_RECONNECT_DELAY:
time.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
proxy._log_exhausted(exc)
raise
return _inner()
return wrapper
def _wrap_async_call(self, name: str) -> Any:
proxy = self
async def wrapper(*args: Any, **kwargs: Any) -> Any:
for attempt in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(proxy._resolve_master(), name)
result = method(*args, **kwargs)
if inspect.iscoroutine(result):
return await result
return result
except _SENTINEL_RETRYABLE as exc:
if proxy._should_retry(attempt):
proxy._log_retry(exc, attempt)
if REDIS_RECONNECT_DELAY:
await asyncio.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
proxy._log_exhausted(exc)
raise
return wrapper
# -- sync wrapper -------------------------------------------------------
def _wrap_sync(self, name: str) -> Any:
proxy = self
def wrapper(*args: Any, **kwargs: Any) -> Any:
for attempt in range(REDIS_SENTINEL_MAX_RETRY_COUNT):
try:
method = getattr(proxy._resolve_master(), name)
return method(*args, **kwargs)
except _SENTINEL_RETRYABLE as exc:
if proxy._should_retry(attempt):
proxy._log_retry(exc, attempt)
if REDIS_RECONNECT_DELAY:
time.sleep(REDIS_RECONNECT_DELAY / 1000)
continue
proxy._log_exhausted(exc)
raise
return wrapper
# ---------------------------------------------------------------------------
# Connection factory
# ---------------------------------------------------------------------------
def _socket_options() -> dict[str, Any]:
"""Collect optional socket-level kwargs once instead of repeating them."""
opts: dict[str, Any] = {}
if REDIS_SOCKET_CONNECT_TIMEOUT is not None:
opts["socket_connect_timeout"] = REDIS_SOCKET_CONNECT_TIMEOUT
if REDIS_SOCKET_KEEPALIVE:
opts["socket_keepalive"] = True
if REDIS_HEALTH_CHECK_INTERVAL:
opts["health_check_interval"] = REDIS_HEALTH_CHECK_INTERVAL
return opts
def _build_sentinel(
redis_module: Any,
url: str,
sentinels: list[tuple[str, int]],
decode_responses: bool,
async_mode: bool,
) -> SentinelRedisProxy:
"""Create a SentinelRedisProxy from a redis URL and sentinel list."""
cfg = parse_redis_url(url)
sentinel = redis_module.sentinel.Sentinel(
sentinels,
port=cfg["port"],
db=cfg["db"],
username=cfg["username"],
password=cfg["password"],
decode_responses=decode_responses,
socket_connect_timeout=REDIS_SOCKET_CONNECT_TIMEOUT,
**{k: v for k, v in _socket_options().items() if k != "socket_connect_timeout"},
)
return SentinelRedisProxy(sentinel, cfg["service"], async_mode=async_mode)
def get_redis_connection( def get_redis_connection(
redis_url, redis_url: str | None,
redis_sentinels, redis_sentinels: list[tuple[str, int]] | None = None,
redis_cluster=False, redis_cluster: bool = False,
async_mode=False, async_mode: bool = False,
decode_responses=True, decode_responses: bool = True,
): ) -> Any | None:
"""Return a cached Redis connection (or create one).
Supports three topologies in order of precedence:
1. **Sentinel** — when ``redis_sentinels`` is non-empty.
2. **Cluster** — when ``redis_cluster`` is True.
3. **Standalone** — plain ``redis://`` connection.
"""
cache_key = ( cache_key = (
redis_url, redis_url,
tuple(redis_sentinels) if redis_sentinels else (), tuple(redis_sentinels) if redis_sentinels else (),
async_mode, async_mode,
decode_responses, decode_responses,
) )
if cache_key in _CONNECTION_POOL:
return _CONNECTION_POOL[cache_key]
if cache_key in _CONNECTION_CACHE: extra = _socket_options()
return _CONNECTION_CACHE[cache_key] connection: Any = None
connection = None
connect_timeout_kwargs = (
{'socket_connect_timeout': REDIS_SOCKET_CONNECT_TIMEOUT} if REDIS_SOCKET_CONNECT_TIMEOUT is not None else {}
)
keepalive_kwargs = {'socket_keepalive': True} if REDIS_SOCKET_KEEPALIVE else {}
health_check_kwargs = {'health_check_interval': REDIS_HEALTH_CHECK_INTERVAL} if REDIS_HEALTH_CHECK_INTERVAL else {}
# Pick the right redis module for sync vs async.
if async_mode: if async_mode:
import redis.asyncio as redis import redis.asyncio as redis_mod
# If using sentinel in async mode
if redis_sentinels:
redis_config = parse_redis_service_url(redis_url)
sentinel = redis.sentinel.Sentinel(
redis_sentinels,
port=redis_config['port'],
db=redis_config['db'],
username=redis_config['username'],
password=redis_config['password'],
decode_responses=decode_responses,
socket_connect_timeout=REDIS_SOCKET_CONNECT_TIMEOUT,
**keepalive_kwargs,
**health_check_kwargs,
)
connection = SentinelRedisProxy(
sentinel,
redis_config['service'],
async_mode=async_mode,
)
elif redis_cluster:
if not redis_url:
raise ValueError('Redis URL must be provided for cluster mode.')
return redis.cluster.RedisCluster.from_url(
redis_url,
decode_responses=decode_responses,
**connect_timeout_kwargs,
**keepalive_kwargs,
**health_check_kwargs,
)
elif redis_url:
connection = redis.from_url(
redis_url,
decode_responses=decode_responses,
**connect_timeout_kwargs,
**keepalive_kwargs,
**health_check_kwargs,
)
else: else:
import redis import redis as redis_mod # type: ignore[no-redef]
if redis_sentinels: if redis_sentinels:
redis_config = parse_redis_service_url(redis_url) connection = _build_sentinel(redis_mod, redis_url, redis_sentinels, decode_responses, async_mode)
sentinel = redis.sentinel.Sentinel( elif redis_cluster:
redis_sentinels, if not redis_url:
port=redis_config['port'], raise ValueError("Redis URL is required for cluster mode.")
db=redis_config['db'], connection = redis_mod.cluster.RedisCluster.from_url(
username=redis_config['username'], redis_url, decode_responses=decode_responses, **extra,
password=redis_config['password'], )
decode_responses=decode_responses, elif redis_url:
socket_connect_timeout=REDIS_SOCKET_CONNECT_TIMEOUT, factory = getattr(redis_mod, "from_url", None) or redis_mod.Redis.from_url
**keepalive_kwargs, connection = factory(redis_url, decode_responses=decode_responses, **extra)
**health_check_kwargs,
)
connection = SentinelRedisProxy(
sentinel,
redis_config['service'],
async_mode=async_mode,
)
elif redis_cluster:
if not redis_url:
raise ValueError('Redis URL must be provided for cluster mode.')
return redis.cluster.RedisCluster.from_url(
redis_url,
decode_responses=decode_responses,
**connect_timeout_kwargs,
**keepalive_kwargs,
**health_check_kwargs,
)
elif redis_url:
connection = redis.Redis.from_url(
redis_url,
decode_responses=decode_responses,
**connect_timeout_kwargs,
**keepalive_kwargs,
**health_check_kwargs,
)
_CONNECTION_CACHE[cache_key] = connection _CONNECTION_POOL[cache_key] = connection
return connection return connection
def get_sentinels_from_env(sentinel_hosts_env, sentinel_port_env):
if sentinel_hosts_env:
sentinel_hosts = sentinel_hosts_env.split(',')
sentinel_port = int(sentinel_port_env)
return [(host, sentinel_port) for host in sentinel_hosts]
return []
def get_sentinel_url_from_env(redis_url, sentinel_hosts_env, sentinel_port_env):
redis_config = parse_redis_service_url(redis_url)
username = redis_config['username'] or ''
password = redis_config['password'] or ''
auth_part = ''
if username or password:
auth_part = f'{username}:{password}@'
hosts_part = ','.join(f'{host}:{sentinel_port_env}' for host in sentinel_hosts_env.split(','))
return f'redis+sentinel://{auth_part}{hosts_part}/{redis_config["db"]}/{redis_config["service"]}'