Production-hardening pass: half-open detection, cascade throttle, OBSERVE refresh, lamp/door coupling
Four failure modes observed in-house since the polling-first refactor
(709fdf4):
1. Half-open DTLS sessions where the socket stays writable but the peer
has gone silent. Ping sends succeed against a wedged peer because
RT-OCF doesn't reliably emit a RST; only successful polls prove the
session is live.
- PollScheduler exposes last_success_ts (bumped on every 2.05).
- KeepaliveTask takes liveness_fn(); ticks fail if no 2.05 in the
last 60s, even when the ping send succeeded.
- Bridge force-closes the session after 120s unreachable so
run_forever() breaks out of sess.join() and reconnects.
2. RT-OCF cascade under load. One wedged path can eat 8s of timeout,
the next tier tick fires immediately and stacks another attempt,
and the device wedges harder.
- PollTier.timeout_s per-tier override (hot=2s, warm=4s, sweep=15s).
- On TimeoutError, the href goes into a 5-60s cooldown via the
existing _defer_until mechanism.
- take_window_stats() now reports successful-poll RTT separately
from a timeout count, exposed as the "Poll Timeouts (window)"
diagnostic entity in HA.
- Active-window throttle: if the previous health window saw >=3
timeouts and is_active=True, drop back to idle cadence -- stops
stacking polls on a stalled responder.
3. OBSERVE table aging across cloud-auth blips. The device stays
DTLS-reachable but the on-device stack clears its observer table
during the blip, so push delivery stays dead even after upstream
recovers.
- New ObserveRefreshTask per bridge; every 6h derregs all current
observer tokens and re-subscribes on the existing session.
4. Oven lamp/door coupling. Oven hardware auto-drives the lamp from
door state but /mode/vs/0 is warm-tier (30s) so HA showed stale lamp
during a cook.
- Track door + lamp value-change timestamps in descriptor_state.
When the door transition is newer, derive lamp from door_open.
When an HA optimistic write is newer, the cache value wins.
Also: ANSI-coloured WARNING/ERROR lines (NO_COLOR=1 opt-out), jittered
reconnect backoff so dryer + oven don't sync up after a router blip.
In-house verification: running on dryer + oven since 2026-06-03.
This commit is contained in:
1 parent
0374f8cf68
commit
36acf3a209
9 files changed
+438
-62
No files matched your search
@@ -17,6 +17,7 @@ MQTT client; each owns one DTLS session, one cache, one scheduler.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -27,6 +28,7 @@ from .coap_dtls import DtlsCoapSession, fmt_code
|
||||
from .config import ApplianceConfig, SharedConfig
|
||||
from .keepalive import KeepaliveTask
|
||||
from .logger import bridge_logger
|
||||
from .observe_refresh import ObserveRefreshTask
|
||||
from .poll_scheduler import PollScheduler
|
||||
from .sensors import index_links
|
||||
from .state_cache import StateCache
|
||||
@@ -42,6 +44,21 @@ def _href_to_segs(href: str) -> list[str]:
|
||||
SERIAL_PATH = '/information/vs/0'
|
||||
SERIAL_FIELD = 'x.com.samsung.da.serialNum'
|
||||
|
||||
# If the keepalive watchdog flags the device unreachable for this long,
|
||||
# force a session reconnect. Catches the half-open case where the DTLS
|
||||
# socket is still writable but the peer has gone silent — without this,
|
||||
# the bridge sits in offline state until the reader thread dies on its
|
||||
# own (which may not happen at all if the OS sees no socket errors).
|
||||
UNREACHABLE_RECONNECT_S = 120.0
|
||||
|
||||
# Periodic OBSERVE re-subscribe interval. Safety net for the case where
|
||||
# the device stays reachable on the DTLS layer but Samsung's RT-OCF
|
||||
# clears its observer table (e.g. during cloud auth blips). Without
|
||||
# this, push delivery stays dead even after upstream connectivity
|
||||
# recovers, since nothing triggers a fresh subscribe on the existing
|
||||
# session.
|
||||
OBSERVE_REFRESH_INTERVAL_S = 6 * 3600.0
|
||||
|
||||
|
||||
class PushBridge:
|
||||
|
||||
@@ -63,6 +80,7 @@ class PushBridge:
|
||||
self.session: DtlsCoapSession | None = None
|
||||
self.scheduler: PollScheduler | None = None
|
||||
self.keepalive: KeepaliveTask | None = None
|
||||
self.observe_refresh: ObserveRefreshTask | None = None
|
||||
|
||||
self.cache = StateCache(descriptor)
|
||||
self.cache.set_on_change(self._on_cache_change)
|
||||
@@ -83,6 +101,11 @@ class PushBridge:
|
||||
self._last_change_source: str | None = None
|
||||
self._last_observe_change_ts: float | None = None
|
||||
self._last_push_active_pub: str | None = None
|
||||
# Wall-clock timestamp the keepalive watchdog first reported the
|
||||
# device unreachable on the current session. Cleared on recovery
|
||||
# or session start. Drives the force-reconnect watchdog below.
|
||||
self._unreachable_since: float | None = None
|
||||
self._force_close_in_flight: bool = False
|
||||
|
||||
# Push is considered "active" if an OBSERVE-sourced change
|
||||
# arrived within this window. Long enough that a quiet but
|
||||
@@ -222,6 +245,8 @@ class PushBridge:
|
||||
self.connect_count += 1
|
||||
self.cache.descriptor_state.clear()
|
||||
self._publish_gate = False
|
||||
self._unreachable_since = None
|
||||
self._force_close_in_flight = False
|
||||
|
||||
self.log.info("DTLS connected — subscribing %d paths",
|
||||
len(self.descriptor.observe_paths))
|
||||
@@ -274,6 +299,10 @@ class PushBridge:
|
||||
is_active_fn=self.descriptor.is_active,
|
||||
logger=self.log,
|
||||
)
|
||||
# Half-open detection: if no successful poll lands inside this
|
||||
# window the session is wedged, regardless of whether ping sends
|
||||
# leave the socket. 60s gives ~60 hot-tier cycles of margin.
|
||||
liveness_window_s = 60.0
|
||||
keepalive = KeepaliveTask(
|
||||
sess,
|
||||
interval_s=float(self.shared.PING_INTERVAL_S),
|
||||
@@ -281,9 +310,18 @@ class PushBridge:
|
||||
on_reachable=self._on_reachable,
|
||||
on_unreachable=self._on_unreachable,
|
||||
logger=self.log,
|
||||
liveness_fn=lambda: (time.monotonic() - scheduler.last_success_ts
|
||||
) < liveness_window_s,
|
||||
)
|
||||
observe_refresh = ObserveRefreshTask(
|
||||
sess,
|
||||
paths=self.descriptor.observe_paths,
|
||||
interval_s=OBSERVE_REFRESH_INTERVAL_S,
|
||||
logger=self.log,
|
||||
)
|
||||
self.scheduler = scheduler
|
||||
self.keepalive = keepalive
|
||||
self.observe_refresh = observe_refresh
|
||||
|
||||
sched_t = threading.Thread(
|
||||
target=scheduler.run_forever, args=(self.stop,),
|
||||
@@ -291,14 +329,19 @@ class PushBridge:
|
||||
ka_t = threading.Thread(
|
||||
target=keepalive.run_forever, args=(self.stop,),
|
||||
daemon=True, name=f'{self.app.klass}-ping')
|
||||
ref_t = threading.Thread(
|
||||
target=observe_refresh.run_forever, args=(self.stop,),
|
||||
daemon=True, name=f'{self.app.klass}-obsref')
|
||||
sched_t.start()
|
||||
ka_t.start()
|
||||
ref_t.start()
|
||||
|
||||
try:
|
||||
sess.join()
|
||||
finally:
|
||||
self.scheduler = None
|
||||
self.keepalive = None
|
||||
self.observe_refresh = None
|
||||
|
||||
def _seed_from_device0(self, sess):
|
||||
code, pl = sess.get(self.descriptor.seed_path, timeout=15.0)
|
||||
@@ -340,12 +383,37 @@ class PushBridge:
|
||||
self.log.warning("oic/res get: %s", e)
|
||||
|
||||
def _on_reachable(self) -> None:
|
||||
self._unreachable_since = None
|
||||
self.set_availability(True)
|
||||
self.reassert_availability()
|
||||
|
||||
def _on_unreachable(self) -> None:
|
||||
if self._unreachable_since is None:
|
||||
self._unreachable_since = time.time()
|
||||
self.set_availability(False)
|
||||
|
||||
def _maybe_force_reconnect(self) -> None:
|
||||
"""If the device has been unreachable for UNREACHABLE_RECONNECT_S,
|
||||
close the DTLS session to break run_forever's session_once() out
|
||||
of its sess.join() and trigger a fresh connect. Without this, a
|
||||
half-open session (writable socket, silent peer) holds the bridge
|
||||
in offline limbo until the OS surfaces a socket error."""
|
||||
if self._unreachable_since is None or self._force_close_in_flight:
|
||||
return
|
||||
elapsed = time.time() - self._unreachable_since
|
||||
if elapsed < UNREACHABLE_RECONNECT_S:
|
||||
return
|
||||
sess = self.session
|
||||
if sess is None:
|
||||
return
|
||||
self.log.warning(
|
||||
"unreachable for %.0fs — forcing session reconnect", elapsed)
|
||||
self._force_close_in_flight = True
|
||||
try:
|
||||
sess.close()
|
||||
except Exception as e:
|
||||
self.log.warning("force-close: %s", e)
|
||||
|
||||
# ---- MQTT publishing --------------------------------------------
|
||||
|
||||
def maybe_publish_state(self, force=False):
|
||||
@@ -519,8 +587,8 @@ class PushBridge:
|
||||
self._win_prev_poll = poll
|
||||
self._win_prev_poll_err = poll_err
|
||||
self._win_prev_ping_fail = ping_fail
|
||||
win_max_rtt, win_slow = (sched.take_window_stats()
|
||||
if sched else (0.0, 0))
|
||||
win_max_rtt, win_slow, win_timeouts = (
|
||||
sched.take_window_stats() if sched else (0.0, 0, 0))
|
||||
window_polls_ok = max(0, d_poll - d_err)
|
||||
|
||||
h = {
|
||||
@@ -536,6 +604,7 @@ class PushBridge:
|
||||
'poll_window_errors': d_err,
|
||||
'poll_window_max_rtt_ms': round(win_max_rtt, 0),
|
||||
'poll_window_slow_count': win_slow,
|
||||
'poll_window_timeout_count': win_timeouts,
|
||||
'ping_count': ka.ping_count if ka else 0,
|
||||
'ping_fail_count': ping_fail,
|
||||
'reachable': ka.reachable if ka else None,
|
||||
@@ -561,13 +630,15 @@ class PushBridge:
|
||||
self.log.warning("health publish: %s", e)
|
||||
|
||||
self.publish_push_active(push_active)
|
||||
self._maybe_force_reconnect()
|
||||
|
||||
if d_poll > 0 or d_err > 0 or d_ping_fail > 0:
|
||||
self.log.info(
|
||||
"poll-window: %d ok, %d err, %d ping-fail, "
|
||||
"p_max=%.0fms, slow=%d (%ds)",
|
||||
"p_max=%.0fms, slow=%d, timeouts=%d (%ds)",
|
||||
window_polls_ok, d_err, d_ping_fail,
|
||||
win_max_rtt, win_slow, self.shared.HEALTH_INTERVAL_S)
|
||||
win_max_rtt, win_slow, win_timeouts,
|
||||
self.shared.HEALTH_INTERVAL_S)
|
||||
|
||||
def publish_push_active(self, active: bool, force: bool = False) -> None:
|
||||
value = 'online' if active else 'offline'
|
||||
@@ -601,8 +672,12 @@ class PushBridge:
|
||||
self.session_started_ts = None
|
||||
if self.stop.is_set():
|
||||
break
|
||||
wait = min(backoff, 30.0)
|
||||
self.log.info("reconnect in %.0fs", wait)
|
||||
# Jitter the backoff so multiple bridges (dryer + oven) don't
|
||||
# reconnect in lockstep after a router blip — synchronized
|
||||
# storms make the broker / DTLS layer flap harder than need
|
||||
# be. ±30% noise spreads the retry attempts.
|
||||
wait = min(backoff, 30.0) * random.uniform(0.7, 1.3)
|
||||
self.log.info("reconnect in %.1fs", wait)
|
||||
if self.stop.wait(wait):
|
||||
break
|
||||
backoff = min(backoff * 2, 30.0)
|
||||
Reference in new issue
Block a user