refactor: extract ocf/ — fold sensors.index_links into StateCache.index_device_tree

ocf/keepalive.py

@@ -0,0 +1,112 @@
+"""DTLS-layer liveness via CoAP empty-CON ping + poll-success watchdog.
+
+Each interval_s the task:
+  1. Sends a CoAP ping. This is fire-and-forget — Samsung's RT-OCF
+     doesn't reliably reply with an RST, so the send itself is the
+     keepalive (it tickles Samsung's observer state). The send only
+     fails if the underlying socket is gone, in which case the failure
+     counts toward fail_threshold.
+  2. Calls liveness_fn() if provided. This is the real half-open
+     detection: PollScheduler exposes last_success_ts, and the bridge
+     wraps it as "did we get a 2.05 in the last 60s". If not, count
+     the tick as a failure even though the ping send succeeded.
+
+After fail_threshold consecutive failures, fires on_unreachable. First
+success after a fail streak fires on_reachable. Bridge wires these to
+MQTT availability.
+"""
+from __future__ import annotations
+
+import threading
+from typing import Callable, Optional
+
+from protocol.dtls_session import DtlsCoapSession
+
+
+class KeepaliveTask:
+
+    def __init__(self,
+                 session: DtlsCoapSession,
+                 interval_s: float = 25.0,
+                 fail_threshold: int = 3,
+                 on_reachable: Optional[Callable[[], None]] = None,
+                 on_unreachable: Optional[Callable[[], None]] = None,
+                 logger=None,
+                 liveness_fn: Optional[Callable[[], bool]] = None):
+        self.session = session
+        self.interval_s = interval_s
+        self.fail_threshold = fail_threshold
+        self.on_reachable = on_reachable
+        self.on_unreachable = on_unreachable
+        self.log = logger
+        self.liveness_fn = liveness_fn
+
+        self._fail_streak = 0
+        self._reachable = True
+        self._ping_count = 0
+        self._ping_fail_count = 0
+
+    @property
+    def ping_count(self) -> int:
+        return self._ping_count
+
+    @property
+    def ping_fail_count(self) -> int:
+        return self._ping_fail_count
+
+    @property
+    def reachable(self) -> bool:
+        return self._reachable
+
+    def run_forever(self, stop: threading.Event) -> None:
+        while not stop.wait(self.interval_s):
+            self._tick()
+
+    def _tick(self) -> None:
+        ok = False
+        try:
+            self.session.ping()
+            ok = True
+        except ConnectionError as e:
+            if self.log: self.log.debug("ping: %s", e)
+        except Exception as e:
+            if self.log: self.log.warning("ping: %s", e)
+        # Real half-open detection: ping sends can succeed against a
+        # silently-wedged peer, but polls won't. If the scheduler
+        # hasn't recorded a 2.05 inside the liveness window, treat
+        # this tick as a failure even though the ping itself went out.
+        if ok and self.liveness_fn is not None:
+            try:
+                alive = bool(self.liveness_fn())
+            except Exception as e:
+                if self.log: self.log.warning("liveness_fn: %s", e)
+                alive = True
+            if not alive:
+                if self.log:
+                    self.log.warning("liveness: no successful poll "
+                                     "in the liveness window")
+                ok = False
+        self._ping_count += 1
+        if ok:
+            if not self._reachable:
+                if self.log:
+                    self.log.info("ping recovered after %d fails",
+                                  self._fail_streak)
+                self._reachable = True
+                if self.on_reachable is not None:
+                    try: self.on_reachable()
+                    except Exception as e:
+                        if self.log: self.log.warning("on_reachable: %s", e)
+            self._fail_streak = 0
+            return
+        self._ping_fail_count += 1
+        self._fail_streak += 1
+        if self._reachable and self._fail_streak >= self.fail_threshold:
+            if self.log:
+                self.log.warning("device unreachable after %d ping failures",
+                                 self._fail_streak)
+            self._reachable = False
+            if self.on_unreachable is not None:
+                try: self.on_unreachable()
+                except Exception as e:
+                    if self.log: self.log.warning("on_unreachable: %s", e)

ocf/observe_refresh.py

@@ -0,0 +1,50 @@
+"""Periodic OBSERVE re-subscribe.
+
+CoAP OBSERVE (RFC 7641) has no built-in TTL, but real-world peers age
+out observer state on their own schedule — Samsung's RT-OCF is known
+to silently drop notify delivery during cloud auth blips even though
+the DTLS session stays healthy. Without a re-subscribe, recovery from
+such a blip requires a full session reconnect.
+
+This task derregisters the current observer tokens and re-subscribes
+every `interval_s`. Cheap (one register CON per path), idempotent
+(Samsung silently no-ops a register on an already-active token), and
+resilient — individual subscribe failures are logged but don't abort
+the task.
+"""
+from __future__ import annotations
+
+import threading
+from typing import Optional
+
+from protocol.dtls_session import DtlsCoapSession
+
+
+class ObserveRefreshTask:
+
+    def __init__(self,
+                 session: DtlsCoapSession,
+                 paths,
+                 interval_s: float = 6 * 3600.0,
+                 logger=None):
+        self.session = session
+        self.paths = [list(p) for p in paths]
+        self.interval_s = interval_s
+        self.log = logger
+        self._refresh_count = 0
+
+    @property
+    def refresh_count(self) -> int:
+        return self._refresh_count
+
+    def run_forever(self, stop: threading.Event) -> None:
+        while not stop.wait(self.interval_s):
+            try:
+                self.session.refresh_observes(self.paths)
+                self._refresh_count += 1
+                if self.log:
+                    self.log.info("OBSERVE refresh #%d (%d paths)",
+                                  self._refresh_count, len(self.paths))
+            except Exception as e:
+                if self.log:
+                    self.log.warning("OBSERVE refresh: %s", e)

ocf/poll_scheduler.py

@@ -0,0 +1,311 @@
+"""Tiered adaptive polling against a DtlsCoapSession.
+
+Tiers are descriptor-declared (hot/warm/cold + sweep). Per tick:
+each tier whose deadline has passed polls all its paths sequentially
+on the shared session, writing into the StateCache. The sweep tier
+issues one Block2 GET of /device/0 and uses index_links to fan its
+result into many href reps.
+
+Adaptive cadence: when descriptor.is_active(cache.links) returns True
+and tier.active_interval_s is set, that tier uses the tighter cadence.
+If the previous health window saw `active_throttle_threshold` timeouts,
+the throttle drops back to idle cadence even when active=True — the
+RT-OCF stack wedges under load and stacking poll attempts only makes
+it worse.
+
+Per-tier timeouts: tier.timeout_s overrides the scheduler default. Hot
+tiers want a tight ceiling (e.g. 2s) so one wedged path can't eat
+several poll cycles.
+
+Cooldown on timeout: when a path times out, it's deferred for ~3 of
+its tier's intervals (clamped 5–60s) via the same _defer_until mechanism
+used for write_in_progress. Breaks the cluster-cascade where the next
+tier tick fires immediately after an 8s wedge and reattempts the same
+stalled path.
+
+Post-write defer: bridge calls write_in_progress(href) before POSTing
+a write; the scheduler skips that href for settle_s to avoid Samsung's
+fetchback-revert bug.
+"""
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+from typing import Callable, Optional, TYPE_CHECKING
+
+import cbor2
+
+from protocol.dtls_session import DtlsCoapSession, fmt_code
+
+if TYPE_CHECKING:
+    from .state_cache import StateCache
+
+
+@dataclass(frozen=True)
+class PollTier:
+    name: str
+    interval_s: float
+    paths: tuple[tuple[str, ...], ...]
+    active_interval_s: Optional[float] = None
+    is_sweep: bool = False
+    # Per-tier CoAP request timeout. Falls back to PollScheduler.timeout_s
+    # when None. Hot tiers want a tight ceiling (e.g. 2s) so one wedged
+    # path can't eat several poll cycles; sweep tiers tolerate longer.
+    timeout_s: Optional[float] = None
+
+
+class PollScheduler:
+
+    def __init__(self,
+                 session: DtlsCoapSession,
+                 cache: 'StateCache',
+                 tiers: list[PollTier],
+                 is_active_fn: Optional[Callable[[dict[str, dict]], bool]] = None,
+                 logger=None,
+                 timeout_s: float = 8.0,
+                 active_throttle_timeout_threshold: int = 3):
+        self.session = session
+        self.cache = cache
+        self.tiers = tiers
+        self.is_active_fn = is_active_fn
+        self.log = logger
+        self.timeout_s = timeout_s
+        self.active_throttle_threshold = active_throttle_timeout_threshold
+
+        now = time.monotonic()
+        self._next_due: dict[str, float] = {t.name: now for t in tiers}
+        self._defer_until: dict[str, float] = {}
+        self._defer_lock = threading.Lock()
+        self._poll_count = 0
+        self._poll_error_count = 0
+        self._last_active: Optional[bool] = None
+        self._last_throttled: bool = False
+        # Real-liveness signal for KeepaliveTask: updated on every 2.05
+        # we receive from the wire. Initialized to "now" so the first
+        # keepalive tick after start doesn't fire a false unreachable.
+        self._last_success_ts: float = now
+
+        # Per-window tail-latency tracking. Bridge consumes-and-resets
+        # these via take_window_stats() once per HEALTH_INTERVAL_S.
+        # _window_max_rtt_ms tracks SUCCESSFUL polls only — timeouts go
+        # into _window_timeout_count so the dashboard sees the real tail
+        # instead of an 8000ms wall.
+        self._stats_lock = threading.Lock()
+        self._window_max_rtt_ms = 0.0
+        self._window_slow_count = 0
+        self._window_timeout_count = 0
+        # Snapshot of the previous window's timeout count, used by the
+        # active-window throttle to back off when polls are wedging.
+        self._last_window_timeouts = 0
+        self.slow_threshold_ms = 1000.0
+
+    def write_in_progress(self, href: str, settle_s: float = 4.0) -> None:
+        with self._defer_lock:
+            self._defer_until[href] = time.monotonic() + settle_s
+
+    def run_forever(self, stop: threading.Event) -> None:
+        while not stop.is_set():
+            self._run_due_tiers()
+            sleep_for = max(0.05, min(1.0, self._earliest_deadline() - time.monotonic()))
+            if stop.wait(sleep_for):
+                return
+
+    @property
+    def poll_count(self) -> int:
+        return self._poll_count
+
+    @property
+    def poll_error_count(self) -> int:
+        return self._poll_error_count
+
+    @property
+    def last_success_ts(self) -> float:
+        """Monotonic timestamp of the most recent 2.05 response from any
+        tier. KeepaliveTask uses this as its half-open-detection signal:
+        if no 2.05 has landed in `liveness_window_s`, the session is
+        wedged regardless of whether ping sends succeed."""
+        return self._last_success_ts
+
+    def take_window_stats(self) -> tuple[float, int, int]:
+        """Return (max RTT ms over successful polls, slow-poll count,
+        timeout count) seen since the last call, and reset all three.
+        Slow threshold is `self.slow_threshold_ms`. The timeout count
+        is snapshotted into `_last_window_timeouts` for the throttle."""
+        with self._stats_lock:
+            out = (self._window_max_rtt_ms,
+                   self._window_slow_count,
+                   self._window_timeout_count)
+            self._last_window_timeouts = self._window_timeout_count
+            self._window_max_rtt_ms = 0.0
+            self._window_slow_count = 0
+            self._window_timeout_count = 0
+        return out
+
+    def _record_rtt(self, rtt_ms: float, *, timed_out: bool = False) -> None:
+        with self._stats_lock:
+            if timed_out:
+                self._window_timeout_count += 1
+                return
+            if rtt_ms > self._window_max_rtt_ms:
+                self._window_max_rtt_ms = rtt_ms
+            if rtt_ms >= self.slow_threshold_ms:
+                self._window_slow_count += 1
+
+    def _earliest_deadline(self) -> float:
+        return min(self._next_due.values())
+
+    def _run_due_tiers(self) -> None:
+        now = time.monotonic()
+        active = False
+        if self.is_active_fn is not None:
+            try:
+                active = bool(self.is_active_fn(self.cache.snapshot()))
+            except Exception as e:
+                if self.log: self.log.warning("is_active: %s", e)
+        if active != self._last_active:
+            if self.log and self._last_active is not None:
+                self.log.info("active=%s", active)
+            self._last_active = active
+        # Active-window throttle: if the previous health window saw a
+        # cluster of timeouts (RT-OCF wedging under load), drop back to
+        # idle cadence even when is_active=True. Lets the device breathe
+        # instead of stacking poll attempts on a stalled responder.
+        with self._stats_lock:
+            recent_to = self._last_window_timeouts
+        throttled = (active
+                     and recent_to >= self.active_throttle_threshold)
+        if throttled != self._last_throttled:
+            if self.log:
+                if throttled:
+                    self.log.warning(
+                        "active-throttle ON (%d timeouts last window) — "
+                        "using idle cadence", recent_to)
+                else:
+                    self.log.info("active-throttle OFF")
+            self._last_throttled = throttled
+        effective_active = active and not throttled
+        for tier in self.tiers:
+            if self._next_due[tier.name] > now:
+                continue
+            interval = (tier.active_interval_s
+                        if (effective_active and tier.active_interval_s is not None)
+                        else tier.interval_s)
+            self._next_due[tier.name] = now + interval
+            try:
+                if tier.is_sweep:
+                    self._do_sweep(tier)
+                else:
+                    self._do_tier(tier)
+            except Exception as e:
+                self._poll_error_count += 1
+                if self.log: self.log.warning("tier %s: %s", tier.name, e)
+
+    def _tier_timeout(self, tier: PollTier) -> float:
+        return tier.timeout_s if tier.timeout_s is not None else self.timeout_s
+
+    def _cooldown_for(self, tier: PollTier) -> float:
+        # On timeout, defer the wedged href for ~3 cycles (clamped to a
+        # 5–60s band) so repeated tier ticks don't stack attempts on a
+        # stalled responder. Breaks the cluster-cascade we see in the
+        # Poll Max RTT chart during heavy device use.
+        return max(5.0, min(60.0, tier.interval_s * 3.0))
+
+    def _set_cooldown(self, href: str, cooldown_s: float) -> None:
+        with self._defer_lock:
+            self._defer_until[href] = time.monotonic() + cooldown_s
+
+    def _do_tier(self, tier: PollTier) -> None:
+        timeout = self._tier_timeout(tier)
+        cooldown = self._cooldown_for(tier)
+        for path in tier.paths:
+            href = '/' + '/'.join(path)
+            with self._defer_lock:
+                if self._defer_until.get(href, 0) > time.monotonic():
+                    continue
+            self._poll_count += 1
+            t0 = time.monotonic()
+            try:
+                code, body = self.session.get(list(path), timeout=timeout)
+            except TimeoutError:
+                self._poll_error_count += 1
+                self._record_rtt(0.0, timed_out=True)
+                self._set_cooldown(href, cooldown)
+                if self.log:
+                    self.log.warning("poll %s timeout (cooldown %.0fs)",
+                                     href, cooldown)
+                return
+            except ConnectionError as e:
+                self._poll_error_count += 1
+                self._record_rtt((time.monotonic() - t0) * 1000.0)
+                if self.log: self.log.debug("poll %s: %s", href, e)
+                return
+            except Exception as e:
+                self._poll_error_count += 1
+                self._record_rtt((time.monotonic() - t0) * 1000.0)
+                if self.log: self.log.warning("poll %s: %s", href, e)
+                return
+            self._record_rtt((time.monotonic() - t0) * 1000.0)
+            if code != 0x45 or not body:
+                self._poll_error_count += 1
+                if self.log: self.log.warning("poll %s -> %s", href, fmt_code(code))
+                continue
+            self._last_success_ts = time.monotonic()
+            try:
+                rep = cbor2.loads(body)
+            except Exception as e:
+                self._poll_error_count += 1
+                if self.log: self.log.warning("poll %s cbor: %s", href, e)
+                continue
+            if isinstance(rep, dict):
+                self.cache.apply_rep(href, rep, source='poll')
+
+    def _do_sweep(self, tier: PollTier) -> None:
+        timeout = self._tier_timeout(tier)
+        cooldown = self._cooldown_for(tier)
+        path = list(tier.paths[0])
+        href = '/' + '/'.join(path)
+        t0 = time.monotonic()
+        self._poll_count += 1
+        try:
+            code, body = self.session.get(path, timeout=timeout)
+        except TimeoutError:
+            self._poll_error_count += 1
+            self._record_rtt(0.0, timed_out=True)
+            self._set_cooldown(href, cooldown)
+            if self.log:
+                self.log.warning("sweep %s timeout (cooldown %.0fs)",
+                                 path, cooldown)
+            return
+        except ConnectionError as e:
+            self._poll_error_count += 1
+            self._record_rtt((time.monotonic() - t0) * 1000.0)
+            if self.log: self.log.debug("sweep %s: %s", path, e)
+            return
+        except Exception as e:
+            self._poll_error_count += 1
+            self._record_rtt((time.monotonic() - t0) * 1000.0)
+            if self.log: self.log.warning("sweep %s: %s", path, e)
+            return
+        self._record_rtt((time.monotonic() - t0) * 1000.0)
+        if code != 0x45 or not body:
+            self._poll_error_count += 1
+            if self.log: self.log.warning("sweep -> %s", fmt_code(code))
+            return
+        self._last_success_ts = time.monotonic()
+        try:
+            tree = cbor2.loads(body)
+        except Exception as e:
+            self._poll_error_count += 1
+            if self.log: self.log.warning("sweep cbor: %s", e)
+            return
+        indexed = self.cache.index_device_tree(tree)
+        for href, rep in indexed.items():
+            with self._defer_lock:
+                if self._defer_until.get(href, 0) > time.monotonic():
+                    continue
+            self.cache.apply_rep(href, rep, source='sweep')
+        if self.log:
+            elapsed_ms = (time.monotonic() - t0) * 1000.0
+            self.log.info("sweep complete (%d links, %.0fms)",
+                          len(indexed), elapsed_ms)

ocf/state_cache.py

@@ -0,0 +1,95 @@
+"""Single source of truth for one appliance's state.
+
+All writers (OBSERVE notify, poll, seed, optimistic) call apply_rep().
+A registered on_change callback fires after any apply that mutated the
+cache, which the bridge wires to its MQTT publish gate.
+"""
+from __future__ import annotations
+
+import threading
+import time
+from typing import Callable, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .appliances.base import ApplianceDescriptor
+
+
+class StateCache:
+
+    def __init__(self, descriptor: 'ApplianceDescriptor'):
+        self.descriptor = descriptor
+        self.links: dict[str, dict] = {}
+        self.last_updated: dict[str, float] = {}
+        self.source: dict[str, str] = {}
+        self.descriptor_state: dict = {}
+        self._on_change: Optional[Callable[[bool, str], None]] = None
+        self._lock = threading.RLock()
+
+    def set_on_change(self, cb: Callable[[bool, str], None]) -> None:
+        self._on_change = cb
+
+    def apply_rep(self, href: str, rep: dict, source: str) -> bool:
+        if not isinstance(rep, dict):
+            return False
+        with self._lock:
+            prior = self.links.get(href)
+            changed = prior != rep
+            self.links[href] = rep
+            self.last_updated[href] = time.time()
+            self.source[href] = source
+        hook = self.descriptor.on_observation
+        if hook is not None:
+            try:
+                hook(self.descriptor_state, href, rep)
+            except Exception:
+                pass
+        if self._on_change is not None:
+            try:
+                self._on_change(changed, source)
+            except Exception:
+                pass
+        return changed
+
+    def apply_optimistic(self, href: str, body: dict) -> bool:
+        if not isinstance(body, dict):
+            return False
+        with self._lock:
+            merged = dict(self.links.get(href) or {})
+            merged.update(body)
+        return self.apply_rep(href, merged, source='optimistic')
+
+    @staticmethod
+    def index_device_tree(device0_body) -> dict[str, dict]:
+        """Turn a /device/0 CBOR list-of-{href, rep} sweep response into
+        a dict keyed by href. Entry [0] is the device-level rep itself
+        and isn't useful here, so it's skipped.
+
+        Replaces the old standalone sensors.index_links — folded in
+        here because every current and future caller immediately feeds
+        the result into apply_rep on this same cache."""
+        out: dict[str, dict] = {}
+        if not isinstance(device0_body, list):
+            return out
+        for entry in device0_body[1:]:
+            if isinstance(entry, dict) and 'href' in entry:
+                out[entry['href']] = entry.get('rep') or {}
+        return out
+
+    def get(self, href: str) -> Optional[dict]:
+        with self._lock:
+            return self.links.get(href)
+
+    def snapshot(self) -> dict[str, dict]:
+        with self._lock:
+            return dict(self.links)
+
+    def freshness_s(self, href: str) -> Optional[float]:
+        ts = self.last_updated.get(href)
+        return None if ts is None else (time.time() - ts)
+
+    def stalest(self) -> Optional[tuple[str, float]]:
+        with self._lock:
+            if not self.last_updated:
+                return None
+            href = min(self.last_updated, key=self.last_updated.get)
+            return href, time.time() - self.last_updated[href]