Source code for fleetmesh_node.transport.ladder

"""Per-rung circuit breaker and the rung base class for the transport ladder.

The ladder itself - which rung is tried first for a peer, and in what order
the rest follow - lives in router.py (TransportRouter); the order is
fips-native, iroh, libp2p, nostr and is not decided here. This module holds
the pieces every rung shares: a CircuitBreaker that stops the router from
hanging on a target that keeps failing, a FaultLog so that what a rung
dropped and why is readable rather than only countable. The rung contract
itself (RungProtocol, Rung) is in rung.py. Standard library only.
"""
from __future__ import annotations

import collections
import threading
import time
from typing import Any, Counter, Deque, Dict, Iterable, Optional, TypedDict


[docs]class Fault(TypedDict): """One thing a rung refused or lost, kept so an operator can read it.""" at: int # the node clock when it happened rung: str # 'nostr', 'fips-native', 'iroh', 'libp2p', or 'router' reason: str # 'publish refused: wss://... invalid: ...'; the prefix before ':' is the family ident: Optional[str] # the wrap, stream or message id it concerned, when there was one
[docs]class FaultLog: """The most recent faults of one rung and a count per family, ever. A count alone said '0 faults' while every wrap was being refused; the reasons are what tell an operator the relay said 'ephemeral event expired'. Thread-safe; readers get copies. """
[docs] def __init__(self, rung: str, clock: Any = None, keep: int = 200) -> None: self.rung: str = rung self.clock: Any = clock or (lambda: int(time.time())) self._recent: Deque[Fault] = collections.deque(maxlen=keep) self.counts: Counter[str] = collections.Counter() self._lock: threading.Lock = threading.Lock()
[docs] def record(self, reason: str, ident: Optional[str] = None) -> Fault: f = Fault(at=int(self.clock()), rung=self.rung, reason=reason, ident=ident) with self._lock: self._recent.append(f) self.counts[reason.split(':', 1)[0].strip()] += 1 return f
[docs] def recent(self, since: Optional[int] = None, family: Optional[str] = None) -> list[Fault]: """Newest last. `since` keeps faults at or after that clock reading; `family` keeps the reason prefix ('publish refused').""" with self._lock: rows = list(self._recent) return [f for f in rows if (since is None or f['at'] >= since) and (family is None or f['reason'].split(':', 1)[0].strip() == family)]
@property def maxlen(self) -> Optional[int]: return self._recent.maxlen def __len__(self) -> int: return len(self._recent) def __iter__(self) -> Iterable[Fault]: with self._lock: return iter(list(self._recent))
[docs]def merge_faults(logs: Iterable[FaultLog], since: Optional[int] = None) -> list[Fault]: """Faults across rungs in time order, for the ladder's aggregate view.""" out: list[Fault] = [] for lg in logs: out.extend(lg.recent(since)) out.sort(key=lambda f: f['at']) return out
[docs]class CircuitBreaker: """Per-target circuit breaker to avoid hanging on repeatedly failing transports.""" STATE_CLOSED: str = 'closed' STATE_OPEN: str = 'open' STATE_HALF_OPEN: str = 'half_open'
[docs] def __init__(self, failure_threshold: int = 3, recovery_timeout: float = 30.0) -> None: self.failure_threshold: int = failure_threshold self.recovery_timeout: float = recovery_timeout self._lock: threading.RLock = threading.RLock() self._failures: collections.Counter[str] = collections.Counter() self._last_failure: Dict[str, float] = {} self._states: Dict[str, str] = {}
[docs] def allow_attempt(self, target: str, now: Optional[float] = None) -> bool: """Return True if an attempt to reach target is permitted.""" with self._lock: t_now = time.time() if now is None else now state = self._states.get(target, self.STATE_CLOSED) if state == self.STATE_CLOSED: return True last = self._last_failure.get(target, 0.0) if t_now - last >= self.recovery_timeout: self._states[target] = self.STATE_HALF_OPEN return True return False
[docs] def record_success(self, target: str) -> None: """Record successful transmission, resetting circuit to CLOSED.""" with self._lock: self._failures[target] = 0 self._states[target] = self.STATE_CLOSED
[docs] def record_failure(self, target: str, now: Optional[float] = None) -> None: """Record transmission failure, potentially tripping circuit to OPEN.""" with self._lock: t_now = time.time() if now is None else now self._last_failure[target] = t_now self._failures[target] += 1 if self._failures[target] >= self.failure_threshold: self._states[target] = self.STATE_OPEN
[docs] def state(self, target: str) -> str: """Current state for target.""" with self._lock: return self._states.get(target, self.STATE_CLOSED)
[docs] def failures(self, target: str) -> int: """Return current consecutive failure count for target.""" with self._lock: return self._failures[target]
[docs] def reset(self, target: Optional[str] = None) -> None: """Reset circuit breaker for a target or all targets.""" with self._lock: if target is None: self._failures.clear() self._last_failure.clear() self._states.clear() else: self._failures.pop(target, None) self._last_failure.pop(target, None) self._states.pop(target, None)
__all__ = ['CircuitBreaker', 'Fault', 'FaultLog', 'merge_faults']