"""Per-rung circuit breaker and the rung base class for the transport ladder.
The ladder itself - which rung is tried first for a peer, and in what order
the rest follow - lives in router.py (TransportRouter); the order is
fips-native, iroh, libp2p, nostr and is not decided here. This module holds
the pieces every rung shares: a CircuitBreaker that stops the router from
hanging on a target that keeps failing, a FaultLog so that what a rung
dropped and why is readable rather than only countable. The rung contract
itself (RungProtocol, Rung) is in rung.py. Standard library only.
"""
from __future__ import annotations
import collections
import threading
import time
from typing import Any, Counter, Deque, Dict, Iterable, Optional, TypedDict
[docs]class Fault(TypedDict):
"""One thing a rung refused or lost, kept so an operator can read it."""
at: int # the node clock when it happened
rung: str # 'nostr', 'fips-native', 'iroh', 'libp2p', or 'router'
reason: str # 'publish refused: wss://... invalid: ...'; the prefix before ':' is the family
ident: Optional[str] # the wrap, stream or message id it concerned, when there was one
[docs]class FaultLog:
"""The most recent faults of one rung and a count per family, ever.
A count alone said '0 faults' while every wrap was being refused; the
reasons are what tell an operator the relay said 'ephemeral event
expired'. Thread-safe; readers get copies.
"""
[docs] def __init__(self, rung: str, clock: Any = None, keep: int = 200) -> None:
self.rung: str = rung
self.clock: Any = clock or (lambda: int(time.time()))
self._recent: Deque[Fault] = collections.deque(maxlen=keep)
self.counts: Counter[str] = collections.Counter()
self._lock: threading.Lock = threading.Lock()
[docs] def record(self, reason: str, ident: Optional[str] = None) -> Fault:
f = Fault(at=int(self.clock()), rung=self.rung, reason=reason, ident=ident)
with self._lock:
self._recent.append(f)
self.counts[reason.split(':', 1)[0].strip()] += 1
return f
[docs] def recent(self, since: Optional[int] = None, family: Optional[str] = None) -> list[Fault]:
"""Newest last. `since` keeps faults at or after that clock reading;
`family` keeps the reason prefix ('publish refused')."""
with self._lock:
rows = list(self._recent)
return [f for f in rows
if (since is None or f['at'] >= since)
and (family is None or f['reason'].split(':', 1)[0].strip() == family)]
@property
def maxlen(self) -> Optional[int]:
return self._recent.maxlen
def __len__(self) -> int:
return len(self._recent)
def __iter__(self) -> Iterable[Fault]:
with self._lock:
return iter(list(self._recent))
[docs]def merge_faults(logs: Iterable[FaultLog], since: Optional[int] = None) -> list[Fault]:
"""Faults across rungs in time order, for the ladder's aggregate view."""
out: list[Fault] = []
for lg in logs:
out.extend(lg.recent(since))
out.sort(key=lambda f: f['at'])
return out
[docs]class CircuitBreaker:
"""Per-target circuit breaker to avoid hanging on repeatedly failing transports."""
STATE_CLOSED: str = 'closed'
STATE_OPEN: str = 'open'
STATE_HALF_OPEN: str = 'half_open'
[docs] def __init__(self, failure_threshold: int = 3, recovery_timeout: float = 30.0) -> None:
self.failure_threshold: int = failure_threshold
self.recovery_timeout: float = recovery_timeout
self._lock: threading.RLock = threading.RLock()
self._failures: collections.Counter[str] = collections.Counter()
self._last_failure: Dict[str, float] = {}
self._states: Dict[str, str] = {}
[docs] def allow_attempt(self, target: str, now: Optional[float] = None) -> bool:
"""Return True if an attempt to reach target is permitted."""
with self._lock:
t_now = time.time() if now is None else now
state = self._states.get(target, self.STATE_CLOSED)
if state == self.STATE_CLOSED:
return True
last = self._last_failure.get(target, 0.0)
if t_now - last >= self.recovery_timeout:
self._states[target] = self.STATE_HALF_OPEN
return True
return False
[docs] def record_success(self, target: str) -> None:
"""Record successful transmission, resetting circuit to CLOSED."""
with self._lock:
self._failures[target] = 0
self._states[target] = self.STATE_CLOSED
[docs] def record_failure(self, target: str, now: Optional[float] = None) -> None:
"""Record transmission failure, potentially tripping circuit to OPEN."""
with self._lock:
t_now = time.time() if now is None else now
self._last_failure[target] = t_now
self._failures[target] += 1
if self._failures[target] >= self.failure_threshold:
self._states[target] = self.STATE_OPEN
[docs] def state(self, target: str) -> str:
"""Current state for target."""
with self._lock:
return self._states.get(target, self.STATE_CLOSED)
[docs] def failures(self, target: str) -> int:
"""Return current consecutive failure count for target."""
with self._lock:
return self._failures[target]
[docs] def reset(self, target: Optional[str] = None) -> None:
"""Reset circuit breaker for a target or all targets."""
with self._lock:
if target is None:
self._failures.clear()
self._last_failure.clear()
self._states.clear()
else:
self._failures.pop(target, None)
self._last_failure.pop(target, None)
self._states.pop(target, None)
__all__ = ['CircuitBreaker', 'Fault', 'FaultLog', 'merge_faults']