Source code for fleetmesh_node.planes.identity.presence.compartment

"""What this node says it is alive, and what it makes of a peer saying so.

Three things that are one thing: the descriptor this node publishes, the
heartbeat that says the descriptor is still true, and the subscription that
reads a peer's. All three ran loose in the node's tick, and the pair of them
that can find a fault - a heartbeat whose class contradicts the peer's live
descriptor, one whose endpoint digest says the descriptor has gone stale - fed
a finding into the budget and recorded nothing an operator could read back.
"""
from __future__ import annotations

import threading
import time
from typing import Any, Callable, Optional, Sequence

from fleetmesh_core.crypto import giftwrap
from fleetmesh_core.identity import relaylist
from fleetmesh_core.services import lifecycle, telemetry
from fleetmesh_core.storage import store as ref_store
from fleetmesh_core.wire.interfaces import NostrEvent
from fleetmesh_node.channels import journal
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.protocols.nostr import client as relay
import dataclasses
from fleetmesh_node.config.parameter import Parameter, ParamBlock
from fleetmesh_node.channels import Channel, NOTE, INFO
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Every, Need, Worker
from .types import PresenceStateReport

# One kind 21802 per window. The counters are the transport's and the
# strain ledger's totals, binned by fleetmesh_core/telemetry so nothing finer than
# an order of magnitude leaves the node and no peer is named.
TELEMETRY_WINDOW: int = 3600


[docs]@dataclasses.dataclass(frozen=True) class PresenceConfig(ParamBlock): key = 'presence' name = 'presence' DERIVED: bool = False PARAMS = ( Parameter( key='heartbeat', name='Heartbeat Enabled', description='Whether to publish periodic heartbeat events', type=bool, default=True, tags=('presence', 'heartbeat'), ), Parameter( key='telemetry_window', name='Telemetry Window', description='Window size in seconds for telemetry aggregation', type=int, default=3600, minimum=60, tags=('presence', 'telemetry'), ), ) heartbeat: bool = True telemetry_window: int = 3600
[docs]class Presence(Compartment): requires: tuple[str, ...] = ('budget', 'strain') # strain totals are one of the five telemetry metrics name: str = 'presence' # `heartbeat` and `class` are read every tick but neither is live: a node's # class is what the node IS, and a heartbeat switched off under a running # node would take its liveness off the mesh without anything on the mesh # being told. Both change on a restart. reads: tuple[str, ...] = ('heartbeat', 'class', 'telemetry') config_cls = PresenceConfig NEEDS = ( Need('keystore', 'ks'), Need('peers'), Need('grant_store', 'stores.grants'), Need('relays'), Need('budget'), Need('stop'), Need('timeout'), Need('strain'), Need('descriptor'), Need('public_descriptor'), Need('rebuild_descriptor', 'build_descriptor'), Need('transport'), ) TELEMETRY_WINDOW: int = TELEMETRY_WINDOW CHANNELS = ( Channel('peer_retiring', 'Peer indicated graceful retirement', level=NOTE, say='peer {peer} says it is retiring; the partition that follows is not stress from that direction'), Channel('inbound_event', 'Inbound heartbeat event processed', level=INFO, say='< {event}'), ) METRICS = ( Metric(name='presence_peers_retiring', description='Times peers reported retirement.', kind=COUNTER, counts='peer_retiring', tags=('presence', 'peers')), Metric(name='presence_events_inbound', description='Times heartbeat events were processed.', kind=COUNTER, counts='inbound_event', tags=('presence', 'heartbeat')), ) EVERY = (Every(seconds=TELEMETRY_WINDOW, run='_run_telemetry'),) WORKERS = (Worker(run='_listen', name='presence-listen'),) STORES = ( st.Store( key='_seen', description='peer -> the id of the last heartbeat read from it', bound=1024, evict=st.LRU, kind=st.TABLE, keyed_by='peer', ), st.Store( key='_retiring', description='peers that said they were going, and when', bound=1024, evict=st.LRU, kind=st.TABLE, keyed_by='peer', ), ) COMMANDS = ( Command( key='show', name='Presence', description='the descriptor this node publishes, when it expires, and the peers it ' 'has heard from', tags=('presence', 'status'), permission='presence/show', run='state', reply='{descriptor, expires, renews_at, last_heartbeat, inbox_relays, heard_from, ' 'retiring}', ), )
[docs] def __init__( self, keystore: Any, peers: Any, grant_store: Any, relays: Sequence[str], budget: Optional[Any] = None, stop: Optional[threading.Event] = None, clock: Optional[Callable[[], int]] = None, timeout: float = 15.0, node: Optional[Any] = None, strain: Optional[Any] = None, descriptor: Optional[Callable[[], Optional[NostrEvent]]] = None, public_descriptor: Optional[Callable[[], Optional[NostrEvent]]] = None, rebuild_descriptor: Optional[Callable[[], NostrEvent]] = None, transport: Optional[Any] = None, publisher: Optional[Callable[[Sequence[str], Any], Any]] = None, ) -> None: super().__init__(node, clock) # How a heartbeat, telemetry or renewed descriptor reaches the relays; # given so a suite can record them without a socket. self.publisher: Callable[[Sequence[str], Any], Any] = publisher or relay.publish_all # The transport whose nostr-rung counters the telemetry event reports. self.transport: Optional[Any] = transport if transport is not None else getattr(node, 'transport', None) # The node's current descriptor, and the one call that rebuilds it # from what the compartments say now. Suites that pass a fake node # get both off it. self.descriptor: Callable[[], Optional[NostrEvent]] = descriptor or (lambda: getattr(node, 'descriptor', None)) # What goes to the public relays, or None for a fully private node # (plan D2). A node with no public copy says nothing there: no relay # list, no renewed descriptor, no heartbeat, no telemetry. A suite that # hands in only a descriptor gets that as the public copy. self.public_descriptor: Callable[[], Optional[NostrEvent]] = public_descriptor or self.descriptor self.rebuild_descriptor: Callable[[], NostrEvent] = rebuild_descriptor or (lambda: getattr(node, 'build_descriptor')()) self.ks: Any = keystore self.peers: Any = peers self.grants: Any = grant_store self.relays: list[str] = list(relays) # not a store: the configured relays self.budget: Optional[Any] = budget self.strain: Optional[Any] = strain self.timeout: float = timeout self._stop: threading.Event = stop or getattr(self, '_stop_event', None) or threading.Event() self._last_heartbeat: Optional[int] = None self._last_telemetry: Optional[int] = None # The relay list this node last published (kind 10050): the relays it # reads, as a tuple, so a changed set is published once and an # unchanged one never again. self._inbox_said: Optional[tuple[str, ...]] = None # The event itself, for whoever projects the floor (the linked face). self.last_inbox: Optional[NostrEvent] = None self._seen: st.Table[str, str] # peer -> the id of the last heartbeat read from it # Peers that said they were going, and when. A partition from one of # these is not stress: this node was told. Cleared when the peer is # heard from again without the flag, because that is a node that came # back and is expecting to be counted like any other. self._retiring: st.Table[str, int]
# ---- what this node says -------------------------------------------------- def _renews_at(self) -> tuple[int, int]: """A descriptor is renewed inside the last tenth of its life.""" desc = self.descriptor() created_at = desc.get('created_at', self.clock()) if desc else self.clock() exp = None if desc: for t in desc.get('tags', []): if isinstance(t, (list, tuple)) and len(t) > 1 and t[0] == 'expiration': try: exp = int(t[1]) break except (ValueError, TypeError): pass if exp is None: exp = created_at + 86400 return exp, exp - (exp - created_at) // 10
[docs] def publish_inbox(self, now: int) -> Optional[NostrEvent]: """The kind 10050 that states this node's relay floor: the relays it reads are where a stored wrap for it goes (NIP-17). Every node publishes one - the relay rung is the one way a node can always be reached, and this is where that is said, in the place any Nostr client already looks. Published on the first tick and again only when the set changes; None when nothing was due. Plan D2: a fully private node says it on its network relays instead of the public relays - relay_destination() is the node's answer to which - so its members still know where to reach it while the public relays learn nothing. A fully private node with no network has nowhere to say it and publishes none.""" urls = self.relay_destination(self.relays) if not urls: return None now_set = tuple(urls) if self._inbox_said == now_set: return None tags = relaylist.build(self.relays) ev: NostrEvent = ref_store.make_signed_event(self.ks.node_sk, now, relaylist.KIND, tags, '') self.publisher(urls, ev) self._inbox_said = now_set self.last_inbox = ev return ev
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: did: list[tuple[Any, ...]] = [] inbox = self.publish_inbox(now) if inbox is not None: did.append(('inbox relays published', inbox['id'])) exp, renew = self._renews_at() if now >= renew: self.rebuild_descriptor() public = self.public_descriptor() if public is not None: if self.relays: self.publisher(self.relays, public) did.append(('descriptor renewed', public['id'])) # anchors must, edges should, leaves never if self.config('heartbeat', True) and lifecycle.heartbeat_due( self.config('class'), self._last_heartbeat, now): did.append(self.heartbeat_now(now)) did += self.run_every(now) return did
[docs] def heartbeat_now(self, now: int) -> tuple[Any, ...]: """Sign and publish one heartbeat now, off the current descriptor's endpoints, and count it as the last. `tick` calls it when one is due; `enrol` calls it so the fleet sees the node under its operator at once rather than at the next due beat. Plan D2: a heartbeat is this node's liveness, so a fully private node still makes one - but to its network relays (relay_destination), where the members that share a private network with it listen, never to the public ones. With no public copy the endpoints come from the full descriptor, which is what a member resolves. A fully private node with no network has nowhere to say it and publishes none, which `enrol` reads as "nothing was published". A public (or narrowed) node publishes to its configured relays exactly as before.""" cur = self.public_descriptor() urls = self.relay_destination(self.relays) if cur is None: # Fully private: the heartbeat is still made, on the network relays. if not urls: # Nothing to say to nobody; counted as done so the due check # does not wake this every tick. self._last_heartbeat = now return ('heartbeat', None) desc = self.descriptor() else: desc = cur eps = [t for t in desc['tags'] if t[0] == 'endpoint'] if desc is not None else [] hb = lifecycle.heartbeat(self.ks.node_pk, self.config('class'), eps, now) signed_hb = ref_store.make_signed_event(self.ks.node_sk, now, hb['kind'], hb['tags'], hb['content']) if urls: self.publisher(urls, signed_hb) self._last_heartbeat = now return ('heartbeat', signed_hb['id'])
_tick_runs_every = True def _run_telemetry(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: now = now if now is not None else self.clock() if not (self.config('telemetry', False) and self.relays) or self.public_descriptor() is None: return [] tm = self._telemetry(now) signed_tm = ref_store.make_signed_event(self.ks.node_sk, now, tm['kind'], tm['tags'], tm['content']) self.publisher(self.relays, signed_tm) self._last_telemetry = now return [('telemetry', signed_tm['id'])] # One kind 21802 per window. The counters are the transport's and the # strain ledger's totals, binned by fleetmesh_core/telemetry so nothing finer than # an order of magnitude leaves the node and no peer is named. def _telemetry(self, now: int) -> Any: tr = getattr(self.transport, 'nostr', None) if self.transport is not None else None counters: dict[str, int] = {'peers': len(self.grants.peers())} if tr is not None: counters['wraps_in'] = sum(getattr(tr, 'wraps_in', {}).values()) counters['wraps_seen_again'] = sum(getattr(tr, 'wraps_seen_again', {}).values()) counters['wraps_dropped'] = len(getattr(tr, 'dropped', ())) if self.strain is not None: counters['strain'] = sum(self.strain.by_direction().values()) return telemetry.telemetry(self.ks.node_pk, now, self.TELEMETRY_WINDOW, counters, expiration=now + 2 * self.TELEMETRY_WINDOW)
[docs] def announce_retiring(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: """Say this node is going, before it starts giving things away. Published directly rather than through the tick's due check: a retirement does not wait for the hour to come round, and the whole value of the flag is that it arrives before the silence it explains. A leaf publishes no heartbeat and so cannot announce one, which is correct - a leaf claims nothing and hands nothing over. Ephemeral, like every heartbeat: only a peer listening now hears it. That is the right answer and it is why it is advisory. A peer that was itself down records the partition, and the partition is true. """ now_val = now if now is not None else self.clock() urls = self.relay_destination(self.relays) if self.config('class') == 'leaf' or not urls or not self.config('heartbeat', True): return [] desc = self.public_descriptor() or self.descriptor() eps = [t for t in desc['tags'] if t[0] == 'endpoint'] if desc is not None else [] hb = lifecycle.heartbeat(self.ks.node_pk, self.config('class'), eps, now_val, load=1.0, retiring=True) signed = ref_store.make_signed_event(self.ks.node_sk, now_val, hb['kind'], hb['tags'], hb['content']) self.publisher(urls, signed) self._last_heartbeat = now_val return [('retiring announced', signed['id'])]
[docs] def is_retiring(self, peer_pk: str) -> bool: """Did this peer tell us it was going? Read by anything that would otherwise price the silence that follows.""" return peer_pk in self._retiring
# ---- what a peer says -----------------------------------------------------
[docs] def on_heartbeat(self, hb: Any) -> list[tuple[Any, ...]]: """One kind 21801 from a peer, checked against what that peer published. A node published one of these every hour and nothing ever read one, so two rules the reference already implements were dead. A heartbeat whose class contradicts the peer's live descriptor is a misreport about itself and an S2. One whose endpoint digest differs from the descriptor's says the descriptor has gone stale, which is the whole reason that digest is in there: a peer can tell without dialling every address it was given. A heartbeat is *ephemeral*, so no relay stores one and none can be asked for after the fact. Liveness is a statement about now, and the only way to hear it is to be listening when it is made - which is why this is fed by a subscription and not by a query. A node that was down misses them, and that is the right answer. """ did: list[tuple[Any, ...]] = [] # A subscription by author returns whatever a relay chooses to serve # under that name. Believing it unsigned would let a relay file an S2 # against any peer this node holds a grant with. if not isinstance(hb, dict) or not giftwrap.verify_event(hb) or hb.get('kind') != lifecycle.HEARTBEAT: self.refusals.refuse('heartbeat', 'not a signed kind 21801', subject=(hb or {}).get('pubkey') if isinstance(hb, dict) else None) return did pk = hb.get('pubkey') if not isinstance(pk, str) or pk not in set(self.grants.peers()): self.refusals.refuse('heartbeat', 'from a node this one holds no grant with', subject=pk) return did hb_id = hb.get('id') if not isinstance(hb_id, str) or self._seen.get(pk) == hb_id: return did self._seen[pk] = hb_id desc = self.peers.descriptor(pk) if desc is None: self.refusals.refuse('heartbeat', 'the peer publishes no descriptor to check it against', subject=pk) return did problems, findings = lifecycle.check_heartbeat(hb, desc) for severity, why in findings: if self.budget is not None: self.budget.findings.append((pk, None, severity, 'heartbeat: %s' % why)) self.refusals.refuse('heartbeat', 'contradicts the descriptor the peer publishes', subject=pk, detail={'severity': severity, 'what': why}) did.append(('heartbeat contradicts its descriptor', pk, severity)) if lifecycle.is_retiring(hb): # Not a finding and not a fault. The peer is telling us the silence # that follows is a departure, so nothing downstream prices it. if pk not in self._retiring: self._retiring[pk] = self.clock() self.say('peer_retiring', peer=pk[:16]) did.append(('peer is retiring', pk)) elif self._retiring.pop(pk, None) is not None: did.append(('peer came back', pk)) if problems: self.refusals.refuse('heartbeat', 'malformed', subject=pk, detail={'problems': problems[:2]}) did.append(('heartbeat is malformed', pk, '; '.join(problems[:2]))) elif lifecycle.descriptor_is_stale(hb, desc): self.peers.descriptor(pk, refresh=True) did.append(('descriptor was stale, refreshed', pk)) return did
def _listen(self) -> None: """One subscription per relay for the liveness of peers we hold grants with.""" while not self._stop.is_set(): peers_held = self.grants.peers() if not peers_held or not self.relays: self._stop.wait(30) continue r = relay.Relay(self.relays[0], timeout=5.0) try: sid = r.connect().subscribe([{'kinds': [lifecycle.HEARTBEAT], 'authors': peers_held}]) started, watching = time.time(), set(peers_held) for hb in r.events(sid): if self._stop.is_set(): break for d in self.on_heartbeat(hb): self.say('inbound_event', event=' '.join(journal.short(x) for x in d)) # re-subscribe when the peer set has moved on if time.time() - started > 300 and set(self.grants.peers()) != watching: break except Exception as e: self.refusals.refuse('subscription', 'could not be held open', subject=self.relays[0], detail={'error': str(e)}) finally: r.close() self._stop.wait(5) # ---- being watched --------------------------------------------------------
[docs] def state(self) -> PresenceStateReport: desc = self.descriptor() exp, renew = self._renews_at() if desc else (None, None) return { 'descriptor': desc['id'] if desc else None, 'expires': exp, 'renews_at': renew, 'last_heartbeat': self._last_heartbeat, 'inbox_relays': list(self._inbox_said or ()), 'heard_from': sorted(pk[:16] for pk in self._seen), 'retiring': sorted(pk[:16] for pk in self._retiring), }
[docs] def trouble(self) -> list[str]: out: list[str] = [] wrong = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='heartbeat', why='contradicts the descriptor the peer publishes') if wrong: out.append('%d heartbeat(s) contradict the descriptor their peer publishes: %s. ' 'Each is a finding against that peer' % (len(wrong), ', '.join(sorted({(r.subject or '')[:16] for r in wrong})))) return out