Source code for fleetmesh_node.planes.services.gossip.core

"""GossipMesh: the gossip/1.0 compartment.

A true epidemic fanout for group topics, independent of any single relay.
Membership in a topic is explicit: an admin-signed kind 30804 (addressable,
republish-supersedes) naming every member pubkey, not ambient trust. A node
only grafts into, or accepts a graft for, a topic it holds a current
membership event naming its own pubkey for - and only toward a peer it
already shares a peering grant with, so this invents no new authorization
and rides the budget/grant model everyone else already accounts against.

The payload gossiped is the NIP-59 seal+wrap this node would otherwise have
sent over the 1:1 transport-ladder path - one sealed envelope per recipient,
exactly as DMs ship today - so gossip never sees plaintext: it floods
already-encrypted bytes keyed by the wrap's own event id. 1:1 DMs are
untouched; this is purely an additional fanout substrate for group topics of
3+ members, which is where a single relay going down costs everyone their
only path.

Loop prevention is strict: a seen-id store (sqlite, TTL + size bounded) keyed
(topic, event_id) is checked before flooding, and an id is never re-sent back
toward the peer it arrived from. graft/prune build and prune a bounded mesh
per topic, degree DEFAULT_MESH_DEGREE (gossipsub's D), maintained by tick().

ihave/iwant is the lazy half (gossipsub's "gossip" proper): every tick, each
topic's recent event ids are offered to a few members *outside* this node's
mesh, and a member that is missing one asks for it by id and is sent the
sealed envelope directly. That is what carries a publish across a mesh
that is partitioned - a member whose grafts all failed, a topic larger than
D where two cliques formed, a node that was down for the flood - without
any relay in the path. Envelopes to answer an iwant with come from a short
in-memory message cache; the durable seen store stays the only thing loop
prevention relies on.
"""
from __future__ import annotations

import threading
import time
from dataclasses import dataclass, field
from typing import Any, Callable, Mapping, Optional, Union

from fleetmesh_node import channels as ch_mod
from fleetmesh_node.storage import grants as grants_mod
from fleetmesh_node import metrics as mx_mod
from fleetmesh_node.protocols.nostr import client as relay_mod
from fleetmesh_core.crypto import giftwrap
from fleetmesh_core.wire.interfaces import NostrEvent
from fleetmesh_node.config.compartments import GossipConfig
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.compartment import Command, Every, Handler, Need, Outbound, WireProtocol, Worker
from fleetmesh_node.compartment import stores as st
from .envelope import (DEFAULT_MESH_DEGREE, GossipError, MEMBERSHIP_KIND, PIURI,
                       make_graft, make_ihave, make_iwant, make_prune, make_publish)
from .mesh import RosterStore, SeenStore

REGRAFT_INTERVAL_SECONDS: int = 300
SEEN_SWEEP_INTERVAL_SECONDS: int = 3600
# Lazy gossip (gossipsub's mcache / D_lazy / history window). Envelopes are
# kept just long enough to answer an iwant for an id offered in an ihave; ids
# are offered to a few non-mesh members per tick, never to the mesh itself,
# which already had the full envelope flooded to it.
MCACHE_SECONDS: int = 120
MCACHE_MAX_PER_TOPIC: int = 512
GOSSIP_LAZY_DEGREE: int = 6
IHAVE_MAX_IDS: int = 256
# The message cache across every topic. MCACHE_MAX_PER_TOPIC still caps one
# topic; this caps the sum, so a node carrying many busy topics holds a
# bounded number of envelopes. It is a cache for lazy gossip and nothing
# depends on a hit: an id that fell out is heard from another member.
MCACHE_MAX: int = MCACHE_MAX_PER_TOPIC * 16
# How many of each kind of gossip event the compartment keeps to look at. The
# totals are counted separately (GossipTotals), so this bounds the memory and
# not the numbers an operator reads.
RECENT: int = 256


@dataclass(slots=True)
class Roster:
    """A kind 30804 membership event this node holds for a topic, and the
    member set read from its tags. The event is kept as it arrived - it is
    what graft names, what `topics` reports and what RosterStore persists."""
    event: Mapping[str, Any]
    members: set[str] = field(default_factory=set)


from .types import (
    GossipTotals,
    TopicView,
    TopicCreationResult,
    OriginateResult,
    GossipStateReport,
)


def _recent(key: str, what: str) -> st.Store:
    return st.Store(key=key, description='%s, newest last' % what, kind=st.RING, shape=tuple, bound=RECENT)


[docs]class GossipMesh(WireProtocol): """The group-topic gossip/1.0 compartment: graft/prune mesh + flood publish.""" name: str = 'gossip' piuris: tuple[str, ...] = (PIURI,) optional: bool = True enabled_by: Optional[str] = 'gossip' # Gossip reads the grant store and the relay list, both context fields; # it never reaches the peering or budget compartments. requires: tuple[str, ...] = () reads: tuple[str, ...] = ('gossip',) config_cls: Any = GossipConfig HANDLERS = ( Handler('graft', run='wire_graft', doc='receive gossip mesh graft'), Handler('prune', run='wire_prune', doc='receive gossip mesh prune'), Handler('publish', run='wire_publish', doc='receive epidemic flood publish'), Handler('ihave', run='wire_ihave', doc='receive lazy gossip ihave offer'), Handler('iwant', run='wire_iwant', doc='receive lazy gossip iwant request'), ) OUTBOUND = ( Outbound('graft', reply=('graft', 'prune'), doc='request to join a topic mesh'), Outbound('prune', doc='leave a topic mesh or prune a peer'), Outbound('publish', doc='epidemic flood of sealed message to topic mesh'), Outbound('ihave', reply='iwant', doc='lazy gossip offer of recently seen message ids'), Outbound('iwant', reply='publish', doc='request message payloads for offered ids'), ) COMMANDS = ( Command( key='show', name='Gossip Status', description='topics this node gossips, mesh size per topic, and seen-store size', tags=('gossip', 'status'), permission='gossip/show', run='state', reply='{topics, mesh, seen}', ), Command( key='topics', name='Gossip Topics', description='every roster held: admin, members, mesh peers, expiry', tags=('gossip', 'topics'), permission='gossip/topics', run='topics_cmd', reply='[{topic, admin, members, mesh, expires, roster_id}]', ), Command( key='create', name='Create Topic', description='sign a kind 30804 roster for <node_pk>:<name>, admit it, publish it to the relays', tags=('gossip', 'create'), parameters=( Parameter(key='name', name='name', description='topic name', type=str, default='', required=True), Parameter(key='members', name='members', description='initial member pubkeys', type=list, default=list, required=False), Parameter(key='ttl_seconds', name='ttl_seconds', description='roster time-to-live in seconds', type=int, default=30 * 86400, required=False), Parameter(key='no_relay', name='no_relay', description='do not publish to relays', type=bool, default=False, required=False), ), permission='gossip/create', run='create_cmd', reply='{topic, event, relays}', ), Command( key='admit', name='Admit Topic', description='admit a kind 30804 roster another admin signed that names this node', tags=('gossip', 'admit'), parameters=( Parameter(key='event', name='event', description='kind 30804 roster event', type=dict, default=dict, required=True), ), permission='gossip/admit', run='admit_cmd', reply='{topic}', ), Command( key='leave', name='Leave Topic', description='drop a roster and prune every mesh peer on it', tags=('gossip', 'leave'), parameters=( Parameter(key='topic', name='topic', description='topic to leave', type=str, default='', required=True), ), permission='gossip/leave', run='leave_cmd', reply='{topic, pruned}', ), Command( key='publish', name='Publish Message', description='flood a sealed NIP-59 wrap into a topic mesh', tags=('gossip', 'publish'), parameters=( Parameter(key='topic', name='topic', description='target topic id', type=str, default='', required=True), Parameter(key='envelope', name='envelope', description='sealed NIP-59 wrap event', type=dict, default=dict, required=True), ), permission='gossip/publish', run='publish_cmd', reply='{topic, event_id, sent_to, mesh}', ), ) # What it keeps. Each of the six history lists below used to be a plain # list appended once per gossip message and never trimmed - memory that # grew with whatever peers sent - while nothing in the node read more # than its length. Now: a ring of the recent ones, and the length kept # as a counter in `totals`, so `state` reports the same numbers. STORES = ( # REFUSE: dropping a roster would silently leave a topic. create() and # admit() refuse at the configured max_topics first, with a reason. st.Store(key='members', description='the roster held for each topic', kind=st.TABLE, shape=Roster, bound=GossipConfig.TOPIC_CEILING, evict=st.REFUSE, keyed_by='topic'), # Derived from members and rebuilt by tick's regraft, so OLDEST is a # safe second line: a topic dropped here is regrafted. st.Store(key='peers', description="the pubkeys in this node's mesh, per topic", kind=st.TABLE, shape=set, bound=GossipConfig.TOPIC_CEILING, keyed_by='topic'), st.Store(key='grafted_at', description='when each mesh peer was grafted, per topic', kind=st.TABLE, shape=int, bound=GossipConfig.TOPIC_CEILING * GossipConfig.MESH_DEGREE_CEILING, keyed_by='topic and peer'), st.Store(key='mcache', description='sealed envelopes kept to answer an iwant, with when they were cached', kind=st.TABLE, shape=tuple, bound=MCACHE_MAX, keyed_by='topic and event'), st.Store(key='totals', description='gossip events this node took part in, since it started', kind=st.COUNTERS, shape=GossipTotals), _recent('published', 'events this node originated, as (topic, event id)'), _recent('delivered', 'events delivered from a mesh peer, as (topic, event id, from)'), _recent('offered', 'ihave offers sent, as (topic, peer, ids)'), _recent('wanted', 'iwant requests sent, as (topic, peer, ids)'), _recent('served', 'iwant answers sent, as (topic, event id, to)'), _recent('auto_admitted', 'rosters admitted from a relay, as (topic, roster id)'), ) CHANNELS = ( ch_mod.Channel('event_published', 'A gossip event was originated and flooded to the topic mesh', level=ch_mod.INFO, say='published gossip event {event_id} to topic {topic}', tags=('gossip', 'publish')), ch_mod.Channel('event_delivered', 'A gossip event was delivered from a mesh peer', level=ch_mod.INFO, say='delivered gossip event {event_id} on topic {topic} from {peer}', tags=('gossip', 'deliver')), ch_mod.Channel('graft_established', "A peer was grafted into this node's topic mesh", level=ch_mod.INFO, say='grafted peer {peer} on topic {topic}', tags=('gossip', 'mesh')), ch_mod.Channel('prune_executed', "A peer was pruned from this node's topic mesh", level=ch_mod.NOTE, say='pruned peer {peer} on topic {topic} (reason: {reason})', tags=('gossip', 'mesh')), ) METRICS = ( mx_mod.Metric(name='gossip_events_published', description='Events originated and flooded to topic mesh.', kind=mx_mod.COUNTER, counts='event_published', tags=('gossip', 'publish')), mx_mod.Metric(name='gossip_events_delivered', description='Events received from mesh peers and delivered.', kind=mx_mod.COUNTER, counts='event_delivered', tags=('gossip', 'deliver')), mx_mod.Metric(name='gossip_peers_grafted', description='Peers added to local gossip mesh.', kind=mx_mod.COUNTER, counts='graft_established', tags=('gossip', 'mesh')), mx_mod.Metric(name='gossip_peers_pruned', description='Peers removed from local gossip mesh.', kind=mx_mod.COUNTER, counts='prune_executed', tags=('gossip', 'mesh')), ) NEEDS = ( Need('keystore', 'ks'), Need('transport', 'transport'), Need('grant_store', 'stores.grants'), Need('cfg', block=GossipConfig), Need('seen_path', path='gossip_seen.sqlite'), Need('roster_path', path='gossip_rosters.sqlite'), Need('relays', 'relays'), ) EVERY = (Every(seconds=SEEN_SWEEP_INTERVAL_SECONDS, run='expire_seen'),) WORKERS = ( Worker(run='_listen', name='gossip-rosters', when='relays'), )
[docs] def __init__( self, keystore: Any, transport: Any, grant_store: grants_mod.GrantStore, clock: Optional[Callable[[], int]] = None, node: Optional[Any] = None, mesh_degree: int = DEFAULT_MESH_DEGREE, seen_ttl_seconds: int = 24 * 3600, max_topics: int = 64, seen_path: str = ':memory:', roster_path: Optional[str] = None, relays: Optional[list[str]] = None, cfg: Optional[Union[GossipConfig, Mapping[str, Any]]] = None, ) -> None: super().__init__(node, clock) self.ks: Any = keystore self.transport: Any = transport self.grants: grants_mod.GrantStore = grant_store if cfg is not None: if isinstance(cfg, GossipConfig): mesh_degree = cfg.mesh_degree seen_ttl_seconds = cfg.seen_ttl_seconds max_topics = cfg.max_topics elif isinstance(cfg, Mapping): mesh_degree = int(cfg.get('mesh_degree', mesh_degree)) seen_ttl_seconds = int(cfg.get('seen_ttl_seconds', seen_ttl_seconds)) max_topics = int(cfg.get('max_topics', max_topics)) self.mesh_degree: int = mesh_degree self.seen_ttl_seconds: int = seen_ttl_seconds self.max_topics: int = max_topics self.seen: SeenStore = SeenStore(seen_path, ttl_seconds=seen_ttl_seconds) self.rosters: RosterStore = RosterStore(roster_path or seen_path) self.relays: list[str] = list(relays or []) # not a store: the configured relays self._lock: threading.RLock = threading.RLock() self._last_seen_sweep: int = 0 self._closed: bool = False self._seen_count: int = 0 self._stop: threading.Event = getattr(self, '_stop_event', None) or threading.Event() self._wake: threading.Event = threading.Event() # set by tick when the granted set moves self._watching: set[str] = set() # not a store: a copy of grants.peers(), replaced whole # The stores were built and bound by the base (see STORES). self.members: st.Table[str, Roster] self.peers: st.Table[str, set[str]] self.grafted_at: st.Table[tuple[str, str], int] self.mcache: st.Table[tuple[str, str], tuple[Mapping[str, Any], int]] self.totals: st.Counters self.published: st.Ring[tuple[str, str]] self.delivered: st.Ring[tuple[str, str, str]] self.offered: st.Ring[tuple[str, str, int]] self.wanted: st.Ring[tuple[str, str, int]] self.served: st.Ring[tuple[str, str, str]] self.auto_admitted: st.Ring[tuple[str, str]] self._reload_rosters()
def _record(self, ring: 'st.Ring[Any]', entry: tuple[Any, ...]) -> None: """One gossip event: kept in its ring, and counted in `totals`.""" ring.append(entry) self.totals.incr(ring.store.key) if ring.store.key == 'published' and len(entry) >= 2: self.say('event_published', topic=entry[0], event_id=entry[1]) elif ring.store.key == 'delivered' and len(entry) >= 3: self.say('event_delivered', topic=entry[0], event_id=entry[1], peer=entry[2]) # ---- learning rosters from the relays ------------------------------------- def _listen(self) -> None: while not self._stop.is_set(): admins = sorted(self.grants.peers()) self._watching = set(admins) # not a store: a copy of grants.peers(), replaced whole self._wake.clear() if not admins: self._wait_either(30) continue # A short socket timeout is how the loop notices _wake between frames. r = relay_mod.Relay(self.relays[0], timeout=1.0) try: # NIP-01 indexes single-letter tags only, so `member` cannot be # filtered at the relay; admit() drops rosters not naming us. sid = r.connect().subscribe([{'kinds': [MEMBERSHIP_KIND], 'authors': admins}]) for ev in r.events(sid, until=None): if self._stop.is_set() or self._wake.is_set(): break self.on_roster(ev) if self._wake.is_set(): continue # re-subscribe now with the new author set except Exception as e: self.refusals.refuse('subscription', 'could not be held open', subject=self.relays[0], detail={'error': str(e)}) finally: r.close() self._wait_either(5) def _wait_either(self, seconds: float) -> None: """Sleep until `seconds` pass, the node stops, or the granted set moves. Waits on `_wake`, which stop() sets too, in slices of a second so a stop that reaches only `_stop` is still seen within one. It polled every 0.2 s on the wall clock (audit 2026-10-15, E8). """ deadline = time.monotonic() + seconds while not self._stop.is_set() and not self._wake.is_set(): left = deadline - time.monotonic() if left <= 0: return self._wake.wait(min(1.0, left))
[docs] def on_roster(self, event: Mapping[str, Any]) -> Optional[str]: """A roster read from a relay: admit it if it names this node and is from a granted peer; the topic admitted, or None (refusal recorded).""" if not isinstance(event, Mapping) or event.get('kind') != MEMBERSHIP_KIND: return None event = dict(event) admin = event.get('pubkey', '') if not self._has_grant(admin): self.refusals.refuse('membership', 'roster from a peer with no grant', subject=admin) return None try: topic = self.admit(event) except GossipError: return None # admit() recorded why, or it was not for us self._record(self.auto_admitted, (topic, event.get('id', ''))) return topic
def _reload_rosters(self) -> None: """Rosters held before the last stop. One that has expired meanwhile is dropped here rather than refused on every tick.""" for topic, event in self.rosters.all().items(): members = [t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'member'] if self.check_membership(topic, event, members) is None: if not self.members.put(topic, Roster(event, set(members))): # More rosters on disk than the topic ceiling - written # by a build with a higher one. Kept on disk, not held. self.refusals.refuse('membership', 'over the topic ceiling; not reloaded', subject=topic) else: self.rosters.delete(topic) # ---- membership -----------------------------------------------------------
[docs] def admit_membership(self, topic: str, event: Mapping[str, Any], member_pubkeys: list[str]) -> None: """Record a current admin-signed kind 30804 membership event for a topic. A republish of the same `d` (topic) supersedes the one held, matching the addressable-event convention every other control event here uses. The roster is read from the event's own `member` tags; the caller's `member_pubkeys` is accepted only when it matches them, so a caller cannot admit a roster the admin did not sign. """ why = self.check_membership(topic, event, member_pubkeys) if why is not None: self.refusals.refuse('membership', why, subject=topic) raise GossipError(why) with self._lock: held = self.members.get(topic) if held and int(held.event.get('created_at', 0)) > int(event.get('created_at', 0)): self.refusals.refuse('membership', 'older than the roster held', subject=topic) raise GossipError('older than the roster held') self.members[topic] = Roster(event, set(member_pubkeys)) self.rosters.put(topic, event)
# ---- operator verbs: the control socket's side of the compartment ----------
[docs] def topic_id(self, name: str) -> str: """The coordinate of a topic this node administers: `<node_pk>:<name>`.""" if not name or ':' in name or len(name) > 64: raise GossipError('a topic name is 1-64 characters with no colon') return self.ks.node_pk + ':' + name
[docs] def make_roster(self, name: str, members: list[str], ttl_seconds: int = 30 * 86400) -> NostrEvent: """Sign a kind 30804 roster for a topic this node administers. This node is always on its own roster: an admin that is not a member would hold a topic it can never graft into. Signing is the slow pure- Python curve, so this is an operator action, not something on a tick. """ from fleetmesh_core.storage import store topic = self.topic_id(name) roster = sorted({self.ks.node_pk, *members}) if len(roster) < 2: raise GossipError('a roster names at least one member besides this node') if ttl_seconds <= 0: raise GossipError('ttl_seconds is positive') now = self.clock() tags = [['d', topic]] + [['member', m] for m in roster] + [['expiration', str(now + int(ttl_seconds))]] return store.make_signed_event(self.ks.node_sk, now, MEMBERSHIP_KIND, tags, '')
[docs] def create_topic(self, name: str, members: list[str], ttl_seconds: int = 30 * 86400, publish_to_relays: bool = True) -> TopicCreationResult: """Sign, admit, and (optionally) publish a roster. Returns {topic, event, relays}. Publishing to the configured relays is how the other members learn the roster before any of them has a gossip mesh to learn it over - the one place a relay is in the path, and only as a bulletin board. """ event = self.make_roster(name, members, ttl_seconds) topic = self.topic_id(name) if topic not in self.members and len(self.members) >= self.max_topics: raise GossipError('this node holds its maximum of %d topics' % self.max_topics) self.admit_membership(topic, event, sorted(self.members_of(event))) out: TopicCreationResult = {'topic': topic, 'event': event, 'relays': {}} # Plan D2: the roster is this node's own signed event; a fully private # node puts it on its network relays (relay_destination), not the public # ones, so it never reveals itself there. urls = self.relay_destination(self.relays) if publish_to_relays and urls: out['relays'] = {u: list(r) for u, r in relay_mod.publish_all(urls, event, 15.0).items()} return out
[docs] @staticmethod def members_of(event: Mapping[str, Any]) -> set[str]: return {t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'member'}
[docs] def admit(self, event: Mapping[str, Any]) -> str: """Admit a roster another admin signed (fetched from a relay, pasted by the operator). The topic is the event's own `d`; returns it.""" d = [t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'd'] if len(d) != 1: raise GossipError('roster has no single d tag') if self.ks.node_pk not in self.members_of(event): raise GossipError('this node is not named on that roster') if d[0] not in self.members and len(self.members) >= self.max_topics: raise GossipError('this node holds its maximum of %d topics' % self.max_topics) self.admit_membership(d[0], event, sorted(self.members_of(event))) return d[0]
[docs] def leave(self, topic: str) -> int: """Drop a roster and prune every mesh peer on it. Returns peers pruned.""" with self._lock: if topic not in self.members: raise GossipError('not a member of %s' % topic) peers = list(self.peers.get(topic, set())) for p in peers: self.prune(topic, p, reason='left') with self._lock: self.members.pop(topic, None) self.peers.pop(topic, None) self.rosters.delete(topic) return len(peers)
[docs] def originate(self, topic: str, envelope: Mapping[str, Any]) -> OriginateResult: """Flood an already-sealed NIP-59 wrap (kind 1059/13/21059) into a topic. The compartment carries ciphertext only; a plaintext kind here is a refusal, not a convenience, because the mesh is every member's peers. """ if not self._is_member(topic, self.ks.node_pk): raise GossipError('not a member of %s' % topic) if envelope.get('kind') not in (13, 1059, 21059): raise GossipError('only sealed NIP-59 envelopes (kind 13, 1059, 21059) travel on gossip') event_id = envelope.get('id') if not isinstance(event_id, str) or len(event_id) != 64: raise GossipError('envelope has no 64-hex id') if not giftwrap.verify_event(envelope): raise GossipError('envelope signature does not verify') sent = self.publish(topic, event_id, envelope) return {'topic': topic, 'event_id': event_id, 'sent_to': sent, 'mesh': len(self.peers.get(topic, set()))}
[docs] def check_membership(self, topic: str, event: Mapping[str, Any], member_pubkeys: list[str]) -> Optional[str]: """Why this event is not a current kind 30804 roster for `topic`, or None. Kind, signature, `d` coordinate scoped to the signing admin, a roster that is what the tags say, and a NIP-40 expiration still ahead - in that order, cheapest refusal first. """ if event.get('kind') != MEMBERSHIP_KIND: return 'not a kind %d' % MEMBERSHIP_KIND tags = event.get('tags') or [] d = [t[1] for t in tags if len(t) > 1 and t[0] == 'd'] if d != [topic]: return 'd tag does not name this topic' admin = event.get('pubkey', '') if not topic.startswith(admin + ':'): return 'topic coordinate is not scoped to the signing admin' members = {t[1] for t in tags if len(t) > 1 and t[0] == 'member'} if not members or members != set(member_pubkeys): return 'member tags do not match the roster offered' exp = [t[1] for t in tags if len(t) > 1 and t[0] == 'expiration'] if len(exp) != 1 or not exp[0].isdigit(): return 'no single expiration tag' if int(exp[0]) <= self.clock(): return 'roster has expired' if not giftwrap.verify_event(event): return 'signature does not verify' return None
def _is_member(self, topic: str, pubkey: str) -> bool: with self._lock: row = self.members.get(topic) return bool(row and pubkey in row.members)
[docs] def i_am_member(self, topic: str) -> bool: return self._is_member(topic, self.ks.node_pk)
# ---- grant integration (no new authorization invented) --------------------- def _has_grant(self, peer: str) -> bool: """Only graft a peer this node already shares a peering grant with. Reuses the existing accounting relationship rather than a second authorization system: a peer with no grant in either direction has no accounted channel for this node to meter gossip traffic against. """ issued = self.grants.get(peer, grants_mod.ISSUED) received = self.grants.get(peer, grants_mod.RECEIVED) live = lambda r: bool(r) and r.get('status', 'live') == 'live' return live(issued) or live(received) # ---- graft / prune: mesh maintenance --------------------------------------
[docs] def handle_graft(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]: topic = body.get('topic') if not isinstance(topic, str) or not topic: self.refusals.refuse('graft', 'no topic named', subject=peer) return ('graft refused', peer, 'no topic named') if not self._is_member(topic, self.ks.node_pk): self.refusals.refuse('graft', 'no membership event for this topic', subject=peer, detail={'topic': topic}) return ('graft refused', peer, 'no membership event') if not self._is_member(topic, peer): self.refusals.refuse('graft', 'peer is not a named member', subject=peer, detail={'topic': topic}) return ('graft refused', peer, 'peer not a member') if not self._has_grant(peer): self.refusals.refuse('graft', 'no peering grant with this peer', subject=peer, detail={'topic': topic}) return ('graft refused', peer, 'no grant') with self._lock: current = self.peers.setdefault(topic, set()) if peer in current: return ('graft already present', peer, topic) if len(current) >= self.mesh_degree: self.refusals.refuse('graft', 'mesh is already at target degree', subject=peer, detail={'topic': topic}) return ('graft refused', peer, 'mesh full') current.add(peer) self.grafted_at[(topic, peer)] = self.clock() self.say('graft_established', topic=topic, peer=peer) return ('grafted', peer, topic)
[docs] def handle_prune(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]: topic = body.get('topic') if not isinstance(topic, str): return ('prune ignored', peer, 'no topic named') with self._lock: self.peers.get(topic, set()).discard(peer) self.grafted_at.pop((topic, peer), None) self.say('prune_executed', topic=topic, peer=peer, reason='remote_prune') return ('pruned', peer, topic)
[docs] def graft(self, topic: str, peer: str) -> bool: """This node asks `peer` to add it to their mesh for `topic`.""" row = self.members.get(topic) if row is None: self.refusals.refuse('graft', 'no membership event for this topic', subject=peer, detail={'topic': topic}) return False msg = make_graft(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer, topic, row.event.get('id', '')) ok = self._send(peer, msg) if ok: with self._lock: self.peers.setdefault(topic, set()).add(peer) self.grafted_at[(topic, peer)] = self.clock() self.say('graft_established', topic=topic, peer=peer) return ok
[docs] def prune(self, topic: str, peer: str, reason: str = 'degree') -> bool: msg = make_prune(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer, topic, reason=reason) with self._lock: self.peers.get(topic, set()).discard(peer) self.grafted_at.pop((topic, peer), None) ok = self._send(peer, msg) self.say('prune_executed', topic=topic, peer=peer, reason=reason) return ok
# ---- publish: the flood ----------------------------------------------------
[docs] def handle_publish(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]: topic = body.get('topic') event_id = body.get('event_id') envelope = body.get('envelope') if not isinstance(topic, str) or not isinstance(event_id, str) or not isinstance(envelope, dict): self.refusals.refuse('publish', 'malformed body', subject=peer) return ('publish refused', peer, 'malformed body') if not self._is_member(topic, self.ks.node_pk): self.refusals.refuse('publish', 'no membership event for this topic', subject=peer, detail={'topic': topic}) return ('publish refused', peer, 'not a member') if not self.seen.mark(topic, event_id, self.clock()): return ('publish dropped', peer, 'already seen') self._record(self.delivered, (topic, event_id, peer)) self._cache(topic, event_id, envelope) self._flood(topic, event_id, envelope, exclude={peer}) return ('publish delivered', peer, event_id)
[docs] def publish(self, topic: str, event_id: str, envelope: Mapping[str, Any]) -> list[str]: """Originate a publish into this topic's mesh. Returns peers flooded to.""" if not self.seen.mark(topic, event_id, self.clock()): return [] self._record(self.published, (topic, event_id)) self._cache(topic, event_id, envelope) return self._flood(topic, event_id, envelope, exclude=set())
# ---- ihave / iwant: lazy gossip to members outside the mesh ---------------- def _cache(self, topic: str, event_id: str, envelope: Mapping[str, Any]) -> None: with self._lock: self.mcache[(topic, event_id)] = (envelope, self.clock()) mine = [k for k in self.mcache if k[0] == topic] if len(mine) > MCACHE_MAX_PER_TOPIC: mine.sort(key=lambda k: self.mcache[k][1]) for k in mine[:len(mine) - MCACHE_MAX_PER_TOPIC]: del self.mcache[k] def _sweep_mcache(self, now: int) -> int: with self._lock: old = [k for k, (_, at) in self.mcache.items() if now - at > MCACHE_SECONDS] for k in old: del self.mcache[k] return len(old) def _recent_ids(self, topic: str) -> list[str]: with self._lock: ids = [(at, k[1]) for k, (_, at) in self.mcache.items() if k[0] == topic] ids.sort(reverse=True) return [i for _, i in ids[:IHAVE_MAX_IDS]] def _me(self) -> str: return self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk
[docs] def emit_gossip(self, topic: str) -> list[tuple[Any, ...]]: """Offer this topic's recent ids to up to GOSSIP_LAZY_DEGREE members that are *not* in this node's mesh (the mesh already got the flood). Only members with a grant: an ihave is a promise to answer iwant.""" ids = self._recent_ids(topic) if not ids: return [] with self._lock: mesh = self.peers.get(topic, set()) row = self.members.get(topic) if row is None: return [] outside = [p for p in row.members if p != self.ks.node_pk and p not in mesh] did: list[tuple[Any, ...]] = [] for peer in outside: if len(did) >= GOSSIP_LAZY_DEGREE: break if not self._has_grant(peer): continue if self._send(peer, make_ihave(self._me(), peer, topic, ids)): self._record(self.offered, (topic, peer, len(ids))) did.append(('ihave', topic, peer)) return did
[docs] def handle_ihave(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]: topic = body.get('topic') ids = body.get('ids') if not isinstance(topic, str) or not isinstance(ids, list): self.refusals.refuse('ihave', 'malformed body', subject=peer) return ('ihave refused', peer, 'malformed body') if not self._is_member(topic, self.ks.node_pk) or not self._is_member(topic, peer): self.refusals.refuse('ihave', 'not both members of this topic', subject=peer, detail={'topic': topic}) return ('ihave refused', peer, 'not a member') missing = [i for i in ids[:IHAVE_MAX_IDS] if isinstance(i, str) and not self.seen.seen(topic, i)] if not missing: return ('ihave nothing missing', peer, topic) if self._send(peer, make_iwant(self._me(), peer, topic, missing)): self._record(self.wanted, (topic, peer, len(missing))) return ('iwant sent', peer, len(missing)) return ('iwant unsent', peer, len(missing))
[docs] def handle_iwant(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]: topic = body.get('topic') ids = body.get('ids') if not isinstance(topic, str) or not isinstance(ids, list): self.refusals.refuse('iwant', 'malformed body', subject=peer) return ('iwant refused', peer, 'malformed body') if not self._is_member(topic, self.ks.node_pk) or not self._is_member(topic, peer): self.refusals.refuse('iwant', 'not both members of this topic', subject=peer, detail={'topic': topic}) return ('iwant refused', peer, 'not a member') sent = 0 for event_id in ids[:IHAVE_MAX_IDS]: if not isinstance(event_id, str): continue with self._lock: hit = self.mcache.get((topic, event_id)) if hit is None: continue # fell out of the cache: the asker will hear it from someone else if self._send(peer, make_publish(self._me(), peer, topic, event_id, hit[0])): self._record(self.served, (topic, event_id, peer)) sent += 1 return ('iwant served', peer, sent)
def _flood(self, topic: str, event_id: str, envelope: Mapping[str, Any], exclude: set[str]) -> list[str]: """Send to every mesh peer for `topic` except `exclude` - which always contains the peer this event arrived from, so it is never re-flooded back toward its source. This is the strict loop-prevention rule.""" with self._lock: targets = list(self.peers.get(topic, set()) - exclude) sent: list[str] = [] for peer in targets: msg = make_publish(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer, topic, event_id, envelope) if self._send(peer, msg): sent.append(peer) return sent # ---- transport (rides the existing transport ladder) ----------------------- def _send(self, peer: str, message: Mapping[str, Any]) -> bool: # TransportRouter.send(message, to_pubkey) -> {rung: (ok, why)}; the # ladder picks the rung. Sent means at least one rung took it. try: res = self.wire_client.send(peer, message) if isinstance(res, dict) and res and not any(ok for ok, _ in res.values()): self.refusals.refuse('transport', 'no rung accepted', subject=peer, detail={'why': [w for _, w in res.values()][:2]}) return False return True except Exception as e: self.refusals.refuse('transport', 'send failed', subject=peer, detail={'error': str(e)}) return False # ---- dispatch ---------------------------------------------------------------
[docs] def wire_graft(self, inbound: Any, sender: str) -> tuple[Any, ...]: msg = getattr(inbound, 'message', inbound) return self.handle_graft(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_prune(self, inbound: Any, sender: str) -> tuple[Any, ...]: msg = getattr(inbound, 'message', inbound) return self.handle_prune(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_publish(self, inbound: Any, sender: str) -> tuple[Any, ...]: msg = getattr(inbound, 'message', inbound) return self.handle_publish(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_ihave(self, inbound: Any, sender: str) -> tuple[Any, ...]: msg = getattr(inbound, 'message', inbound) return self.handle_ihave(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_iwant(self, inbound: Any, sender: str) -> tuple[Any, ...]: msg = getattr(inbound, 'message', inbound) return self.handle_iwant(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
# ---- the compartment contract -------------------------------------------------
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: did: list[tuple[Any, ...]] = [] if self.relays and set(self.grants.peers()) != self._watching: self._wake.set() # a new grant: the roster subscription needs its author with self._lock: topics = list(self.members.keys()) for topic in topics: if not self.i_am_member(topic): continue with self._lock: current = self.peers.get(topic, set()) degree = len(current) candidates = [p for p in self.members[topic].members if p != self.ks.node_pk and p not in current] if degree < self.mesh_degree: for peer in candidates: if len(current) >= self.mesh_degree: break if not self._has_grant(peer): continue if self.graft(topic, peer): did.append(('regrafted', topic, peer)) current = self.peers.get(topic, set()) # Prune a peer no longer named in a superseding membership event. with self._lock: stale = [p for p in self.peers.get(topic, set()) if not self._is_member(topic, p)] for peer in stale: self.prune(topic, peer, reason='no longer a member') did.append(('pruned stale', topic, peer)) did.extend(self.emit_gossip(topic)) self._sweep_mcache(now) did += self.run_every(now) return did
_tick_runs_every = True
[docs] def expire_seen(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: now = now if now is not None else self.clock() removed = self.seen.expire(now) self._last_seen_sweep = now if removed: return [('seen store swept', removed, None)] return []
[docs] def state(self) -> GossipStateReport: with self._lock: mesh = {t: len(p) for t, p in self.peers.items()} topics = sorted(self.members.keys()) totals = self.totals.as_dict() return { 'topics': topics, 'mesh': mesh, # The store is closed after stop(); health() still reads state. 'seen': self._seen_count if getattr(self, '_closed', False) else self._count_seen(), # Counted, not measured: each was len() of a list that grew # forever. The rings beside them keep only the recent ones. 'published': totals['published'], 'auto_admitted': totals['auto_admitted'], 'delivered': totals['delivered'], 'offered': totals['offered'], 'wanted': totals['wanted'], 'served': totals['served'], 'mcache': len(self.mcache), 'refused': self.refusals.total(), }
[docs] def trouble(self) -> list[str]: out: list[str] = [] refused = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='graft') if refused: peers = sorted({(r.subject or '')[:16] for r in refused}) out.append('%d graft(s) refused in the last window from %s: no membership event, ' 'not a named member, no grant, or the mesh is already at target degree' % (len(refused), ', '.join(peers))) with self._lock: thin = [t for t, p in self.peers.items() if len(p) < max(1, self.mesh_degree // 2)] if thin: out.append('%d topic(s) below half target mesh degree: %s' % (len(thin), ', '.join(sorted(thin)[:5]))) return out
def _count_seen(self) -> int: self._seen_count = self.seen.count() return self._seen_count
[docs] def topics_cmd(self) -> list[TopicView]: with self._lock: return [{'topic': t, 'admin': row.event.get('pubkey'), 'members': sorted(row.members), 'mesh': sorted(self.peers.get(t, set())), 'expires': int(next((x[1] for x in row.event.get('tags', []) if x[0] == 'expiration'), 0)), 'roster_id': row.event.get('id')} for t, row in sorted(self.members.items())]
[docs] def create_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: try: return self.create_topic(str(args['name']), list(args.get('members') or []), int(args.get('ttl_seconds') or 30 * 86400), publish_to_relays=not args.get('no_relay')) except GossipError as e: from fleetmesh_node.app import socket as control_mod raise control_mod.ControlError(str(e))
[docs] def admit_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: try: return {'topic': self.admit(dict(args['event']))} except GossipError as e: from fleetmesh_node.app import socket as control_mod raise control_mod.ControlError(str(e))
[docs] def leave_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: try: return {'topic': args['topic'], 'pruned': self.leave(str(args['topic']))} except GossipError as e: from fleetmesh_node.app import socket as control_mod raise control_mod.ControlError(str(e))
[docs] def publish_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: try: return self.originate(str(args['topic']), dict(args['envelope'])) except GossipError as e: from fleetmesh_node.app import socket as control_mod raise control_mod.ControlError(str(e))
[docs] def stop(self) -> Any: self._closed = True self._stop.set() self._wake.set() # ends _wait_either now, not at its next slice for s in (self.seen, self.rosters): try: s.close() except Exception: pass return super().stop()
__all__ = ['GossipMesh', 'PIURI', 'MEMBERSHIP_KIND', 'MCACHE_SECONDS', 'GOSSIP_LAZY_DEGREE']