"""GossipMesh: the gossip/1.0 compartment.
A true epidemic fanout for group topics, independent of any single relay.
Membership in a topic is explicit: an admin-signed kind 30804 (addressable,
republish-supersedes) naming every member pubkey, not ambient trust. A node
only grafts into, or accepts a graft for, a topic it holds a current
membership event naming its own pubkey for - and only toward a peer it
already shares a peering grant with, so this invents no new authorization
and rides the budget/grant model everyone else already accounts against.
The payload gossiped is the NIP-59 seal+wrap this node would otherwise have
sent over the 1:1 transport-ladder path - one sealed envelope per recipient,
exactly as DMs ship today - so gossip never sees plaintext: it floods
already-encrypted bytes keyed by the wrap's own event id. 1:1 DMs are
untouched; this is purely an additional fanout substrate for group topics of
3+ members, which is where a single relay going down costs everyone their
only path.
Loop prevention is strict: a seen-id store (sqlite, TTL + size bounded) keyed
(topic, event_id) is checked before flooding, and an id is never re-sent back
toward the peer it arrived from. graft/prune build and prune a bounded mesh
per topic, degree DEFAULT_MESH_DEGREE (gossipsub's D), maintained by tick().
ihave/iwant is the lazy half (gossipsub's "gossip" proper): every tick, each
topic's recent event ids are offered to a few members *outside* this node's
mesh, and a member that is missing one asks for it by id and is sent the
sealed envelope directly. That is what carries a publish across a mesh
that is partitioned - a member whose grafts all failed, a topic larger than
D where two cliques formed, a node that was down for the flood - without
any relay in the path. Envelopes to answer an iwant with come from a short
in-memory message cache; the durable seen store stays the only thing loop
prevention relies on.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Callable, Mapping, Optional, Union
from fleetmesh_node import channels as ch_mod
from fleetmesh_node.storage import grants as grants_mod
from fleetmesh_node import metrics as mx_mod
from fleetmesh_node.protocols.nostr import client as relay_mod
from fleetmesh_core.crypto import giftwrap
from fleetmesh_core.wire.interfaces import NostrEvent
from fleetmesh_node.config.compartments import GossipConfig
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.compartment import Command, Every, Handler, Need, Outbound, WireProtocol, Worker
from fleetmesh_node.compartment import stores as st
from .envelope import (DEFAULT_MESH_DEGREE, GossipError, MEMBERSHIP_KIND, PIURI,
make_graft, make_ihave, make_iwant, make_prune, make_publish)
from .mesh import RosterStore, SeenStore
REGRAFT_INTERVAL_SECONDS: int = 300
SEEN_SWEEP_INTERVAL_SECONDS: int = 3600
# Lazy gossip (gossipsub's mcache / D_lazy / history window). Envelopes are
# kept just long enough to answer an iwant for an id offered in an ihave; ids
# are offered to a few non-mesh members per tick, never to the mesh itself,
# which already had the full envelope flooded to it.
MCACHE_SECONDS: int = 120
MCACHE_MAX_PER_TOPIC: int = 512
GOSSIP_LAZY_DEGREE: int = 6
IHAVE_MAX_IDS: int = 256
# The message cache across every topic. MCACHE_MAX_PER_TOPIC still caps one
# topic; this caps the sum, so a node carrying many busy topics holds a
# bounded number of envelopes. It is a cache for lazy gossip and nothing
# depends on a hit: an id that fell out is heard from another member.
MCACHE_MAX: int = MCACHE_MAX_PER_TOPIC * 16
# How many of each kind of gossip event the compartment keeps to look at. The
# totals are counted separately (GossipTotals), so this bounds the memory and
# not the numbers an operator reads.
RECENT: int = 256
@dataclass(slots=True)
class Roster:
"""A kind 30804 membership event this node holds for a topic, and the
member set read from its tags. The event is kept as it arrived - it is
what graft names, what `topics` reports and what RosterStore persists."""
event: Mapping[str, Any]
members: set[str] = field(default_factory=set)
from .types import (
GossipTotals,
TopicView,
TopicCreationResult,
OriginateResult,
GossipStateReport,
)
def _recent(key: str, what: str) -> st.Store:
return st.Store(key=key, description='%s, newest last' % what, kind=st.RING, shape=tuple, bound=RECENT)
[docs]class GossipMesh(WireProtocol):
"""The group-topic gossip/1.0 compartment: graft/prune mesh + flood publish."""
name: str = 'gossip'
piuris: tuple[str, ...] = (PIURI,)
optional: bool = True
enabled_by: Optional[str] = 'gossip'
# Gossip reads the grant store and the relay list, both context fields;
# it never reaches the peering or budget compartments.
requires: tuple[str, ...] = ()
reads: tuple[str, ...] = ('gossip',)
config_cls: Any = GossipConfig
HANDLERS = (
Handler('graft', run='wire_graft', doc='receive gossip mesh graft'),
Handler('prune', run='wire_prune', doc='receive gossip mesh prune'),
Handler('publish', run='wire_publish', doc='receive epidemic flood publish'),
Handler('ihave', run='wire_ihave', doc='receive lazy gossip ihave offer'),
Handler('iwant', run='wire_iwant', doc='receive lazy gossip iwant request'),
)
OUTBOUND = (
Outbound('graft', reply=('graft', 'prune'), doc='request to join a topic mesh'),
Outbound('prune', doc='leave a topic mesh or prune a peer'),
Outbound('publish', doc='epidemic flood of sealed message to topic mesh'),
Outbound('ihave', reply='iwant', doc='lazy gossip offer of recently seen message ids'),
Outbound('iwant', reply='publish', doc='request message payloads for offered ids'),
)
COMMANDS = (
Command(
key='show',
name='Gossip Status',
description='topics this node gossips, mesh size per topic, and seen-store size',
tags=('gossip', 'status'),
permission='gossip/show',
run='state',
reply='{topics, mesh, seen}',
),
Command(
key='topics',
name='Gossip Topics',
description='every roster held: admin, members, mesh peers, expiry',
tags=('gossip', 'topics'),
permission='gossip/topics',
run='topics_cmd',
reply='[{topic, admin, members, mesh, expires, roster_id}]',
),
Command(
key='create',
name='Create Topic',
description='sign a kind 30804 roster for <node_pk>:<name>, admit it, publish it to the relays',
tags=('gossip', 'create'),
parameters=(
Parameter(key='name', name='name', description='topic name', type=str, default='', required=True),
Parameter(key='members', name='members', description='initial member pubkeys', type=list, default=list, required=False),
Parameter(key='ttl_seconds', name='ttl_seconds', description='roster time-to-live in seconds', type=int, default=30 * 86400, required=False),
Parameter(key='no_relay', name='no_relay', description='do not publish to relays', type=bool, default=False, required=False),
),
permission='gossip/create',
run='create_cmd',
reply='{topic, event, relays}',
),
Command(
key='admit',
name='Admit Topic',
description='admit a kind 30804 roster another admin signed that names this node',
tags=('gossip', 'admit'),
parameters=(
Parameter(key='event', name='event', description='kind 30804 roster event', type=dict, default=dict, required=True),
),
permission='gossip/admit',
run='admit_cmd',
reply='{topic}',
),
Command(
key='leave',
name='Leave Topic',
description='drop a roster and prune every mesh peer on it',
tags=('gossip', 'leave'),
parameters=(
Parameter(key='topic', name='topic', description='topic to leave', type=str, default='', required=True),
),
permission='gossip/leave',
run='leave_cmd',
reply='{topic, pruned}',
),
Command(
key='publish',
name='Publish Message',
description='flood a sealed NIP-59 wrap into a topic mesh',
tags=('gossip', 'publish'),
parameters=(
Parameter(key='topic', name='topic', description='target topic id', type=str, default='', required=True),
Parameter(key='envelope', name='envelope', description='sealed NIP-59 wrap event', type=dict, default=dict, required=True),
),
permission='gossip/publish',
run='publish_cmd',
reply='{topic, event_id, sent_to, mesh}',
),
)
# What it keeps. Each of the six history lists below used to be a plain
# list appended once per gossip message and never trimmed - memory that
# grew with whatever peers sent - while nothing in the node read more
# than its length. Now: a ring of the recent ones, and the length kept
# as a counter in `totals`, so `state` reports the same numbers.
STORES = (
# REFUSE: dropping a roster would silently leave a topic. create() and
# admit() refuse at the configured max_topics first, with a reason.
st.Store(key='members', description='the roster held for each topic', kind=st.TABLE,
shape=Roster, bound=GossipConfig.TOPIC_CEILING, evict=st.REFUSE, keyed_by='topic'),
# Derived from members and rebuilt by tick's regraft, so OLDEST is a
# safe second line: a topic dropped here is regrafted.
st.Store(key='peers', description="the pubkeys in this node's mesh, per topic", kind=st.TABLE,
shape=set, bound=GossipConfig.TOPIC_CEILING, keyed_by='topic'),
st.Store(key='grafted_at', description='when each mesh peer was grafted, per topic', kind=st.TABLE,
shape=int, bound=GossipConfig.TOPIC_CEILING * GossipConfig.MESH_DEGREE_CEILING,
keyed_by='topic and peer'),
st.Store(key='mcache', description='sealed envelopes kept to answer an iwant, with when they were cached',
kind=st.TABLE, shape=tuple, bound=MCACHE_MAX, keyed_by='topic and event'),
st.Store(key='totals', description='gossip events this node took part in, since it started',
kind=st.COUNTERS, shape=GossipTotals),
_recent('published', 'events this node originated, as (topic, event id)'),
_recent('delivered', 'events delivered from a mesh peer, as (topic, event id, from)'),
_recent('offered', 'ihave offers sent, as (topic, peer, ids)'),
_recent('wanted', 'iwant requests sent, as (topic, peer, ids)'),
_recent('served', 'iwant answers sent, as (topic, event id, to)'),
_recent('auto_admitted', 'rosters admitted from a relay, as (topic, roster id)'),
)
CHANNELS = (
ch_mod.Channel('event_published', 'A gossip event was originated and flooded to the topic mesh',
level=ch_mod.INFO, say='published gossip event {event_id} to topic {topic}',
tags=('gossip', 'publish')),
ch_mod.Channel('event_delivered', 'A gossip event was delivered from a mesh peer',
level=ch_mod.INFO, say='delivered gossip event {event_id} on topic {topic} from {peer}',
tags=('gossip', 'deliver')),
ch_mod.Channel('graft_established', "A peer was grafted into this node's topic mesh",
level=ch_mod.INFO, say='grafted peer {peer} on topic {topic}',
tags=('gossip', 'mesh')),
ch_mod.Channel('prune_executed', "A peer was pruned from this node's topic mesh",
level=ch_mod.NOTE, say='pruned peer {peer} on topic {topic} (reason: {reason})',
tags=('gossip', 'mesh')),
)
METRICS = (
mx_mod.Metric(name='gossip_events_published', description='Events originated and flooded to topic mesh.',
kind=mx_mod.COUNTER, counts='event_published', tags=('gossip', 'publish')),
mx_mod.Metric(name='gossip_events_delivered', description='Events received from mesh peers and delivered.',
kind=mx_mod.COUNTER, counts='event_delivered', tags=('gossip', 'deliver')),
mx_mod.Metric(name='gossip_peers_grafted', description='Peers added to local gossip mesh.',
kind=mx_mod.COUNTER, counts='graft_established', tags=('gossip', 'mesh')),
mx_mod.Metric(name='gossip_peers_pruned', description='Peers removed from local gossip mesh.',
kind=mx_mod.COUNTER, counts='prune_executed', tags=('gossip', 'mesh')),
)
NEEDS = (
Need('keystore', 'ks'),
Need('transport', 'transport'),
Need('grant_store', 'stores.grants'),
Need('cfg', block=GossipConfig),
Need('seen_path', path='gossip_seen.sqlite'),
Need('roster_path', path='gossip_rosters.sqlite'),
Need('relays', 'relays'),
)
EVERY = (Every(seconds=SEEN_SWEEP_INTERVAL_SECONDS, run='expire_seen'),)
WORKERS = (
Worker(run='_listen', name='gossip-rosters', when='relays'),
)
[docs] def __init__(
self,
keystore: Any,
transport: Any,
grant_store: grants_mod.GrantStore,
clock: Optional[Callable[[], int]] = None,
node: Optional[Any] = None,
mesh_degree: int = DEFAULT_MESH_DEGREE,
seen_ttl_seconds: int = 24 * 3600,
max_topics: int = 64,
seen_path: str = ':memory:',
roster_path: Optional[str] = None,
relays: Optional[list[str]] = None,
cfg: Optional[Union[GossipConfig, Mapping[str, Any]]] = None,
) -> None:
super().__init__(node, clock)
self.ks: Any = keystore
self.transport: Any = transport
self.grants: grants_mod.GrantStore = grant_store
if cfg is not None:
if isinstance(cfg, GossipConfig):
mesh_degree = cfg.mesh_degree
seen_ttl_seconds = cfg.seen_ttl_seconds
max_topics = cfg.max_topics
elif isinstance(cfg, Mapping):
mesh_degree = int(cfg.get('mesh_degree', mesh_degree))
seen_ttl_seconds = int(cfg.get('seen_ttl_seconds', seen_ttl_seconds))
max_topics = int(cfg.get('max_topics', max_topics))
self.mesh_degree: int = mesh_degree
self.seen_ttl_seconds: int = seen_ttl_seconds
self.max_topics: int = max_topics
self.seen: SeenStore = SeenStore(seen_path, ttl_seconds=seen_ttl_seconds)
self.rosters: RosterStore = RosterStore(roster_path or seen_path)
self.relays: list[str] = list(relays or []) # not a store: the configured relays
self._lock: threading.RLock = threading.RLock()
self._last_seen_sweep: int = 0
self._closed: bool = False
self._seen_count: int = 0
self._stop: threading.Event = getattr(self, '_stop_event', None) or threading.Event()
self._wake: threading.Event = threading.Event() # set by tick when the granted set moves
self._watching: set[str] = set() # not a store: a copy of grants.peers(), replaced whole
# The stores were built and bound by the base (see STORES).
self.members: st.Table[str, Roster]
self.peers: st.Table[str, set[str]]
self.grafted_at: st.Table[tuple[str, str], int]
self.mcache: st.Table[tuple[str, str], tuple[Mapping[str, Any], int]]
self.totals: st.Counters
self.published: st.Ring[tuple[str, str]]
self.delivered: st.Ring[tuple[str, str, str]]
self.offered: st.Ring[tuple[str, str, int]]
self.wanted: st.Ring[tuple[str, str, int]]
self.served: st.Ring[tuple[str, str, str]]
self.auto_admitted: st.Ring[tuple[str, str]]
self._reload_rosters()
def _record(self, ring: 'st.Ring[Any]', entry: tuple[Any, ...]) -> None:
"""One gossip event: kept in its ring, and counted in `totals`."""
ring.append(entry)
self.totals.incr(ring.store.key)
if ring.store.key == 'published' and len(entry) >= 2:
self.say('event_published', topic=entry[0], event_id=entry[1])
elif ring.store.key == 'delivered' and len(entry) >= 3:
self.say('event_delivered', topic=entry[0], event_id=entry[1], peer=entry[2])
# ---- learning rosters from the relays -------------------------------------
def _listen(self) -> None:
while not self._stop.is_set():
admins = sorted(self.grants.peers())
self._watching = set(admins) # not a store: a copy of grants.peers(), replaced whole
self._wake.clear()
if not admins:
self._wait_either(30)
continue
# A short socket timeout is how the loop notices _wake between frames.
r = relay_mod.Relay(self.relays[0], timeout=1.0)
try:
# NIP-01 indexes single-letter tags only, so `member` cannot be
# filtered at the relay; admit() drops rosters not naming us.
sid = r.connect().subscribe([{'kinds': [MEMBERSHIP_KIND], 'authors': admins}])
for ev in r.events(sid, until=None):
if self._stop.is_set() or self._wake.is_set():
break
self.on_roster(ev)
if self._wake.is_set():
continue # re-subscribe now with the new author set
except Exception as e:
self.refusals.refuse('subscription', 'could not be held open',
subject=self.relays[0], detail={'error': str(e)})
finally:
r.close()
self._wait_either(5)
def _wait_either(self, seconds: float) -> None:
"""Sleep until `seconds` pass, the node stops, or the granted set moves.
Waits on `_wake`, which stop() sets too, in slices of a second so a
stop that reaches only `_stop` is still seen within one. It polled
every 0.2 s on the wall clock (audit 2026-10-15, E8).
"""
deadline = time.monotonic() + seconds
while not self._stop.is_set() and not self._wake.is_set():
left = deadline - time.monotonic()
if left <= 0:
return
self._wake.wait(min(1.0, left))
[docs] def on_roster(self, event: Mapping[str, Any]) -> Optional[str]:
"""A roster read from a relay: admit it if it names this node and is
from a granted peer; the topic admitted, or None (refusal recorded)."""
if not isinstance(event, Mapping) or event.get('kind') != MEMBERSHIP_KIND:
return None
event = dict(event)
admin = event.get('pubkey', '')
if not self._has_grant(admin):
self.refusals.refuse('membership', 'roster from a peer with no grant', subject=admin)
return None
try:
topic = self.admit(event)
except GossipError:
return None # admit() recorded why, or it was not for us
self._record(self.auto_admitted, (topic, event.get('id', '')))
return topic
def _reload_rosters(self) -> None:
"""Rosters held before the last stop. One that has expired meanwhile is
dropped here rather than refused on every tick."""
for topic, event in self.rosters.all().items():
members = [t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'member']
if self.check_membership(topic, event, members) is None:
if not self.members.put(topic, Roster(event, set(members))):
# More rosters on disk than the topic ceiling - written
# by a build with a higher one. Kept on disk, not held.
self.refusals.refuse('membership', 'over the topic ceiling; not reloaded', subject=topic)
else:
self.rosters.delete(topic)
# ---- membership -----------------------------------------------------------
[docs] def admit_membership(self, topic: str, event: Mapping[str, Any], member_pubkeys: list[str]) -> None:
"""Record a current admin-signed kind 30804 membership event for a topic.
A republish of the same `d` (topic) supersedes the one held, matching
the addressable-event convention every other control event here uses.
The roster is read from the event's own `member` tags; the caller's
`member_pubkeys` is accepted only when it matches them, so a caller
cannot admit a roster the admin did not sign.
"""
why = self.check_membership(topic, event, member_pubkeys)
if why is not None:
self.refusals.refuse('membership', why, subject=topic)
raise GossipError(why)
with self._lock:
held = self.members.get(topic)
if held and int(held.event.get('created_at', 0)) > int(event.get('created_at', 0)):
self.refusals.refuse('membership', 'older than the roster held', subject=topic)
raise GossipError('older than the roster held')
self.members[topic] = Roster(event, set(member_pubkeys))
self.rosters.put(topic, event)
# ---- operator verbs: the control socket's side of the compartment ----------
[docs] def topic_id(self, name: str) -> str:
"""The coordinate of a topic this node administers: `<node_pk>:<name>`."""
if not name or ':' in name or len(name) > 64:
raise GossipError('a topic name is 1-64 characters with no colon')
return self.ks.node_pk + ':' + name
[docs] def make_roster(self, name: str, members: list[str], ttl_seconds: int = 30 * 86400) -> NostrEvent:
"""Sign a kind 30804 roster for a topic this node administers.
This node is always on its own roster: an admin that is not a member
would hold a topic it can never graft into. Signing is the slow pure-
Python curve, so this is an operator action, not something on a tick.
"""
from fleetmesh_core.storage import store
topic = self.topic_id(name)
roster = sorted({self.ks.node_pk, *members})
if len(roster) < 2:
raise GossipError('a roster names at least one member besides this node')
if ttl_seconds <= 0:
raise GossipError('ttl_seconds is positive')
now = self.clock()
tags = [['d', topic]] + [['member', m] for m in roster] + [['expiration', str(now + int(ttl_seconds))]]
return store.make_signed_event(self.ks.node_sk, now, MEMBERSHIP_KIND, tags, '')
[docs] def create_topic(self, name: str, members: list[str], ttl_seconds: int = 30 * 86400,
publish_to_relays: bool = True) -> TopicCreationResult:
"""Sign, admit, and (optionally) publish a roster. Returns {topic, event, relays}.
Publishing to the configured relays is how the other members learn the
roster before any of them has a gossip mesh to learn it over - the one
place a relay is in the path, and only as a bulletin board.
"""
event = self.make_roster(name, members, ttl_seconds)
topic = self.topic_id(name)
if topic not in self.members and len(self.members) >= self.max_topics:
raise GossipError('this node holds its maximum of %d topics' % self.max_topics)
self.admit_membership(topic, event, sorted(self.members_of(event)))
out: TopicCreationResult = {'topic': topic, 'event': event, 'relays': {}}
# Plan D2: the roster is this node's own signed event; a fully private
# node puts it on its network relays (relay_destination), not the public
# ones, so it never reveals itself there.
urls = self.relay_destination(self.relays)
if publish_to_relays and urls:
out['relays'] = {u: list(r) for u, r in relay_mod.publish_all(urls, event, 15.0).items()}
return out
[docs] @staticmethod
def members_of(event: Mapping[str, Any]) -> set[str]:
return {t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'member'}
[docs] def admit(self, event: Mapping[str, Any]) -> str:
"""Admit a roster another admin signed (fetched from a relay, pasted by
the operator). The topic is the event's own `d`; returns it."""
d = [t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == 'd']
if len(d) != 1:
raise GossipError('roster has no single d tag')
if self.ks.node_pk not in self.members_of(event):
raise GossipError('this node is not named on that roster')
if d[0] not in self.members and len(self.members) >= self.max_topics:
raise GossipError('this node holds its maximum of %d topics' % self.max_topics)
self.admit_membership(d[0], event, sorted(self.members_of(event)))
return d[0]
[docs] def leave(self, topic: str) -> int:
"""Drop a roster and prune every mesh peer on it. Returns peers pruned."""
with self._lock:
if topic not in self.members:
raise GossipError('not a member of %s' % topic)
peers = list(self.peers.get(topic, set()))
for p in peers:
self.prune(topic, p, reason='left')
with self._lock:
self.members.pop(topic, None)
self.peers.pop(topic, None)
self.rosters.delete(topic)
return len(peers)
[docs] def originate(self, topic: str, envelope: Mapping[str, Any]) -> OriginateResult:
"""Flood an already-sealed NIP-59 wrap (kind 1059/13/21059) into a topic.
The compartment carries ciphertext only; a plaintext kind here is a
refusal, not a convenience, because the mesh is every member's peers.
"""
if not self._is_member(topic, self.ks.node_pk):
raise GossipError('not a member of %s' % topic)
if envelope.get('kind') not in (13, 1059, 21059):
raise GossipError('only sealed NIP-59 envelopes (kind 13, 1059, 21059) travel on gossip')
event_id = envelope.get('id')
if not isinstance(event_id, str) or len(event_id) != 64:
raise GossipError('envelope has no 64-hex id')
if not giftwrap.verify_event(envelope):
raise GossipError('envelope signature does not verify')
sent = self.publish(topic, event_id, envelope)
return {'topic': topic, 'event_id': event_id, 'sent_to': sent,
'mesh': len(self.peers.get(topic, set()))}
[docs] def check_membership(self, topic: str, event: Mapping[str, Any], member_pubkeys: list[str]) -> Optional[str]:
"""Why this event is not a current kind 30804 roster for `topic`, or None.
Kind, signature, `d` coordinate scoped to the signing admin, a roster
that is what the tags say, and a NIP-40 expiration still ahead - in
that order, cheapest refusal first.
"""
if event.get('kind') != MEMBERSHIP_KIND:
return 'not a kind %d' % MEMBERSHIP_KIND
tags = event.get('tags') or []
d = [t[1] for t in tags if len(t) > 1 and t[0] == 'd']
if d != [topic]:
return 'd tag does not name this topic'
admin = event.get('pubkey', '')
if not topic.startswith(admin + ':'):
return 'topic coordinate is not scoped to the signing admin'
members = {t[1] for t in tags if len(t) > 1 and t[0] == 'member'}
if not members or members != set(member_pubkeys):
return 'member tags do not match the roster offered'
exp = [t[1] for t in tags if len(t) > 1 and t[0] == 'expiration']
if len(exp) != 1 or not exp[0].isdigit():
return 'no single expiration tag'
if int(exp[0]) <= self.clock():
return 'roster has expired'
if not giftwrap.verify_event(event):
return 'signature does not verify'
return None
def _is_member(self, topic: str, pubkey: str) -> bool:
with self._lock:
row = self.members.get(topic)
return bool(row and pubkey in row.members)
[docs] def i_am_member(self, topic: str) -> bool:
return self._is_member(topic, self.ks.node_pk)
# ---- grant integration (no new authorization invented) ---------------------
def _has_grant(self, peer: str) -> bool:
"""Only graft a peer this node already shares a peering grant with.
Reuses the existing accounting relationship rather than a second
authorization system: a peer with no grant in either direction has no
accounted channel for this node to meter gossip traffic against.
"""
issued = self.grants.get(peer, grants_mod.ISSUED)
received = self.grants.get(peer, grants_mod.RECEIVED)
live = lambda r: bool(r) and r.get('status', 'live') == 'live'
return live(issued) or live(received)
# ---- graft / prune: mesh maintenance --------------------------------------
[docs] def handle_graft(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]:
topic = body.get('topic')
if not isinstance(topic, str) or not topic:
self.refusals.refuse('graft', 'no topic named', subject=peer)
return ('graft refused', peer, 'no topic named')
if not self._is_member(topic, self.ks.node_pk):
self.refusals.refuse('graft', 'no membership event for this topic', subject=peer, detail={'topic': topic})
return ('graft refused', peer, 'no membership event')
if not self._is_member(topic, peer):
self.refusals.refuse('graft', 'peer is not a named member', subject=peer, detail={'topic': topic})
return ('graft refused', peer, 'peer not a member')
if not self._has_grant(peer):
self.refusals.refuse('graft', 'no peering grant with this peer', subject=peer, detail={'topic': topic})
return ('graft refused', peer, 'no grant')
with self._lock:
current = self.peers.setdefault(topic, set())
if peer in current:
return ('graft already present', peer, topic)
if len(current) >= self.mesh_degree:
self.refusals.refuse('graft', 'mesh is already at target degree', subject=peer, detail={'topic': topic})
return ('graft refused', peer, 'mesh full')
current.add(peer)
self.grafted_at[(topic, peer)] = self.clock()
self.say('graft_established', topic=topic, peer=peer)
return ('grafted', peer, topic)
[docs] def handle_prune(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]:
topic = body.get('topic')
if not isinstance(topic, str):
return ('prune ignored', peer, 'no topic named')
with self._lock:
self.peers.get(topic, set()).discard(peer)
self.grafted_at.pop((topic, peer), None)
self.say('prune_executed', topic=topic, peer=peer, reason='remote_prune')
return ('pruned', peer, topic)
[docs] def graft(self, topic: str, peer: str) -> bool:
"""This node asks `peer` to add it to their mesh for `topic`."""
row = self.members.get(topic)
if row is None:
self.refusals.refuse('graft', 'no membership event for this topic', subject=peer, detail={'topic': topic})
return False
msg = make_graft(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer, topic,
row.event.get('id', ''))
ok = self._send(peer, msg)
if ok:
with self._lock:
self.peers.setdefault(topic, set()).add(peer)
self.grafted_at[(topic, peer)] = self.clock()
self.say('graft_established', topic=topic, peer=peer)
return ok
[docs] def prune(self, topic: str, peer: str, reason: str = 'degree') -> bool:
msg = make_prune(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer, topic, reason=reason)
with self._lock:
self.peers.get(topic, set()).discard(peer)
self.grafted_at.pop((topic, peer), None)
ok = self._send(peer, msg)
self.say('prune_executed', topic=topic, peer=peer, reason=reason)
return ok
# ---- publish: the flood ----------------------------------------------------
[docs] def handle_publish(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]:
topic = body.get('topic')
event_id = body.get('event_id')
envelope = body.get('envelope')
if not isinstance(topic, str) or not isinstance(event_id, str) or not isinstance(envelope, dict):
self.refusals.refuse('publish', 'malformed body', subject=peer)
return ('publish refused', peer, 'malformed body')
if not self._is_member(topic, self.ks.node_pk):
self.refusals.refuse('publish', 'no membership event for this topic', subject=peer, detail={'topic': topic})
return ('publish refused', peer, 'not a member')
if not self.seen.mark(topic, event_id, self.clock()):
return ('publish dropped', peer, 'already seen')
self._record(self.delivered, (topic, event_id, peer))
self._cache(topic, event_id, envelope)
self._flood(topic, event_id, envelope, exclude={peer})
return ('publish delivered', peer, event_id)
[docs] def publish(self, topic: str, event_id: str, envelope: Mapping[str, Any]) -> list[str]:
"""Originate a publish into this topic's mesh. Returns peers flooded to."""
if not self.seen.mark(topic, event_id, self.clock()):
return []
self._record(self.published, (topic, event_id))
self._cache(topic, event_id, envelope)
return self._flood(topic, event_id, envelope, exclude=set())
# ---- ihave / iwant: lazy gossip to members outside the mesh ----------------
def _cache(self, topic: str, event_id: str, envelope: Mapping[str, Any]) -> None:
with self._lock:
self.mcache[(topic, event_id)] = (envelope, self.clock())
mine = [k for k in self.mcache if k[0] == topic]
if len(mine) > MCACHE_MAX_PER_TOPIC:
mine.sort(key=lambda k: self.mcache[k][1])
for k in mine[:len(mine) - MCACHE_MAX_PER_TOPIC]:
del self.mcache[k]
def _sweep_mcache(self, now: int) -> int:
with self._lock:
old = [k for k, (_, at) in self.mcache.items() if now - at > MCACHE_SECONDS]
for k in old:
del self.mcache[k]
return len(old)
def _recent_ids(self, topic: str) -> list[str]:
with self._lock:
ids = [(at, k[1]) for k, (_, at) in self.mcache.items() if k[0] == topic]
ids.sort(reverse=True)
return [i for _, i in ids[:IHAVE_MAX_IDS]]
def _me(self) -> str:
return self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk
[docs] def emit_gossip(self, topic: str) -> list[tuple[Any, ...]]:
"""Offer this topic's recent ids to up to GOSSIP_LAZY_DEGREE members
that are *not* in this node's mesh (the mesh already got the flood).
Only members with a grant: an ihave is a promise to answer iwant."""
ids = self._recent_ids(topic)
if not ids:
return []
with self._lock:
mesh = self.peers.get(topic, set())
row = self.members.get(topic)
if row is None:
return []
outside = [p for p in row.members if p != self.ks.node_pk and p not in mesh]
did: list[tuple[Any, ...]] = []
for peer in outside:
if len(did) >= GOSSIP_LAZY_DEGREE:
break
if not self._has_grant(peer):
continue
if self._send(peer, make_ihave(self._me(), peer, topic, ids)):
self._record(self.offered, (topic, peer, len(ids)))
did.append(('ihave', topic, peer))
return did
[docs] def handle_ihave(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]:
topic = body.get('topic')
ids = body.get('ids')
if not isinstance(topic, str) or not isinstance(ids, list):
self.refusals.refuse('ihave', 'malformed body', subject=peer)
return ('ihave refused', peer, 'malformed body')
if not self._is_member(topic, self.ks.node_pk) or not self._is_member(topic, peer):
self.refusals.refuse('ihave', 'not both members of this topic', subject=peer, detail={'topic': topic})
return ('ihave refused', peer, 'not a member')
missing = [i for i in ids[:IHAVE_MAX_IDS] if isinstance(i, str) and not self.seen.seen(topic, i)]
if not missing:
return ('ihave nothing missing', peer, topic)
if self._send(peer, make_iwant(self._me(), peer, topic, missing)):
self._record(self.wanted, (topic, peer, len(missing)))
return ('iwant sent', peer, len(missing))
return ('iwant unsent', peer, len(missing))
[docs] def handle_iwant(self, peer: str, body: Mapping[str, Any]) -> tuple[Any, ...]:
topic = body.get('topic')
ids = body.get('ids')
if not isinstance(topic, str) or not isinstance(ids, list):
self.refusals.refuse('iwant', 'malformed body', subject=peer)
return ('iwant refused', peer, 'malformed body')
if not self._is_member(topic, self.ks.node_pk) or not self._is_member(topic, peer):
self.refusals.refuse('iwant', 'not both members of this topic', subject=peer, detail={'topic': topic})
return ('iwant refused', peer, 'not a member')
sent = 0
for event_id in ids[:IHAVE_MAX_IDS]:
if not isinstance(event_id, str):
continue
with self._lock:
hit = self.mcache.get((topic, event_id))
if hit is None:
continue # fell out of the cache: the asker will hear it from someone else
if self._send(peer, make_publish(self._me(), peer, topic, event_id, hit[0])):
self._record(self.served, (topic, event_id, peer))
sent += 1
return ('iwant served', peer, sent)
def _flood(self, topic: str, event_id: str, envelope: Mapping[str, Any], exclude: set[str]) -> list[str]:
"""Send to every mesh peer for `topic` except `exclude` - which always
contains the peer this event arrived from, so it is never re-flooded
back toward its source. This is the strict loop-prevention rule."""
with self._lock:
targets = list(self.peers.get(topic, set()) - exclude)
sent: list[str] = []
for peer in targets:
msg = make_publish(self.ks.did if hasattr(self.ks, 'did') else self.ks.node_pk, peer,
topic, event_id, envelope)
if self._send(peer, msg):
sent.append(peer)
return sent
# ---- transport (rides the existing transport ladder) -----------------------
def _send(self, peer: str, message: Mapping[str, Any]) -> bool:
# TransportRouter.send(message, to_pubkey) -> {rung: (ok, why)}; the
# ladder picks the rung. Sent means at least one rung took it.
try:
res = self.wire_client.send(peer, message)
if isinstance(res, dict) and res and not any(ok for ok, _ in res.values()):
self.refusals.refuse('transport', 'no rung accepted', subject=peer,
detail={'why': [w for _, w in res.values()][:2]})
return False
return True
except Exception as e:
self.refusals.refuse('transport', 'send failed', subject=peer, detail={'error': str(e)})
return False
# ---- dispatch ---------------------------------------------------------------
[docs] def wire_graft(self, inbound: Any, sender: str) -> tuple[Any, ...]:
msg = getattr(inbound, 'message', inbound)
return self.handle_graft(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_prune(self, inbound: Any, sender: str) -> tuple[Any, ...]:
msg = getattr(inbound, 'message', inbound)
return self.handle_prune(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_publish(self, inbound: Any, sender: str) -> tuple[Any, ...]:
msg = getattr(inbound, 'message', inbound)
return self.handle_publish(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_ihave(self, inbound: Any, sender: str) -> tuple[Any, ...]:
msg = getattr(inbound, 'message', inbound)
return self.handle_ihave(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
[docs] def wire_iwant(self, inbound: Any, sender: str) -> tuple[Any, ...]:
msg = getattr(inbound, 'message', inbound)
return self.handle_iwant(sender, (msg.get('body') or {}) if isinstance(msg, dict) else {})
# ---- the compartment contract -------------------------------------------------
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
did: list[tuple[Any, ...]] = []
if self.relays and set(self.grants.peers()) != self._watching:
self._wake.set() # a new grant: the roster subscription needs its author
with self._lock:
topics = list(self.members.keys())
for topic in topics:
if not self.i_am_member(topic):
continue
with self._lock:
current = self.peers.get(topic, set())
degree = len(current)
candidates = [p for p in self.members[topic].members
if p != self.ks.node_pk and p not in current]
if degree < self.mesh_degree:
for peer in candidates:
if len(current) >= self.mesh_degree:
break
if not self._has_grant(peer):
continue
if self.graft(topic, peer):
did.append(('regrafted', topic, peer))
current = self.peers.get(topic, set())
# Prune a peer no longer named in a superseding membership event.
with self._lock:
stale = [p for p in self.peers.get(topic, set()) if not self._is_member(topic, p)]
for peer in stale:
self.prune(topic, peer, reason='no longer a member')
did.append(('pruned stale', topic, peer))
did.extend(self.emit_gossip(topic))
self._sweep_mcache(now)
did += self.run_every(now)
return did
_tick_runs_every = True
[docs] def expire_seen(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
now = now if now is not None else self.clock()
removed = self.seen.expire(now)
self._last_seen_sweep = now
if removed:
return [('seen store swept', removed, None)]
return []
[docs] def state(self) -> GossipStateReport:
with self._lock:
mesh = {t: len(p) for t, p in self.peers.items()}
topics = sorted(self.members.keys())
totals = self.totals.as_dict()
return {
'topics': topics,
'mesh': mesh,
# The store is closed after stop(); health() still reads state.
'seen': self._seen_count if getattr(self, '_closed', False) else self._count_seen(),
# Counted, not measured: each was len() of a list that grew
# forever. The rings beside them keep only the recent ones.
'published': totals['published'],
'auto_admitted': totals['auto_admitted'],
'delivered': totals['delivered'],
'offered': totals['offered'],
'wanted': totals['wanted'],
'served': totals['served'],
'mcache': len(self.mcache),
'refused': self.refusals.total(),
}
[docs] def trouble(self) -> list[str]:
out: list[str] = []
refused = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='graft')
if refused:
peers = sorted({(r.subject or '')[:16] for r in refused})
out.append('%d graft(s) refused in the last window from %s: no membership event, '
'not a named member, no grant, or the mesh is already at target degree'
% (len(refused), ', '.join(peers)))
with self._lock:
thin = [t for t, p in self.peers.items() if len(p) < max(1, self.mesh_degree // 2)]
if thin:
out.append('%d topic(s) below half target mesh degree: %s'
% (len(thin), ', '.join(sorted(thin)[:5])))
return out
def _count_seen(self) -> int:
self._seen_count = self.seen.count()
return self._seen_count
[docs] def topics_cmd(self) -> list[TopicView]:
with self._lock:
return [{'topic': t, 'admin': row.event.get('pubkey'),
'members': sorted(row.members),
'mesh': sorted(self.peers.get(t, set())),
'expires': int(next((x[1] for x in row.event.get('tags', [])
if x[0] == 'expiration'), 0)),
'roster_id': row.event.get('id')}
for t, row in sorted(self.members.items())]
[docs] def create_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
try:
return self.create_topic(str(args['name']), list(args.get('members') or []),
int(args.get('ttl_seconds') or 30 * 86400),
publish_to_relays=not args.get('no_relay'))
except GossipError as e:
from fleetmesh_node.app import socket as control_mod
raise control_mod.ControlError(str(e))
[docs] def admit_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
try:
return {'topic': self.admit(dict(args['event']))}
except GossipError as e:
from fleetmesh_node.app import socket as control_mod
raise control_mod.ControlError(str(e))
[docs] def leave_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
try:
return {'topic': args['topic'], 'pruned': self.leave(str(args['topic']))}
except GossipError as e:
from fleetmesh_node.app import socket as control_mod
raise control_mod.ControlError(str(e))
[docs] def publish_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
try:
return self.originate(str(args['topic']), dict(args['envelope']))
except GossipError as e:
from fleetmesh_node.app import socket as control_mod
raise control_mod.ControlError(str(e))
[docs] def stop(self) -> Any:
self._closed = True
self._stop.set()
self._wake.set() # ends _wait_either now, not at its next slice
for s in (self.seen, self.rosters):
try:
s.close()
except Exception:
pass
return super().stop()
__all__ = ['GossipMesh', 'PIURI', 'MEMBERSHIP_KIND', 'MCACHE_SECONDS', 'GOSSIP_LAZY_DEGREE']