Source code for fleetmesh_node.planes.identity.announce.compartment

"""The gossipsub announce topic, reached through a sidecar, as a compartment.

Bridge 2 in the specification: a node republishes its descriptor on
`/fleetmesh/announce/1` so that nodes which have never shared a relay with it
hear about it anyway. The relay path (bridge 1, `discovery_wire.py`) always
works; this one reaches further.

It is a sidecar and not a library because of what a library would cost. `fleetmesh_core/`
is standard library only and `fleetmesh_node/` depends on `cryptography` and nothing
else - `ws.py` is a hand-rolled RFC 6455 for exactly that reason. Installing
the one maintained Python libp2p pulls in sixty-three packages, eight of them
compiled; it does not install on Python 3.14 at all; and it is trio where this
node is threads. A second implementer reading the specification should not
inherit that, so the libp2p stack lives in its own process with its own
dependencies and this file speaks to it over a unix socket in the node's
directory, one JSON object per line, the same shape `control.py` already uses.

The node never starts the sidecar and never needs it. No socket is not an
error, it is a node with one discovery bridge instead of two, and it is said
once rather than once a tick.

**Nothing here trusts the sidecar.** Every row it hands over goes through
`admit` - the same function the sidecar ran before gossipsub propagated
anything - and then through `peers.learn`, exactly as a descriptor fetched
from a relay does. The work floor is re-applied here because it is this
node's policy and not the sidecar's.

The one check `admit` needs from the sidecar is which libp2p peer originated
the message, because a Multikey is compared against it. A sidecar that lies
about the originator can only make this node reject a descriptor it would
have accepted; it cannot make it accept one, because the Multikey is inside
an event the node key signed.

The pilot bridge (fleetmesh_node/sidecar_bridge): the link to the sidecar, saying it
once each way, the version gate, the `announce.sidecar` and `announce.probe`
commands and the restart all come from `Bridge`, as they will for every other
sidecar. What is left here is what is announce's own - the topic, `admit`,
what a drain may teach this node.
"""
from __future__ import annotations

import hashlib
import json
import os
import tempfile
from typing import Any, Callable, Optional, cast

from fleetmesh_node import channels as ch_mod
from fleetmesh_node.protocols.nostr import descriptor as descriptor_mod
from fleetmesh_node.compartment import lifecycle as lifecycle_mod
from fleetmesh_node import metrics as mx_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.config import AnnounceConfig
from fleetmesh_core.crypto import keys
from fleetmesh_core.identity import identity
from fleetmesh_core.wire.interfaces import NostrEvent
from fleetmesh_node.compartment import Command, Need
from fleetmesh_node.compartment.sidecar import UP, Bridge, Sidecar, SidecarError, jsonline_call

SOCK: str = 'announce.sock'
PROTOCOL: str = 'fleetmesh-announce/1'


from .types import AnnounceStateReport
TOPIC: str = '/fleetmesh/announce/1'
MAX_LINE: int = 1 << 20
# What one drain may bring back. A topic is a flood channel with no grant in
# front of it, so the bound is here rather than in the sender's good manners.
DRAIN_MAX: int = 64
KIND: int = 11801
# What one descriptor may weigh on the topic. A kind 11801 carrying every tag
# the specification allows is a couple of kilobytes; ten times that is a
# payload nobody needs and everybody would carry.
PAYLOAD_MAX: int = 20000
# The proof-of-work floor this node requires of a descriptor heard on the
# topic. Local policy, not a protocol constant: two nodes with different
# floors interoperate, the stricter one simply propagates less. Zero means
# the descriptor's own declared target is the only bar, which is what a node
# on a private mesh wants.
POW_BITS: int = 0


[docs]def socket_path(directory: str, configured: Optional[str] = None) -> str: """Where the sidecar listens. Same derivation as the control socket, for the same reason: macOS caps a unix socket path near a hundred bytes and a test's temporary directory is longer than that.""" if configured: return os.path.abspath(os.path.expanduser(configured)) real = os.path.realpath(directory) p = os.path.join(real, SOCK) if len(p.encode()) < 100: return p return os.path.join(tempfile.gettempdir(), 'fleetmesh-announce-%s.sock' % hashlib.sha256(real.encode()).hexdigest()[:16])
[docs]def transport_multikey(desc: NostrEvent) -> Optional[str]: """The `key transport` Multikey a descriptor declares, or None.""" for t in desc.get('tags', []): if t and t[0] == 'key' and len(t) > 2 and t[1] == 'transport': return t[2] return None
[docs]def binds(desc: NostrEvent, ed25519_hex: str) -> bool: """Whether the descriptor names this Ed25519 key as its transport key. The gossipsub message was signed under StrictSign by the *originator*, and a forwarder does not re-sign, so this is the originator's key and the descriptor must be the originator's own. A descriptor relayed by a third node is still valid on a flood topic - the check is against what the descriptor claims, never against who handed it over. """ mk = transport_multikey(desc) if not mk or not ed25519_hex: return False try: kind, raw = keys.parse_multikey(mk) except ValueError: return False return kind == 'transport' and raw.hex() == ed25519_hex.lower()
[docs]def admit( payload: Any, origin: Optional[str], now: int, pow_bits: int = 0, payload_max: int = PAYLOAD_MAX, ) -> tuple[Optional[NostrEvent], Optional[str]]: """Whether the topic should carry this. (descriptor, None) or (None, why). The whole of the defence on a flood channel with no grant in front of it, and the one place it is written. The sidecar calls this before gossipsub propagates anything, and the node calls it again on what the sidecar hands over, because a separate process is not a reason to believe a descriptor. Ordered cheapest first: a size that needs no parse, a parse, a kind, a signature, the descriptor rules, the work, and last the binding, which is the only check that needs to know who sent it. `origin` is the *originator's* Ed25519 key - gossipsub StrictSign's `from`, not the peer who handed the message over. A forwarder does not re-sign, and a descriptor relayed by a third node is still valid here. """ if isinstance(payload, (bytes, bytearray)): if len(payload) > payload_max: return None, 'payload over %d bytes' % payload_max try: desc = json.loads(bytes(payload).decode('utf-8')) except (ValueError, UnicodeDecodeError): return None, 'payload is not JSON' else: desc = payload if not isinstance(desc, dict): return None, 'not an event' if desc.get('kind') != KIND: return None, 'not a kind %d' % KIND if not descriptor_mod.genuine(desc): return None, 'signature does not verify' problems = identity.validate_descriptor(desc, now=now, digest=descriptor_mod.DIGEST) if problems: return None, 'descriptor: %s' % str(problems[0])[:48] if pow_bits and identity.pow_bits(str(desc.get('id') or '')) < pow_bits: return None, 'below %d bits of work' % pow_bits desc_ev = cast(NostrEvent, desc) if not binds(desc_ev, origin or ''): return None, 'transport key does not bind' return desc_ev, None
[docs]def call(path: str, cmd: str, timeout: float = 5.0, **args: Any) -> Any: """One request, one reply, one connection. Raises SidecarError for all of no sidecar, a sidecar that will not answer, and a sidecar that says no. For a caller outside a node (a script, the sidecar's own tests). Inside one, the sidecar is reached through the bridge - `Announce.call` - which also moves the link and checks the protocol the sidecar answered in. """ return jsonline_call(path, cmd, timeout=timeout, **args)[0]
from .types import AnnounceTotals
[docs]class Announce(Bridge): name: str = 'announce' reads: tuple[str, ...] = ('announce',) config_cls = AnnounceConfig SIDECAR = Sidecar( name='announce', protocol=PROTOCOL, upstream='py-libp2p gossipsub on %s' % TOPIC, standalone='fleetmesh-announce NODE_DIR [--bootstrap MULTIADDR]', probe='status') CONSEQUENCE = 'this node is discoverable over relays only' # Declared by the bridge from SIDECAR, under these names, with their # counters (announce_sidecar_reachable / _unreachable / _incompatible). SIDECAR_REACHABLE: str = 'announce.sidecar.reachable' SIDECAR_UNREACHABLE: str = 'announce.sidecar.unreachable' SIDECAR_INCOMPATIBLE: str = 'announce.sidecar.incompatible' DESCRIPTOR_PUBLISHED: str = 'announce.descriptor.published' PEERS_LEARNED: str = 'announce.peers.learned' CHANNELS = ( ch_mod.Channel( name=DESCRIPTOR_PUBLISHED, description='The node descriptor was republished to the gossipsub announce topic.', level=ch_mod.INFO, wire=lambda id, topic: {'id': id, 'topic': topic}, say='published {id} on {topic}', tags=('announce', 'descriptor')), ch_mod.Channel( name=PEERS_LEARNED, description='New descriptors heard on the announce topic were verified and learned.', level=ch_mod.INFO, wire=lambda learned, heard, topic: {'learned': learned, 'heard': heard, 'topic': topic}, say='{learned} new peers learned from {heard} heard on {topic}', tags=('announce', 'peers')), ) METRICS = ( mx_mod.Metric( name='announce_published', description='Times this node published its descriptor on the announce topic.', kind=mx_mod.COUNTER, counts=DESCRIPTOR_PUBLISHED, tags=('announce', 'descriptor')), mx_mod.Metric( name='announce_peers_learned', description='Times new peers were learned from the announce topic.', kind=mx_mod.COUNTER, counts=PEERS_LEARNED, tags=('announce', 'peers')), ) COMMANDS = ( Command( key='show', name='Announce Status', description='the announce topic: whether the sidecar is there, what it has carried, ' 'and what this node refused off the topic', tags=('announce', 'sidecar', 'status'), parameters=(), permission='announce/show', process=('sidecar',), run='state', reply='{enabled, socket, connected, topic, peer_id, mesh_peers, published, ' 'heard, learned, sidecar}', reads=('totals',), ), ) # What the topic has carried since this node first ran - not since its # last upgrade: SNAPSHOT, so the lifecycle hands it to the next process. # Not `published`: the next process's link starts unknown and the sidecar # must hear the descriptor again (it may have restarted too). STORES = ( st.Store(key='totals', description='descriptors carried up the topic, learned, and echoed back', kind=st.COUNTERS, shape=AnnounceTotals, persist=st.SNAPSHOT), ) # Restartable like every bridge; snapshot v1 is the shape of `totals`. LIFECYCLE = lifecycle_mod.Lifecycle(restart=True, snapshot=1) NEEDS = ( Need('peers', 'peers'), Need('directory', 'dir'), Need('timeout', 'timeout'), Need('descriptor', 'descriptor'), Need('public_descriptor', 'public_descriptor'), )
[docs] def __init__( self, peers: Any, directory: str, clock: Optional[Callable[[], int]] = None, timeout: float = 5.0, node: Optional[Any] = None, descriptor: Optional[Callable[[], Optional[NostrEvent]]] = None, public_descriptor: Optional[Callable[[], Optional[NostrEvent]]] = None, ) -> None: super().__init__(node, clock) # The node's own descriptor, read each publish through a getter # because it is built after this compartment is. self.descriptor: Callable[[], Optional[NostrEvent]] = descriptor or (lambda: getattr(node, 'descriptor', None)) # What may go on the public topic: the public copy, or None when the # node is fully private (plan D2) and announces nothing at all. A suite # that hands in only a descriptor gets that as the public copy. self.public_descriptor: Callable[[], Optional[NostrEvent]] = public_descriptor or self.descriptor self.peers: Any = peers self.dir: str = directory # Read once: a node that heard itself would otherwise cache itself. self.node_pk: str = getattr(getattr(node, 'ks', None), 'node_pk', '') or '' self.timeout: float = timeout self.published: Optional[str] = None # the descriptor id last handed to the sidecar self.totals: st.Counters # built and bound by the base, from STORES self.mesh_peers: int = 0 self.peer_id: Optional[str] = None
# The totals by name, as they were read before they were a store. @property def heard(self) -> int: """Descriptors the sidecar carried up.""" return self.totals['heard'] @heard.setter def heard(self, n: int) -> None: self.totals['heard'] = n @property def learned(self) -> int: """Of those, the ones this node did not already hold.""" return self.totals['learned'] @learned.setter def learned(self, n: int) -> None: self.totals['learned'] = n @property def echoed(self) -> int: """And the ones that were this node's own publish returning.""" return self.totals['echoed'] @echoed.setter def echoed(self, n: int) -> None: self.totals['echoed'] = n # ---- the bridge -----------------------------------------------------------
[docs] def endpoint(self) -> str: return self.path
[docs] def wanted(self) -> bool: return self.enabled
[docs] def forget(self) -> None: # A sidecar that went away, or a bridge restarted, must hear the # descriptor again when the link comes back. self.published = None
# ---- configuration -------------------------------------------------------- @property def enabled(self) -> bool: return self.settings().enabled @property def path(self) -> str: return socket_path(self.dir, self.settings().socket) @property def pow_bits(self) -> int: return self.settings().pow_bits # ---- the tick -------------------------------------------------------------
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: if not self.enabled: return [] did: list[tuple[Any, ...]] = [] try: did += self._publish(now) did += self._drain(now) except SidecarError: # The bridge has moved the link, said so once and refused it; # a missing sidecar costs this tick its gossip and nothing else. return did return did
def _publish(self, now: int) -> list[tuple[Any, ...]]: """The node's own descriptor on the topic, republished only when it changes.""" desc = self.public_descriptor() if not isinstance(desc, dict) or not desc.get('id'): # Fully private: nothing goes on the public announce topic. Forget # what was published so going public again republishes at once. self.published = None return [] if self.published == desc['id']: return [] self.call('publish', descriptor=desc) self.published = desc['id'] self.say(self.DESCRIPTOR_PUBLISHED, desc['id'][:12] + '…', TOPIC) return [('announce', desc['id'][:12] + '…', 'published on ' + TOPIC)] def _drain(self, now: int) -> list[tuple[Any, ...]]: """Everything the sidecar heard since the last tick, checked here again.""" res = self.call('drain', max=DRAIN_MAX) if not isinstance(res, dict): bad = 'drain answered something that is not an object' self._down(bad, now) raise SidecarError(bad) self.peer_id = res.get('peer_id') or self.peer_id try: self.mesh_peers = int(res.get('mesh_peers') or 0) except (TypeError, ValueError): self.mesh_peers = 0 learned = 0 for row in (res.get('heard') or [])[:DRAIN_MAX]: if not isinstance(row, dict): continue self.totals.incr('heard') payload = row.get('descriptor') origin = str(row.get('from') or '') # The same check the sidecar ran before propagating it. Run again # because a separate process is not a reason to believe an event, # and because the work floor is this node's policy and not its. desc, why = admit(payload, origin, now, pow_bits=self.pow_bits) if desc is None: subject = (payload.get('pubkey') if isinstance(payload, dict) else None) or None self.refusals.refuse('heard', str(why), subject=subject, detail={'from': origin[:16]}, at=now) continue if self.node_pk and desc['pubkey'] == self.node_pk: # A node hears its own publish: gossipsub runs the validator on # the way out, so the descriptor this node just put on the topic # comes straight back. Learning it would put this node in its own # peer cache, which is where `known()` is read from. Not a # refusal - it is the topic working. self.totals.incr('echoed') continue if self.peers.learn(desc): learned += 1 # A descriptor this node already holds at the same age is ordinary # on a flood topic and is not a refusal: the same descriptor # arrives many times over, which is what the topic is for. self.totals.incr('learned', learned) for reason, n in sorted((res.get('dropped') or {}).items()): # What the sidecar's own validator refused before propagating it. # Counted here so `meshnode refusals` shows one topic, not two. self.refusals.refuse('sidecar', str(reason)[:64], detail={'count': n}, at=now) if learned: self.say(self.PEERS_LEARNED, learned, len(res.get('heard') or []), TOPIC) return [('announce', '%d heard' % len(res.get('heard') or []), '%d new' % learned)] if learned else [] # ---- being watched --------------------------------------------------------
[docs] def state(self) -> AnnounceStateReport: return { 'enabled': self.enabled, 'socket': self.path, 'connected': self.link == UP, 'topic': TOPIC, 'peer_id': self.peer_id, 'mesh_peers': self.mesh_peers, 'pow_bits': self.pow_bits, 'published': self.published, 'heard': self.heard, 'learned': self.learned, 'echoed': self.echoed, }
[docs] def trouble(self) -> list[str]: # Down or incompatible is the bridge's to say, the same way for every # sidecar; an empty topic is announce's own. out = super().trouble() if out or not self.wanted(): return out if self.link == UP and not self.mesh_peers: return ['on %s with no gossipsub peers: nothing hears this node there' % TOPIC] return []