Source code for fleetmesh_node.planes.store.sync.core

"""The SyncPeer compartment: construction, the metered send/receive every
role shares, the session table, and the compartment contract. The roles are
in initiator.py / responder.py; the handover in handover.py."""
from __future__ import annotations

import threading
from typing import Any, Callable, Mapping, Optional

from fleetmesh_core.storage import sync
from fleetmesh_node.channels import bus as eventbus_mod
from fleetmesh_core.identity.did import did_nostr
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Handler, Need, Outbound, WireProtocol
from .envelope import KEEP, RECENT, SESSIONS_CEILING, SESSION_IDLE, SyncError
from .handover import HandoverMixin
from .initiator import InitiatorMixin
from .responder import ResponderMixin
from .shape import SyncSessionResult
from .types import SyncStateReport


from .types import SyncTotals


def _log(key: str, what: str, bound: int) -> st.Store:
    return st.Store(key=key, description='%s, newest last' % what, kind=st.RING, shape=tuple, bound=bound)


import dataclasses
from fleetmesh_node.config.parameter import Parameter, ParamBlock


[docs]@dataclasses.dataclass(frozen=True) class SyncConfig(ParamBlock): key = 'sync' name = 'sync' DERIVED: bool = False PARAMS = ( Parameter( key='sessions_ceiling', name='Sessions Ceiling', description='Maximum concurrent open sync sessions answered', type=int, default=SESSIONS_CEILING, minimum=1, tags=('sync', 'concurrency'), ), Parameter( key='recent_kept', name='Recent Kept', description='Number of recent completed or refused sessions kept', type=int, default=RECENT, minimum=10, tags=('sync', 'history'), ), ) sessions_ceiling: int = SESSIONS_CEILING recent_kept: int = RECENT
[docs]class SyncPeer(InitiatorMixin, ResponderMixin, HandoverMixin, WireProtocol): """Both roles of sync/1.0 for one node, as a compartment. Its refusals are the sessions it would not open - a grant that does not cover one, a peer with too many already, a node over its own ceiling - and the deliveries it would not store. Those lived in three separate lists that only the CLI knew how to read. """ name: str = 'sync' requires: tuple[str, ...] = ('budget', 'strain') piuris: tuple[str, ...] = (sync.PIURI,) reads: tuple[str, ...] = ('handover', 'coverage') config_cls = SyncConfig live: tuple[str, ...] = ('handover',) CHANNELS = ( Channel('handed', 'Handover signed and offered to peer', level=INFO, say='handed {shard} to {peer} ({events} events)'), Channel('held', 'Handover accepted from peer', level=INFO, say='held {shard} handed over by {peer} ({events} events, {chain_len} in the chain)'), ) METRICS = ( Metric(name='sync_handovers_offered', description='Times shard coverage was handed over.', kind=COUNTER, counts='handed', tags=('sync', 'handover')), Metric(name='sync_handovers_accepted', description='Times shard coverage was accepted.', kind=COUNTER, counts='held', tags=('sync', 'handover')), ) HANDLERS = ( Handler('open', run='handle_open', doc='answer inbound sync/1.0 open session request'), Handler('handover', run='handle_handover', doc='accept inbound sync/1.0 handover envelope'), Handler('range', run='handle_range', doc='answer inbound sync/1.0 range request'), Handler('need', run='handle_need', doc='answer inbound sync/1.0 need request'), Handler('done', run='handle_done', doc='answer inbound sync/1.0 done session finish'), ) OUTBOUND = ( Outbound('open', reply=('range', 'deliver', 'done'), doc='open sync session against peer'), Outbound('range', reply=('range', 'have', 'need', 'deliver', 'done'), doc='query or answer negentropy range'), Outbound('have', reply=('need', 'done'), doc='answer negentropy have with event ids'), Outbound('need', reply=('deliver', 'done'), doc='request event delivery for missing ids'), Outbound('deliver', reply=('range', 'need', 'done'), doc='deliver events in signed envelope attachment'), Outbound('done', doc='conclude sync session with final status'), Outbound('handover', doc='hand over verified shard coverage chain'), ) COMMANDS = ( Command( key='show', name='Sync', description='sessions open, refused and aborted, and what could not be sent', tags=('sync', 'status'), permission='sync/show', run='state', reply='{open, refused, aborted, completed, unsent, malformed, handed, held}', ), ) # What it keeps. aborted, refused and completed were plain lists appended # once per session and never trimmed - memory that grew with every peer's # sessions for the life of the process - while nothing read more than # their length and the newest few. Now a ring of the recent ones, with the # length kept in `totals` so `state` reports the same numbers. The four # that were already cut to 200 by hand (`del x[:-200]`) keep that bound # as a declaration instead, and `state` still reports their length. STORES = ( st.Store(key='totals', description='sessions since the process started, by how they ended', kind=st.COUNTERS, shape=SyncTotals), _log('aborted', 'sessions a peer stopped answering, as (peer, thid)', RECENT), _log('refused', 'sessions this node would not open, as (peer, thid, why)', RECENT), _log('completed', 'sessions that ended, as (peer, role, result)', RECENT), _log('rejected', 'messages without the shape of their type, as (peer, kind, why)', KEEP), _log('unsent', 'messages no relay would take, as (peer, type, why, at)', KEEP), _log('handed', 'handovers this node signed and sent, as (peer, shard, fingerprint, at)', KEEP), _log('held', 'handovers this node accepted, as (peer, shard, at)', KEEP), # REFUSE: an open session dropped for room is a peer left waiting on a # done that never comes. A full table refuses the open instead, with # a done and a reason, like the per-peer cap in front of it. st.Store(key='sessions', description='sessions this node is answering, with when each last moved', kind=st.TABLE, shape=tuple, bound=SESSIONS_CEILING, evict=st.REFUSE, keyed_by='thid'), # OLDEST, which the hand trim this replaces meant to be and was not: it # cut a set to an arbitrary 2048 of its members, not the newest. A thid # forgotten here only charges a late message on a long-finished # session to the wrong side. st.Store(key='_initiated', description='sessions this node opened, by thid', kind=st.SET, shape=str, bound=4096, keyed_by='thid'), # OLDEST is the harmless direction: a memo forgotten is a handover # offered twice, and the receiver fingerprints the second copy as the # first (see HandoverMixin._unoffer). st.Store(key='_offered', description='the fingerprint last handed over, per peer and shard', kind=st.TABLE, shape=str, bound=4096, keyed_by='peer and shard'), ) NEEDS = ( Need('keystore', 'ks'), Need('my_store', 'stores.events'), Need('transport'), Need('grant_store', 'stores.grants'), Need('budget'), Need('session_cls', 'cfg', field='_session_cls'), Need('strain'), Need('accounts', 'stores.accounts'), Need('events'), Need('discovery', late=True), Need('presence', late=True), Need('reporter', late=True), )
[docs] def bind(self, discovery: Any = None, presence: Any = None) -> None: """The two soft dependencies that tick after sync: discovery holds the coverage claims a handover rests on, presence says whether a peer that went quiet had announced it was leaving. Both are built after sync, so they are bound once all three exist; either may stay None (a bare SyncPeer in a suite), and the handover and partition paths check for that.""" self.discovery = discovery self.presence = presence
[docs] def start(self) -> None: if self.reporter is not None and self.reporter.on_session not in self.on_session: self.on_session.append(self.reporter.on_session) if self.events is not None: events = self.events self.on_session.append(lambda *a: events.emit(eventbus_mod.TOPIC_SESSION_COMPLETED, *a))
[docs] def __init__( self, keystore: Any, my_store: Any, transport: Any, grant_store: Any, clock: Optional[Callable[[], int]] = None, budget: Optional[Any] = None, session_cls: Optional[Any] = None, strain: Optional[Any] = None, node: Optional[Any] = None, accounts: Optional[Any] = None, events: Optional[Any] = None, discovery: Optional[Any] = None, presence: Optional[Any] = None, reporter: Optional[Any] = None, ) -> None: super().__init__(node, clock) self.strain: Optional[Any] = strain # what this node has survived, or None self.accounts: Optional[Any] = accounts # where handover chains are kept, or None self.discovery = discovery # bound after assembly (ticks after sync) self.presence = presence # bound after assembly (ticks after sync) self.reporter = reporter self.events = events self.session_cls: Any = session_cls or sync.Session # the bad peer plugs in here self.ks: Any = keystore self.store: Any = my_store self.transport: Any = transport self.grants: Any = grant_store self.budget: Optional[Any] = budget # meters every message both ways, if present self.on_session: list[Callable[[str, str, SyncSessionResult], Any]] = [] # callbacks (peer, role, result) when a session ends # The stores were built and bound by the base (see STORES). self.totals: st.Counters self.aborted: st.Ring[tuple[str, str]] # (peer, thid) sessions a peer stopped answering self.refused: st.Ring[tuple[str, Optional[str], str]] # (peer, thid, why) sessions the grant did not cover self.sessions: st.Table[str, tuple[str, Any, int]] # thid -> (peer_pk, Session, last activity) # The sessions this node opened, by thid. A message is charged to the # session's initiator on both sides, and this is how the wire knows # which side that was. Kept after the session ends: a late message on # a finished thid is still that session's, and still the initiator's. self._initiated: st.Members[str] self.rejected: st.Ring[tuple[str, str, str]] # (peer, kind, why) messages that did not have the shape of their type # (peer, type, why, at). The time is what stops one message lost in a # restart warning for the rest of the process: this peering completed # thirty-four sessions while still reporting it. self.unsent: st.Ring[tuple[str, str, str, int]] # (peer, type, why, at) self.completed: st.Ring[tuple[str, str, SyncSessionResult]] # (peer_pk, role, counters, findings, status) self.handed: st.Ring[tuple[str, str, str, int]] # (peer_pk, shard, fingerprint, at) handovers this node signed and sent self.held: st.Ring[tuple[str, str, int]] # (peer_pk, shard, at) handovers this node accepted self._offered: st.Table[tuple[str, str], str] # (peer_pk, shard) -> fingerprint last handed, so equality twice is one offer self._lock: threading.Lock = threading.Lock()
def _record(self, ring: 'st.Ring[Any]', entry: tuple[Any, ...]) -> None: """One session's end: kept in its ring, and counted in `totals`.""" ring.append(entry) self.totals.incr(ring.store.key) def _grant(self, peer_pk: str, direction: str) -> Any: row = self.grants.get(peer_pk, direction) if row is None or row['status'] != 'live': raise SyncError('no live %s grant with %s' % (direction, peer_pk[:16])) return sync.Grant.from_payload(row['payload'], row['envelope_id']) def _send(self, m: Mapping[str, Any], peer_pk: str) -> dict[str, tuple[bool, str]]: msg_dict = dict(m) msg_dict['from'] = self.ks.did msg_dict['to'] = [did_nostr(peer_pk)] try: res: dict[str, tuple[bool, str]] = self.wire_client.send(peer_pk, msg_dict) except Exception as e: # Nothing left the node: not metered, and said where an operator # can see it rather than left as the peer's timeout. self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1], '%s: %s' % (type(e).__name__, e), self.clock())) return {} # Meter what a relay took, not what was attempted. A deliver every # relay refused never reached the peer; counting it claims traffic the # peer will measure as absent, and the window prices that gap as a # misreport against the sender. Three machines showed it: 111 events # claimed, 37 arrived, S2 against an honest node. if res and not any(ok for ok, _ in res.values()): why = '; '.join(n for _, n in res.values() if n)[:160] or 'refused' self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1], why, self.clock())) return res if self.budget is not None: self.budget.observe_out(peer_pk, msg_dict, i_initiated=msg_dict.get('thid') in self._initiated) return res def _seen(self, peer_pk: str, m: Mapping[str, Any]) -> None: if self.budget is not None: self.budget.observe_in(peer_pk, m, i_initiated=m.get('thid') in self._initiated) # ---- the compartment contract -------------------------------------------------
[docs] def state(self) -> SyncStateReport: return { 'open': len(self.sessions), 'refused': self.totals['refused'], 'aborted': self.totals['aborted'], 'completed': self.totals['completed'], 'unsent': len(self.unsent), 'malformed': len(self.rejected), 'handed': [{'peer': p, 'shard': sh, 'at': at} for p, sh, _, at in self.handed.last(20)], 'held': [{'peer': p, 'shard': sh, 'at': at} for p, sh, at in self.held.last(20)], }
[docs] def trouble(self) -> list[str]: out: list[str] = [] recent = self.refusals.within(self.TROUBLE_WINDOW, self.clock()) if recent: out.append('%d sync session(s) refused: %s' % (len(recent), '; '.join(sorted({r.why for r in recent}))[:120])) still = [u for u in self.unsent if self.clock() - u[-1] <= self.TROUBLE_WINDOW] if still: # A relay that refused a message says nothing to the peer, which # waits on an envelope that never finishes. Only the sender knows. out.append('%d message(s) no relay would take - the peer is waiting on an ' 'envelope that will never finish' % len(still)) return out
def _expire(self, now: int) -> list[str]: """Forget sessions whose peer went quiet. No finding: quiet is not lying.""" with self._lock: stale = [t for t, v in self.sessions.items() if now - v[2] > SESSION_IDLE] for t in stale: self.sessions.pop(t, None) return stale def _finish(self, thid: str, peer_pk: str, sess: Any, status: Any) -> None: with self._lock: self.sessions.pop(thid, None) self._record(self.completed, (peer_pk, 'responder', {'status': status, 'counters': dict(sess.counters), 'findings': list(sess.findings), 'thid': thid}))