"""The SyncPeer compartment: construction, the metered send/receive every
role shares, the session table, and the compartment contract. The roles are
in initiator.py / responder.py; the handover in handover.py."""
from __future__ import annotations
import threading
from typing import Any, Callable, Mapping, Optional
from fleetmesh_core.storage import sync
from fleetmesh_node.channels import bus as eventbus_mod
from fleetmesh_core.identity.did import did_nostr
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Handler, Need, Outbound, WireProtocol
from .envelope import KEEP, RECENT, SESSIONS_CEILING, SESSION_IDLE, SyncError
from .handover import HandoverMixin
from .initiator import InitiatorMixin
from .responder import ResponderMixin
from .shape import SyncSessionResult
from .types import SyncStateReport
from .types import SyncTotals
def _log(key: str, what: str, bound: int) -> st.Store:
return st.Store(key=key, description='%s, newest last' % what, kind=st.RING, shape=tuple, bound=bound)
import dataclasses
from fleetmesh_node.config.parameter import Parameter, ParamBlock
[docs]@dataclasses.dataclass(frozen=True)
class SyncConfig(ParamBlock):
key = 'sync'
name = 'sync'
DERIVED: bool = False
PARAMS = (
Parameter(
key='sessions_ceiling', name='Sessions Ceiling',
description='Maximum concurrent open sync sessions answered',
type=int, default=SESSIONS_CEILING, minimum=1, tags=('sync', 'concurrency'),
),
Parameter(
key='recent_kept', name='Recent Kept',
description='Number of recent completed or refused sessions kept',
type=int, default=RECENT, minimum=10, tags=('sync', 'history'),
),
)
sessions_ceiling: int = SESSIONS_CEILING
recent_kept: int = RECENT
[docs]class SyncPeer(InitiatorMixin, ResponderMixin, HandoverMixin, WireProtocol):
"""Both roles of sync/1.0 for one node, as a compartment.
Its refusals are the sessions it would not open - a grant that does not
cover one, a peer with too many already, a node over its own ceiling - and
the deliveries it would not store. Those lived in three separate lists that
only the CLI knew how to read.
"""
name: str = 'sync'
requires: tuple[str, ...] = ('budget', 'strain')
piuris: tuple[str, ...] = (sync.PIURI,)
reads: tuple[str, ...] = ('handover', 'coverage')
config_cls = SyncConfig
live: tuple[str, ...] = ('handover',)
CHANNELS = (
Channel('handed', 'Handover signed and offered to peer', level=INFO,
say='handed {shard} to {peer} ({events} events)'),
Channel('held', 'Handover accepted from peer', level=INFO,
say='held {shard} handed over by {peer} ({events} events, {chain_len} in the chain)'),
)
METRICS = (
Metric(name='sync_handovers_offered', description='Times shard coverage was handed over.',
kind=COUNTER, counts='handed', tags=('sync', 'handover')),
Metric(name='sync_handovers_accepted', description='Times shard coverage was accepted.',
kind=COUNTER, counts='held', tags=('sync', 'handover')),
)
HANDLERS = (
Handler('open', run='handle_open', doc='answer inbound sync/1.0 open session request'),
Handler('handover', run='handle_handover', doc='accept inbound sync/1.0 handover envelope'),
Handler('range', run='handle_range', doc='answer inbound sync/1.0 range request'),
Handler('need', run='handle_need', doc='answer inbound sync/1.0 need request'),
Handler('done', run='handle_done', doc='answer inbound sync/1.0 done session finish'),
)
OUTBOUND = (
Outbound('open', reply=('range', 'deliver', 'done'), doc='open sync session against peer'),
Outbound('range', reply=('range', 'have', 'need', 'deliver', 'done'), doc='query or answer negentropy range'),
Outbound('have', reply=('need', 'done'), doc='answer negentropy have with event ids'),
Outbound('need', reply=('deliver', 'done'), doc='request event delivery for missing ids'),
Outbound('deliver', reply=('range', 'need', 'done'), doc='deliver events in signed envelope attachment'),
Outbound('done', doc='conclude sync session with final status'),
Outbound('handover', doc='hand over verified shard coverage chain'),
)
COMMANDS = (
Command(
key='show',
name='Sync',
description='sessions open, refused and aborted, and what could not be sent',
tags=('sync', 'status'),
permission='sync/show',
run='state',
reply='{open, refused, aborted, completed, unsent, malformed, handed, held}',
),
)
# What it keeps. aborted, refused and completed were plain lists appended
# once per session and never trimmed - memory that grew with every peer's
# sessions for the life of the process - while nothing read more than
# their length and the newest few. Now a ring of the recent ones, with the
# length kept in `totals` so `state` reports the same numbers. The four
# that were already cut to 200 by hand (`del x[:-200]`) keep that bound
# as a declaration instead, and `state` still reports their length.
STORES = (
st.Store(key='totals', description='sessions since the process started, by how they ended',
kind=st.COUNTERS, shape=SyncTotals),
_log('aborted', 'sessions a peer stopped answering, as (peer, thid)', RECENT),
_log('refused', 'sessions this node would not open, as (peer, thid, why)', RECENT),
_log('completed', 'sessions that ended, as (peer, role, result)', RECENT),
_log('rejected', 'messages without the shape of their type, as (peer, kind, why)', KEEP),
_log('unsent', 'messages no relay would take, as (peer, type, why, at)', KEEP),
_log('handed', 'handovers this node signed and sent, as (peer, shard, fingerprint, at)', KEEP),
_log('held', 'handovers this node accepted, as (peer, shard, at)', KEEP),
# REFUSE: an open session dropped for room is a peer left waiting on a
# done that never comes. A full table refuses the open instead, with
# a done and a reason, like the per-peer cap in front of it.
st.Store(key='sessions', description='sessions this node is answering, with when each last moved',
kind=st.TABLE, shape=tuple, bound=SESSIONS_CEILING, evict=st.REFUSE, keyed_by='thid'),
# OLDEST, which the hand trim this replaces meant to be and was not: it
# cut a set to an arbitrary 2048 of its members, not the newest. A thid
# forgotten here only charges a late message on a long-finished
# session to the wrong side.
st.Store(key='_initiated', description='sessions this node opened, by thid', kind=st.SET,
shape=str, bound=4096, keyed_by='thid'),
# OLDEST is the harmless direction: a memo forgotten is a handover
# offered twice, and the receiver fingerprints the second copy as the
# first (see HandoverMixin._unoffer).
st.Store(key='_offered', description='the fingerprint last handed over, per peer and shard',
kind=st.TABLE, shape=str, bound=4096, keyed_by='peer and shard'),
)
NEEDS = (
Need('keystore', 'ks'),
Need('my_store', 'stores.events'),
Need('transport'),
Need('grant_store', 'stores.grants'),
Need('budget'),
Need('session_cls', 'cfg', field='_session_cls'),
Need('strain'),
Need('accounts', 'stores.accounts'),
Need('events'),
Need('discovery', late=True),
Need('presence', late=True),
Need('reporter', late=True),
)
[docs] def bind(self, discovery: Any = None, presence: Any = None) -> None:
"""The two soft dependencies that tick after sync: discovery holds
the coverage claims a handover rests on, presence says whether a
peer that went quiet had announced it was leaving. Both are built
after sync, so they are bound once all three exist; either may stay
None (a bare SyncPeer in a suite), and the handover and partition
paths check for that."""
self.discovery = discovery
self.presence = presence
[docs] def start(self) -> None:
if self.reporter is not None and self.reporter.on_session not in self.on_session:
self.on_session.append(self.reporter.on_session)
if self.events is not None:
events = self.events
self.on_session.append(lambda *a: events.emit(eventbus_mod.TOPIC_SESSION_COMPLETED, *a))
[docs] def __init__(
self,
keystore: Any,
my_store: Any,
transport: Any,
grant_store: Any,
clock: Optional[Callable[[], int]] = None,
budget: Optional[Any] = None,
session_cls: Optional[Any] = None,
strain: Optional[Any] = None,
node: Optional[Any] = None,
accounts: Optional[Any] = None,
events: Optional[Any] = None,
discovery: Optional[Any] = None,
presence: Optional[Any] = None,
reporter: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
self.strain: Optional[Any] = strain # what this node has survived, or None
self.accounts: Optional[Any] = accounts # where handover chains are kept, or None
self.discovery = discovery # bound after assembly (ticks after sync)
self.presence = presence # bound after assembly (ticks after sync)
self.reporter = reporter
self.events = events
self.session_cls: Any = session_cls or sync.Session # the bad peer plugs in here
self.ks: Any = keystore
self.store: Any = my_store
self.transport: Any = transport
self.grants: Any = grant_store
self.budget: Optional[Any] = budget # meters every message both ways, if present
self.on_session: list[Callable[[str, str, SyncSessionResult], Any]] = [] # callbacks (peer, role, result) when a session ends
# The stores were built and bound by the base (see STORES).
self.totals: st.Counters
self.aborted: st.Ring[tuple[str, str]] # (peer, thid) sessions a peer stopped answering
self.refused: st.Ring[tuple[str, Optional[str], str]] # (peer, thid, why) sessions the grant did not cover
self.sessions: st.Table[str, tuple[str, Any, int]] # thid -> (peer_pk, Session, last activity)
# The sessions this node opened, by thid. A message is charged to the
# session's initiator on both sides, and this is how the wire knows
# which side that was. Kept after the session ends: a late message on
# a finished thid is still that session's, and still the initiator's.
self._initiated: st.Members[str]
self.rejected: st.Ring[tuple[str, str, str]] # (peer, kind, why) messages that did not have the shape of their type
# (peer, type, why, at). The time is what stops one message lost in a
# restart warning for the rest of the process: this peering completed
# thirty-four sessions while still reporting it.
self.unsent: st.Ring[tuple[str, str, str, int]] # (peer, type, why, at)
self.completed: st.Ring[tuple[str, str, SyncSessionResult]] # (peer_pk, role, counters, findings, status)
self.handed: st.Ring[tuple[str, str, str, int]] # (peer_pk, shard, fingerprint, at) handovers this node signed and sent
self.held: st.Ring[tuple[str, str, int]] # (peer_pk, shard, at) handovers this node accepted
self._offered: st.Table[tuple[str, str], str] # (peer_pk, shard) -> fingerprint last handed, so equality twice is one offer
self._lock: threading.Lock = threading.Lock()
def _record(self, ring: 'st.Ring[Any]', entry: tuple[Any, ...]) -> None:
"""One session's end: kept in its ring, and counted in `totals`."""
ring.append(entry)
self.totals.incr(ring.store.key)
def _grant(self, peer_pk: str, direction: str) -> Any:
row = self.grants.get(peer_pk, direction)
if row is None or row['status'] != 'live':
raise SyncError('no live %s grant with %s' % (direction, peer_pk[:16]))
return sync.Grant.from_payload(row['payload'], row['envelope_id'])
def _send(self, m: Mapping[str, Any], peer_pk: str) -> dict[str, tuple[bool, str]]:
msg_dict = dict(m)
msg_dict['from'] = self.ks.did
msg_dict['to'] = [did_nostr(peer_pk)]
try:
res: dict[str, tuple[bool, str]] = self.wire_client.send(peer_pk, msg_dict)
except Exception as e:
# Nothing left the node: not metered, and said where an operator
# can see it rather than left as the peer's timeout.
self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1],
'%s: %s' % (type(e).__name__, e), self.clock()))
return {}
# Meter what a relay took, not what was attempted. A deliver every
# relay refused never reached the peer; counting it claims traffic the
# peer will measure as absent, and the window prices that gap as a
# misreport against the sender. Three machines showed it: 111 events
# claimed, 37 arrived, S2 against an honest node.
if res and not any(ok for ok, _ in res.values()):
why = '; '.join(n for _, n in res.values() if n)[:160] or 'refused'
self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1], why, self.clock()))
return res
if self.budget is not None:
self.budget.observe_out(peer_pk, msg_dict, i_initiated=msg_dict.get('thid') in self._initiated)
return res
def _seen(self, peer_pk: str, m: Mapping[str, Any]) -> None:
if self.budget is not None:
self.budget.observe_in(peer_pk, m, i_initiated=m.get('thid') in self._initiated)
# ---- the compartment contract -------------------------------------------------
[docs] def state(self) -> SyncStateReport:
return {
'open': len(self.sessions),
'refused': self.totals['refused'],
'aborted': self.totals['aborted'],
'completed': self.totals['completed'],
'unsent': len(self.unsent),
'malformed': len(self.rejected),
'handed': [{'peer': p, 'shard': sh, 'at': at} for p, sh, _, at in self.handed.last(20)],
'held': [{'peer': p, 'shard': sh, 'at': at} for p, sh, at in self.held.last(20)],
}
[docs] def trouble(self) -> list[str]:
out: list[str] = []
recent = self.refusals.within(self.TROUBLE_WINDOW, self.clock())
if recent:
out.append('%d sync session(s) refused: %s'
% (len(recent), '; '.join(sorted({r.why for r in recent}))[:120]))
still = [u for u in self.unsent if self.clock() - u[-1] <= self.TROUBLE_WINDOW]
if still:
# A relay that refused a message says nothing to the peer, which
# waits on an envelope that never finishes. Only the sender knows.
out.append('%d message(s) no relay would take - the peer is waiting on an '
'envelope that will never finish' % len(still))
return out
def _expire(self, now: int) -> list[str]:
"""Forget sessions whose peer went quiet. No finding: quiet is not lying."""
with self._lock:
stale = [t for t, v in self.sessions.items() if now - v[2] > SESSION_IDLE]
for t in stale:
self.sessions.pop(t, None)
return stale
def _finish(self, thid: str, peer_pk: str, sess: Any, status: Any) -> None:
with self._lock:
self.sessions.pop(thid, None)
self._record(self.completed, (peer_pk, 'responder', {'status': status, 'counters': dict(sess.counters),
'findings': list(sess.findings), 'thid': thid}))