Source code for fleetmesh_core.storage.sync

"""sync/1.0: carrying reconciliation between two nodes under a grant.

This is where the pieces meet. fleetmesh_core/storage/negentropy.py finds which events two nodes
differ on; fleetmesh_core/storage/store.py says which events are in the set; a grant says how much
capacity the peer has and which events it is allowed to send at all. A sync
session is the only place all three are true at once, and the only place the
counters a receipt reports actually come from.

Three rules are enforced here rather than described:

  the interest contract   an event delivered outside the interest filter is an
                          S3 finding, not an inefficiency. It is how a peer
                          would make its traffic somebody else's problem.
  the budget              sync_ranges, events_in and bytes_in are grant limits.
                          They are counted as the session runs, so the receipt
                          is a record of what happened rather than an estimate.
  duplicates              an event delivered that the receiver already held is
                          counted. A peer with working reconciliation sits near
                          zero; dup_ratio is what tells a broken one from a
                          replaying one.

Messages are DIDComm plaintext bodies as schemas/didcomm/sync/ defines them.
Negentropy frames travel hex-encoded, which is what NIP-77 specifies and what
messages.json now pins - the schema had said "hex or base64", which is two wire
formats and therefore none.

Standard library only.
"""
from __future__ import annotations

import json
from typing import Any, Callable, Mapping, Optional, TypedDict

from . import negentropy, store

from ..wire import schema_files

from ..accounting import window
from ..wire.interfaces import (
    GrantPayload,
    GrantProtocol,
    EventLike,
    MeshStoreProtocol,
    NostrEvent,
    NostrFilter,
    ReceiptBody,
)

CONSTANTS: Mapping[str, Any] = schema_files.load_constants()
MESSAGES: Mapping[str, Any] = schema_files.load_messages()
PIURI: str = 'https://fleetmesh.org/sync/1.0'
FRAME_ENCODING: str = MESSAGES['protocols'][PIURI]['frame_encoding']
assert FRAME_ENCODING == 'hex'

COUNTERS: tuple[str, ...] = (
    'conn_open', 'req', 'events_in', 'bytes_in', 'bytes_out',
    'blob_bytes', 'errors', 'dups_sent', 'sync_ranges'
)
# One deliver carries at most this much event JSON. The nostr transport
# reassembles at most 64 fragments of 24000 bytes, so about 1.5 MB reaches a
# peer in one message, and an event costs 1.06x inside the delivery envelope
# that carries the sender's hop, plus the base64 the envelope encryption adds.
# What does not fit is left for the next session, which negentropy asks for
# again. A responder that ignored this would emit fragments nobody can
# reassemble and wait out the receiver's timeout.
DELIVER_MAX_BYTES: int = 768 * 1024


ENVELOPE_ATTACHMENT: str = 'delivery-envelope'
assert ENVELOPE_ATTACHMENT in MESSAGES['protocols'][PIURI]['attachments']


[docs]def delivered(message: Mapping[str, Any]) -> int: """How many events a deliver actually carries. The envelopes are the carrier, so they are what is counted. body.count is the sender's claim about its own message and a meter measures what arrived, not what was claimed; body.events is the in-process form, before the node layer moves them into the attachment. Counting the body alone is how delivered events briefly went unmetered the moment they stopped riding there. """ for att in message.get('attachments') or []: if att.get('id') == ENVELOPE_ATTACHMENT: envs = (att.get('data', {}).get('json') or {}).get('envelopes') if envs is not None: return len(envs) return len(message.get('body', {}).get('events') or [])
[docs]def wire_bytes(message: Mapping[str, Any]) -> int: """What the body costs on the wire, before DIDComm enveloping. The envelope adds a constant per message; what a grant meters is the payload.""" return len(json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode())
[docs]class Grant: """The part of a decrypted kind 30801 a sync session needs. scope is {direction: filter}. A grant with no push scope forbids the subject from sending anything, which is what a pull-only interest means. """
[docs] def __init__( self, event_id: str, tier: str, limits: Mapping[str, Any], scope: Optional[Mapping[str, Any]] = None, window: int = 3600, ) -> None: self.event_id: str = event_id self.tier: str = tier self.limits: Mapping[str, Any] = dict(limits) self.scope: Mapping[str, Any] = scope or {} self.window: int = window
[docs] @classmethod def from_payload(cls, payload: GrantPayload | Mapping[str, Any], event_id: str) -> Grant: """Build from a decrypted kind 30801 payload, which is what fleetmesh_core/peering/peering.py produces. The two modules meet here and nowhere else.""" return cls(event_id, payload['tier'], payload['limits'], payload.get('scope'), payload.get('window', 3600))
[docs] def permits(self, event: EventLike, direction: str = 'push') -> bool: filt = self.scope.get(direction) if filt is None: return False return store.matches(event, filt)
[docs] def exceeded(self, counters: Mapping[str, Any]) -> dict[str, float]: """Which limits this session's counters have gone past, and by how much. Reported as overrun so fleetmesh_core/accounting/window.py can classify it unchanged. A limit of zero is a limit, not an absence: the window prices any use over it as an infinite overrun, and this said the opposite by skipping it, so a grant with `blob_bytes` 0 - the specification's own example - was priced S3 by the ladder and refused by nobody.""" out: dict[str, float] = {} for name, limit in self.limits.items(): if name == 'dup_ratio' or limit is None: continue used = counters.get(window.counter_of(name), 0) if used > limit: out[name] = round(used / limit - 1, 6) if limit else float('inf') return out
[docs]class Session: """One side of one sync/1.0 session over one shard. The initiator reconciles, asks for what it lacks, and inserts what arrives. The responder answers frames and serves events its grant permits it to send. Neither trusts the other: the initiator checks every delivered event against its own interest filter, and the responder refuses to serve outside the scope its own grant declared. """
[docs] def __init__( self, my_store: MeshStoreProtocol, grant: GrantProtocol, shard: str, interest: NostrFilter | Mapping[str, Any], since: int = 0, until: int = 2 ** 31, now: Optional[int] = None, is_initiator: bool = True, verify: Optional[Callable[[NostrEvent], bool]] = None, ) -> None: # An event is what its author signed or it is nothing, and the store # says so itself: its interface takes an independently authenticated # event and never checks one. On the mesh path this session is that # authenticator - nothing else between a peer's deliver and the store # looks at a signature. The parameter exists for the suites that # exercise the reconciliation arithmetic over synthetic corpora, where # signing thousands of events in pure Python would cost minutes and # prove nothing about the arithmetic. A node never passes it. self.verify: Callable[[NostrEvent], bool] = verify or store.verify_event self.store: MeshStoreProtocol = my_store self.grant: GrantProtocol = grant self.shard: str = shard self.interest: NostrFilter | Mapping[str, Any] = interest self.since: int = since self.until: int = until # The caller's clock. Optional at construction so a session can be # built for inspection without one, but never defaulted: every use # goes through the `now` property, which refuses to stand in with 0. self._now: Optional[int] = now self.is_initiator: bool = is_initiator self.counters: dict[str, int] = dict.fromkeys(COUNTERS, 0) self.findings: list[tuple[str, str]] = [] self.forged: list[NostrEvent] = [] # events delivered whose signatures did not verify: the S4's evidence self.thid: str = 'sync-%s' % shard self._neg: Optional[negentropy.Negentropy] = None self._msg_no: int = 0 self.need: list[str] = [] self.have: list[str] = []
# ---- message plumbing ------------------------------------------------ @property def now(self) -> int: """The instant this session runs at, supplied by whoever built it. Raised rather than defaulted: a session that ran at epoch 0 would enumerate against an expiration nobody meant and write a receipt for window 0, and both would look valid. A missing clock is a programming error, so it is reported as one at the point of use. """ if self._now is None: raise ValueError('now is required') return self._now @now.setter def now(self, value: int) -> None: self._now = value def _msg(self, kind: str, body: Mapping[str, Any]) -> Mapping[str, Any]: self._msg_no += 1 return { 'id': '%s-%d' % (self.thid, self._msg_no), 'type': '%s/%s' % (PIURI, kind), 'thid': self.thid, 'body': body, } def _items(self) -> list[tuple[int, str]]: return self.store.items_range(self.interest, self.since, self.until, self.now) def _finding(self, severity: str, why: str) -> None: self.findings.append((severity, why)) # ---- the session -----------------------------------------------------
[docs] def open(self) -> Mapping[str, Any]: """sync/1.0/open, carrying the first negentropy frame.""" self._neg = negentropy.Negentropy(self._items(), is_client=True) frame = self._neg.initiate() self.counters['conn_open'] += 1 self.counters['req'] += 1 body = { 'shard': self.shard, 'filter': self.interest, 'since': self.since, 'until': self.until, 'initial_frame': frame.hex(), } m = self._msg('open', body) self.counters['bytes_out'] += wire_bytes(m) return m
[docs] def on_open(self, message: Mapping[str, Any]) -> Mapping[str, Any]: """Responder. Checks the interest against its own grant before answering. An interest wider than the scope the grant declared is refused rather than trimmed: quietly serving a narrower set than was asked for looks to the asker like missing data, which is the failure this protocol exists to make impossible to confuse with anything else. """ body = message['body'] self.counters['bytes_in'] += wire_bytes(message) self.counters['req'] += 1 if self.grant.scope.get('pull') is not None and body['filter'] != self.grant.scope['pull']: self._finding('S3', 'interest %r is not the granted pull scope' % body['filter']) return self._msg('done', {'status': 'aborted', 'events_transferred': 0}) self.interest = body['filter'] self.since, self.until = body.get('since', 0), body.get('until', 2 ** 31) self._neg = negentropy.Negentropy(self._items(), is_client=False) return self._frame_reply(bytes.fromhex(body['initial_frame']))
def _frame_reply(self, frame: bytes) -> Mapping[str, Any]: assert self._neg is not None reply, _, _ = self._neg.reconcile(frame) ranges = len(negentropy.decode_message(frame)) self.counters['sync_ranges'] += ranges if reply is None: m = self._msg('done', {'status': 'in_sync', 'events_transferred': 0}) else: m = self._msg('range', { 'range_id': '%s-%d' % (self.shard, self._msg_no), 'since': self.since, 'until': self.until, 'frame': reply.hex(), }) self.counters['bytes_out'] += wire_bytes(m) return m
[docs] def on_range(self, message: Mapping[str, Any]) -> Mapping[str, Any]: """Either side. Reconciles one frame and says what follows from it. The responder only ever answers frames. The initiator also accumulates the difference, and when reconciliation converges it asks for the half it is missing - which is the point at which this stops being a conversation about fingerprints and starts costing bandwidth. """ assert self._neg is not None self.counters['bytes_in'] += wire_bytes(message) frame = bytes.fromhex(message['body']['frame']) self.counters['sync_ranges'] += len(negentropy.decode_message(frame)) reply, have, need = self._neg.reconcile(frame) self.need = sorted(set(getattr(self, 'need', []) + need)) self.have = sorted(set(getattr(self, 'have', []) + have)) if reply is not None: body = { 'range_id': '%s-%d' % (self.shard, self._msg_no), 'since': self.since, 'until': self.until, 'frame': reply.hex(), } m = self._msg('range', body) elif self.is_initiator and self.need: m = self._msg('need', {'ids': self.need}) else: m = self._msg('done', {'status': 'in_sync', 'events_transferred': 0}) self.counters['bytes_out'] += wire_bytes(m) return m
[docs] def on_need(self, message: Mapping[str, Any]) -> Mapping[str, Any]: """Responder. Serves only what its grant permits it to send. An event it holds but may not push is simply not sent. Sending it would be the S3 the receiver would then file, so refusing is the responder protecting its own standing. """ self.counters['bytes_in'] += wire_bytes(message) wanted: list[str] = message['body']['ids'] events: list[Mapping[str, Any]] = [] refused: int = 0 held: int = 0 size: int = 0 for eid in wanted: ev = self.store.get_event(eid) if ev is None: continue if not self.grant.permits(ev, 'push'): refused += 1 continue size += len(json.dumps(ev, separators=(',', ':'))) if size > DELIVER_MAX_BYTES and events: held += 1 continue events.append(ev) if refused: self._finding('S0', '%d events withheld: outside the granted push scope' % refused) if held: self._finding('S0', '%d events held for the next session: a deliver is capped at %d bytes' % (held, DELIVER_MAX_BYTES)) m = self._msg('deliver', {'count': len(events), 'events': events}) self.counters['bytes_out'] += wire_bytes(m) return m
[docs] def on_deliver(self, message: Mapping[str, Any]) -> Mapping[str, Any]: """Initiator. Inserts what arrived, and prices what should not have. Every event is checked against the interest this session declared. One outside it is an S3: the interest contract is the whole basis on which a peer accepted the traffic. """ self.counters['bytes_in'] += wire_bytes(message) body = message['body'] events: list[NostrEvent] = body.get('events') or [] if body.get('count') is not None and body['count'] != len(events): self.counters['errors'] += 1 self._finding('S2', 'count says %d, %d events present' % (body['count'], len(events))) asked = set(getattr(self, 'need', [])) stored: int = 0 dups: int = 0 out_of_scope: list[str] = [] unsolicited: list[str] = [] for ev in events: if not store.matches(ev, self.interest): out_of_scope.append(ev['id']) continue if asked and ev['id'] not in asked: unsolicited.append(ev['id']) self.counters['events_in'] += 1 # An event is what its author signed or it is nothing. Nothing # else checks this on the way into the store: the relay face does # for what clients post, and this must for what peers deliver, or # a peer poisons the shard with what it likes under any name. if not self.verify(ev): self.counters['errors'] += 1 self.forged.append(ev) continue result = self.store.insert_event(ev, self.now) if result == 'duplicate': dups += 1 elif result == 'stored': stored += 1 self.counters['dups_sent'] += dups if self.forged: # Forged signatures are the ladder's S4, and the delivery envelope # the peer signed around each one is the evidence. self._finding('S4', '%d events whose signatures do not verify: %s' % (len(self.forged), ', '.join(e.get('id', '?')[:12] for e in self.forged))) if out_of_scope: self._finding('S3', '%d events outside the interest filter: %s' % (len(out_of_scope), ', '.join(i[:12] for i in out_of_scope))) if unsolicited: # In scope, but never asked for. The receiver is still charged the # bytes and the events_in against its own grant, so an unsolicited # delivery is an overrun in the only sense the ladder measures - # which is why it is priced as one rather than given a new class. self._finding('S1', '%d events delivered that were never requested' % len(unsolicited)) m = self._msg('done', {'status': 'complete', 'events_transferred': stored}) self.counters['bytes_out'] += wire_bytes(m) return m
[docs] def dup_ratio(self) -> float: delivered_count = self.counters['events_in'] return (self.counters['dups_sent'] / delivered_count) if delivered_count else 0.0
[docs] def receipt(self, seq: int = 0, prev: Optional[str] = None, at: Optional[int] = None) -> ReceiptBody: """The budget/1.0/receipt body this session's traffic accounts for. The counters are what happened, not a summary of it, which is the whole claim the accounting rests on. `at` defaults to the session clock; an explicit 0 is an explicit 0, not a request for the default. """ return { 'grant': self.grant.event_id, 'window': self.now // self.grant.window, 'seq': seq, 'prev': prev or '00' * 32, 'counters': {k: v for k, v in self.counters.items() if v}, 'at': at if at is not None else self.now, }
[docs]class SyncRunReport(TypedDict): transcript: list[Mapping[str, Any]] messages: int initiator: dict[str, int] responder: dict[str, int] findings: list[tuple[str, str]] need: list[str] dup_ratio: float bytes: int status: Optional[str]
[docs]def run(initiator: Session, responder: Session, max_messages: int = 64) -> SyncRunReport: """Drive a whole session and report it. Turn-taking is strict alternation: whoever did not send the last message handles it. Returns the transcript, both sides' counters, the findings either side would file, and what it cost. A session that has not finished inside max_messages is itself the finding. """ handlers = ('open', 'range', 'need', 'deliver') m = initiator.open() transcript: list[Mapping[str, Any]] = [m] side: Session = responder finished: bool = False for _ in range(max_messages): kind = m['type'].rsplit('/', 1)[1] if kind == 'done': finished = True break if kind not in handlers: side._finding('S2', 'unexpected sync message %r' % kind) break m = getattr(side, 'on_' + kind)(m) transcript.append(m) side = initiator if side is responder else responder if not finished: initiator._finding('S2', 'session did not terminate in %d messages' % max_messages) return { 'transcript': transcript, 'messages': len(transcript), 'initiator': dict(initiator.counters), 'responder': dict(responder.counters), 'findings': initiator.findings + responder.findings, 'need': sorted(getattr(initiator, 'need', [])), 'dup_ratio': initiator.dup_ratio(), 'bytes': initiator.counters['bytes_out'] + responder.counters['bytes_out'], 'status': transcript[-1]['body'].get('status'), }