"""sync/1.0: carrying reconciliation between two nodes under a grant.
This is where the pieces meet. fleetmesh_core/storage/negentropy.py finds which events two nodes
differ on; fleetmesh_core/storage/store.py says which events are in the set; a grant says how much
capacity the peer has and which events it is allowed to send at all. A sync
session is the only place all three are true at once, and the only place the
counters a receipt reports actually come from.
Three rules are enforced here rather than described:
the interest contract an event delivered outside the interest filter is an
S3 finding, not an inefficiency. It is how a peer
would make its traffic somebody else's problem.
the budget sync_ranges, events_in and bytes_in are grant limits.
They are counted as the session runs, so the receipt
is a record of what happened rather than an estimate.
duplicates an event delivered that the receiver already held is
counted. A peer with working reconciliation sits near
zero; dup_ratio is what tells a broken one from a
replaying one.
Messages are DIDComm plaintext bodies as schemas/didcomm/sync/ defines them.
Negentropy frames travel hex-encoded, which is what NIP-77 specifies and what
messages.json now pins - the schema had said "hex or base64", which is two wire
formats and therefore none.
Standard library only.
"""
from __future__ import annotations
import json
from typing import Any, Callable, Mapping, Optional, TypedDict
from . import negentropy, store
from ..wire import schema_files
from ..accounting import window
from ..wire.interfaces import (
GrantPayload,
GrantProtocol,
EventLike,
MeshStoreProtocol,
NostrEvent,
NostrFilter,
ReceiptBody,
)
CONSTANTS: Mapping[str, Any] = schema_files.load_constants()
MESSAGES: Mapping[str, Any] = schema_files.load_messages()
PIURI: str = 'https://fleetmesh.org/sync/1.0'
FRAME_ENCODING: str = MESSAGES['protocols'][PIURI]['frame_encoding']
assert FRAME_ENCODING == 'hex'
COUNTERS: tuple[str, ...] = (
'conn_open', 'req', 'events_in', 'bytes_in', 'bytes_out',
'blob_bytes', 'errors', 'dups_sent', 'sync_ranges'
)
# One deliver carries at most this much event JSON. The nostr transport
# reassembles at most 64 fragments of 24000 bytes, so about 1.5 MB reaches a
# peer in one message, and an event costs 1.06x inside the delivery envelope
# that carries the sender's hop, plus the base64 the envelope encryption adds.
# What does not fit is left for the next session, which negentropy asks for
# again. A responder that ignored this would emit fragments nobody can
# reassemble and wait out the receiver's timeout.
DELIVER_MAX_BYTES: int = 768 * 1024
ENVELOPE_ATTACHMENT: str = 'delivery-envelope'
assert ENVELOPE_ATTACHMENT in MESSAGES['protocols'][PIURI]['attachments']
[docs]def delivered(message: Mapping[str, Any]) -> int:
"""How many events a deliver actually carries.
The envelopes are the carrier, so they are what is counted. body.count is
the sender's claim about its own message and a meter measures what
arrived, not what was claimed; body.events is the in-process form, before
the node layer moves them into the attachment. Counting the body alone is
how delivered events briefly went unmetered the moment they stopped
riding there.
"""
for att in message.get('attachments') or []:
if att.get('id') == ENVELOPE_ATTACHMENT:
envs = (att.get('data', {}).get('json') or {}).get('envelopes')
if envs is not None:
return len(envs)
return len(message.get('body', {}).get('events') or [])
[docs]def wire_bytes(message: Mapping[str, Any]) -> int:
"""What the body costs on the wire, before DIDComm enveloping. The envelope
adds a constant per message; what a grant meters is the payload."""
return len(json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode())
[docs]class Grant:
"""The part of a decrypted kind 30801 a sync session needs.
scope is {direction: filter}. A grant with no push scope forbids the subject
from sending anything, which is what a pull-only interest means.
"""
[docs] def __init__(
self,
event_id: str,
tier: str,
limits: Mapping[str, Any],
scope: Optional[Mapping[str, Any]] = None,
window: int = 3600,
) -> None:
self.event_id: str = event_id
self.tier: str = tier
self.limits: Mapping[str, Any] = dict(limits)
self.scope: Mapping[str, Any] = scope or {}
self.window: int = window
[docs] @classmethod
def from_payload(cls, payload: GrantPayload | Mapping[str, Any], event_id: str) -> Grant:
"""Build from a decrypted kind 30801 payload, which is what
fleetmesh_core/peering/peering.py produces. The two modules meet here and nowhere else."""
return cls(event_id, payload['tier'], payload['limits'],
payload.get('scope'), payload.get('window', 3600))
[docs] def permits(self, event: EventLike, direction: str = 'push') -> bool:
filt = self.scope.get(direction)
if filt is None:
return False
return store.matches(event, filt)
[docs] def exceeded(self, counters: Mapping[str, Any]) -> dict[str, float]:
"""Which limits this session's counters have gone past, and by how much.
Reported as overrun so fleetmesh_core/accounting/window.py can classify it unchanged.
A limit of zero is a limit, not an absence: the window prices any use
over it as an infinite overrun, and this said the opposite by skipping
it, so a grant with `blob_bytes` 0 - the specification's own example -
was priced S3 by the ladder and refused by nobody."""
out: dict[str, float] = {}
for name, limit in self.limits.items():
if name == 'dup_ratio' or limit is None:
continue
used = counters.get(window.counter_of(name), 0)
if used > limit:
out[name] = round(used / limit - 1, 6) if limit else float('inf')
return out
[docs]class Session:
"""One side of one sync/1.0 session over one shard.
The initiator reconciles, asks for what it lacks, and inserts what arrives.
The responder answers frames and serves events its grant permits it to send.
Neither trusts the other: the initiator checks every delivered event against
its own interest filter, and the responder refuses to serve outside the
scope its own grant declared.
"""
[docs] def __init__(
self,
my_store: MeshStoreProtocol,
grant: GrantProtocol,
shard: str,
interest: NostrFilter | Mapping[str, Any],
since: int = 0,
until: int = 2 ** 31,
now: Optional[int] = None,
is_initiator: bool = True,
verify: Optional[Callable[[NostrEvent], bool]] = None,
) -> None:
# An event is what its author signed or it is nothing, and the store
# says so itself: its interface takes an independently authenticated
# event and never checks one. On the mesh path this session is that
# authenticator - nothing else between a peer's deliver and the store
# looks at a signature. The parameter exists for the suites that
# exercise the reconciliation arithmetic over synthetic corpora, where
# signing thousands of events in pure Python would cost minutes and
# prove nothing about the arithmetic. A node never passes it.
self.verify: Callable[[NostrEvent], bool] = verify or store.verify_event
self.store: MeshStoreProtocol = my_store
self.grant: GrantProtocol = grant
self.shard: str = shard
self.interest: NostrFilter | Mapping[str, Any] = interest
self.since: int = since
self.until: int = until
# The caller's clock. Optional at construction so a session can be
# built for inspection without one, but never defaulted: every use
# goes through the `now` property, which refuses to stand in with 0.
self._now: Optional[int] = now
self.is_initiator: bool = is_initiator
self.counters: dict[str, int] = dict.fromkeys(COUNTERS, 0)
self.findings: list[tuple[str, str]] = []
self.forged: list[NostrEvent] = [] # events delivered whose signatures did not verify: the S4's evidence
self.thid: str = 'sync-%s' % shard
self._neg: Optional[negentropy.Negentropy] = None
self._msg_no: int = 0
self.need: list[str] = []
self.have: list[str] = []
# ---- message plumbing ------------------------------------------------
@property
def now(self) -> int:
"""The instant this session runs at, supplied by whoever built it.
Raised rather than defaulted: a session that ran at epoch 0 would
enumerate against an expiration nobody meant and write a receipt for
window 0, and both would look valid. A missing clock is a programming
error, so it is reported as one at the point of use.
"""
if self._now is None:
raise ValueError('now is required')
return self._now
@now.setter
def now(self, value: int) -> None:
self._now = value
def _msg(self, kind: str, body: Mapping[str, Any]) -> Mapping[str, Any]:
self._msg_no += 1
return {
'id': '%s-%d' % (self.thid, self._msg_no),
'type': '%s/%s' % (PIURI, kind),
'thid': self.thid,
'body': body,
}
def _items(self) -> list[tuple[int, str]]:
return self.store.items_range(self.interest, self.since, self.until, self.now)
def _finding(self, severity: str, why: str) -> None:
self.findings.append((severity, why))
# ---- the session -----------------------------------------------------
[docs] def open(self) -> Mapping[str, Any]:
"""sync/1.0/open, carrying the first negentropy frame."""
self._neg = negentropy.Negentropy(self._items(), is_client=True)
frame = self._neg.initiate()
self.counters['conn_open'] += 1
self.counters['req'] += 1
body = {
'shard': self.shard,
'filter': self.interest,
'since': self.since,
'until': self.until,
'initial_frame': frame.hex(),
}
m = self._msg('open', body)
self.counters['bytes_out'] += wire_bytes(m)
return m
[docs] def on_open(self, message: Mapping[str, Any]) -> Mapping[str, Any]:
"""Responder. Checks the interest against its own grant before answering.
An interest wider than the scope the grant declared is refused rather
than trimmed: quietly serving a narrower set than was asked for looks
to the asker like missing data, which is the failure this protocol
exists to make impossible to confuse with anything else.
"""
body = message['body']
self.counters['bytes_in'] += wire_bytes(message)
self.counters['req'] += 1
if self.grant.scope.get('pull') is not None and body['filter'] != self.grant.scope['pull']:
self._finding('S3', 'interest %r is not the granted pull scope' % body['filter'])
return self._msg('done', {'status': 'aborted', 'events_transferred': 0})
self.interest = body['filter']
self.since, self.until = body.get('since', 0), body.get('until', 2 ** 31)
self._neg = negentropy.Negentropy(self._items(), is_client=False)
return self._frame_reply(bytes.fromhex(body['initial_frame']))
def _frame_reply(self, frame: bytes) -> Mapping[str, Any]:
assert self._neg is not None
reply, _, _ = self._neg.reconcile(frame)
ranges = len(negentropy.decode_message(frame))
self.counters['sync_ranges'] += ranges
if reply is None:
m = self._msg('done', {'status': 'in_sync', 'events_transferred': 0})
else:
m = self._msg('range', {
'range_id': '%s-%d' % (self.shard, self._msg_no),
'since': self.since,
'until': self.until,
'frame': reply.hex(),
})
self.counters['bytes_out'] += wire_bytes(m)
return m
[docs] def on_range(self, message: Mapping[str, Any]) -> Mapping[str, Any]:
"""Either side. Reconciles one frame and says what follows from it.
The responder only ever answers frames. The initiator also accumulates
the difference, and when reconciliation converges it asks for the half
it is missing - which is the point at which this stops being a
conversation about fingerprints and starts costing bandwidth.
"""
assert self._neg is not None
self.counters['bytes_in'] += wire_bytes(message)
frame = bytes.fromhex(message['body']['frame'])
self.counters['sync_ranges'] += len(negentropy.decode_message(frame))
reply, have, need = self._neg.reconcile(frame)
self.need = sorted(set(getattr(self, 'need', []) + need))
self.have = sorted(set(getattr(self, 'have', []) + have))
if reply is not None:
body = {
'range_id': '%s-%d' % (self.shard, self._msg_no),
'since': self.since,
'until': self.until,
'frame': reply.hex(),
}
m = self._msg('range', body)
elif self.is_initiator and self.need:
m = self._msg('need', {'ids': self.need})
else:
m = self._msg('done', {'status': 'in_sync', 'events_transferred': 0})
self.counters['bytes_out'] += wire_bytes(m)
return m
[docs] def on_need(self, message: Mapping[str, Any]) -> Mapping[str, Any]:
"""Responder. Serves only what its grant permits it to send.
An event it holds but may not push is simply not sent. Sending it would
be the S3 the receiver would then file, so refusing is the responder
protecting its own standing.
"""
self.counters['bytes_in'] += wire_bytes(message)
wanted: list[str] = message['body']['ids']
events: list[Mapping[str, Any]] = []
refused: int = 0
held: int = 0
size: int = 0
for eid in wanted:
ev = self.store.get_event(eid)
if ev is None:
continue
if not self.grant.permits(ev, 'push'):
refused += 1
continue
size += len(json.dumps(ev, separators=(',', ':')))
if size > DELIVER_MAX_BYTES and events:
held += 1
continue
events.append(ev)
if refused:
self._finding('S0', '%d events withheld: outside the granted push scope' % refused)
if held:
self._finding('S0', '%d events held for the next session: a deliver is capped at %d bytes'
% (held, DELIVER_MAX_BYTES))
m = self._msg('deliver', {'count': len(events), 'events': events})
self.counters['bytes_out'] += wire_bytes(m)
return m
[docs] def on_deliver(self, message: Mapping[str, Any]) -> Mapping[str, Any]:
"""Initiator. Inserts what arrived, and prices what should not have.
Every event is checked against the interest this session declared. One
outside it is an S3: the interest contract is the whole basis on which
a peer accepted the traffic.
"""
self.counters['bytes_in'] += wire_bytes(message)
body = message['body']
events: list[NostrEvent] = body.get('events') or []
if body.get('count') is not None and body['count'] != len(events):
self.counters['errors'] += 1
self._finding('S2', 'count says %d, %d events present' % (body['count'], len(events)))
asked = set(getattr(self, 'need', []))
stored: int = 0
dups: int = 0
out_of_scope: list[str] = []
unsolicited: list[str] = []
for ev in events:
if not store.matches(ev, self.interest):
out_of_scope.append(ev['id'])
continue
if asked and ev['id'] not in asked:
unsolicited.append(ev['id'])
self.counters['events_in'] += 1
# An event is what its author signed or it is nothing. Nothing
# else checks this on the way into the store: the relay face does
# for what clients post, and this must for what peers deliver, or
# a peer poisons the shard with what it likes under any name.
if not self.verify(ev):
self.counters['errors'] += 1
self.forged.append(ev)
continue
result = self.store.insert_event(ev, self.now)
if result == 'duplicate':
dups += 1
elif result == 'stored':
stored += 1
self.counters['dups_sent'] += dups
if self.forged:
# Forged signatures are the ladder's S4, and the delivery envelope
# the peer signed around each one is the evidence.
self._finding('S4', '%d events whose signatures do not verify: %s'
% (len(self.forged), ', '.join(e.get('id', '?')[:12] for e in self.forged)))
if out_of_scope:
self._finding('S3', '%d events outside the interest filter: %s'
% (len(out_of_scope), ', '.join(i[:12] for i in out_of_scope)))
if unsolicited:
# In scope, but never asked for. The receiver is still charged the
# bytes and the events_in against its own grant, so an unsolicited
# delivery is an overrun in the only sense the ladder measures -
# which is why it is priced as one rather than given a new class.
self._finding('S1', '%d events delivered that were never requested'
% len(unsolicited))
m = self._msg('done', {'status': 'complete', 'events_transferred': stored})
self.counters['bytes_out'] += wire_bytes(m)
return m
[docs] def dup_ratio(self) -> float:
delivered_count = self.counters['events_in']
return (self.counters['dups_sent'] / delivered_count) if delivered_count else 0.0
[docs] def receipt(self, seq: int = 0, prev: Optional[str] = None, at: Optional[int] = None) -> ReceiptBody:
"""The budget/1.0/receipt body this session's traffic accounts for.
The counters are what happened, not a summary of it, which is the whole
claim the accounting rests on. `at` defaults to the session clock; an
explicit 0 is an explicit 0, not a request for the default.
"""
return {
'grant': self.grant.event_id,
'window': self.now // self.grant.window,
'seq': seq,
'prev': prev or '00' * 32,
'counters': {k: v for k, v in self.counters.items() if v},
'at': at if at is not None else self.now,
}
[docs]class SyncRunReport(TypedDict):
transcript: list[Mapping[str, Any]]
messages: int
initiator: dict[str, int]
responder: dict[str, int]
findings: list[tuple[str, str]]
need: list[str]
dup_ratio: float
bytes: int
status: Optional[str]
[docs]def run(initiator: Session, responder: Session, max_messages: int = 64) -> SyncRunReport:
"""Drive a whole session and report it.
Turn-taking is strict alternation: whoever did not send the last message
handles it. Returns the transcript, both sides' counters, the findings
either side would file, and what it cost. A session that has not finished
inside max_messages is itself the finding.
"""
handlers = ('open', 'range', 'need', 'deliver')
m = initiator.open()
transcript: list[Mapping[str, Any]] = [m]
side: Session = responder
finished: bool = False
for _ in range(max_messages):
kind = m['type'].rsplit('/', 1)[1]
if kind == 'done':
finished = True
break
if kind not in handlers:
side._finding('S2', 'unexpected sync message %r' % kind)
break
m = getattr(side, 'on_' + kind)(m)
transcript.append(m)
side = initiator if side is responder else responder
if not finished:
initiator._finding('S2', 'session did not terminate in %d messages' % max_messages)
return {
'transcript': transcript,
'messages': len(transcript),
'initiator': dict(initiator.counters),
'responder': dict(responder.counters),
'findings': initiator.findings + responder.findings,
'need': sorted(getattr(initiator, 'need', [])),
'dup_ratio': initiator.dup_ratio(),
'bytes': initiator.counters['bytes_out'] + responder.counters['bytes_out'],
'status': transcript[-1]['body'].get('status'),
}