Source code for fleetmesh_core.peering.coverage

"""Kind 30803: what a node says it holds, and the one door through the rule
about what it may say.

A coverage claim is how data discovery happens at all - a node declares which
shards it holds, as a filter plus a completeness claim and a NIP-77
fingerprint over the set. `asserted` is falsifiable and `best-effort` promises
nothing, and the specification forbids asserting completeness over a shard
held only through mesh replication, because that is how third-party
replication launders unverified peer data into an authoritative record.

The door through that rule is the handover, and it lives here because it is
not a separate mechanism: it is the one shape of `asserted` a node may
publish over a set it did not receive first-hand, and every rule about it is
a rule about what a claim may say. The node that holds the shard first-hand
signs over the set it has already published, names one replica, and the
replica carries that statement in its own claim.

Three rules keep the door narrow, and all three are decidable by a reader
holding the claim and nothing else:

  the handed set     the replica asserts exactly what it was handed, over a
                     range that had closed when it was signed for
  shared liability   a handover is the signer's own asserted claim over the
                     same set, so a hole in it is a hole in theirs
  no signer reports  a hole filed by anyone who signed for the set is the
                     signer's own fault, and the report would be the
                     mechanism for laundering it onto the replica

The third is why a claim carries the whole CHAIN and not one link. A withheld
event handed on twice would otherwise let the node that withheld it report
the hole against the node two links down - liable for it by its own
signature, and unbarred. Every signer travels so that every signer is barred.

Withdrawal is here for the same reason the claim is: a claim is addressable,
so taking one back is republishing it at the same coordinate saying
best-effort with an expiration, and a node that stopped republishing has not
taken anything back.
"""
from __future__ import annotations

import json
from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, cast

from ..crypto import crypto

from ..storage import replication, store

from ..wire import schema_files
from ..wire.interfaces import EventLike, NostrFilter, UnsignedNostrEvent
from ..wire.wire_types import HandoverLink
from ..wire import spelling

KINDS: Mapping[str, Any] = schema_files.load_kinds()['kinds']

COVERAGE: int = 30803
COMPLETENESS: tuple[str, ...] = ('asserted', 'best-effort')


[docs]def coverage_claim( pubkey: str, shard_id: str, filt: NostrFilter, completeness: str, event_ids: Iterable[str], now: int, since: Optional[int] = None, until: Optional[int] = None, handover: Optional[Mapping[str, Any] | Sequence[Mapping[str, Any]]] = None, expiration: Optional[int] = None, ) -> UnsignedNostrEvent: """A kind 30803 over the events the node holds for one shard. The fingerprint is NIP-77's own, over the ids, and the count travels beside it. `asserted` is a falsifiable claim; `best-effort` is not reportable. `handover` is the signed statement from the node that held the shard first-hand, and it is what lets a replica say `asserted` at all. It is carried inline in the content for the reason a report carries its evidence inline: a reader hashes it and checks the signature and resolves nothing. It is a *chain*, ordered from the first-hand root to this claim, because a single link is not enough to say who answers for the set. A withheld event handed on twice would otherwise let the node that withheld it report the hole against the node two links down - liable for the hole and unbarred from reporting it, which is the attack the bar exists to stop, laundered through one intermediary. Every signer in the chain is carried, so every signer can be barred. A single artefact is accepted and means a chain of one. Whether the claim then satisfies the chain's terms is check_coverage's business, not this builder's. """ if completeness not in COMPLETENESS: raise ValueError('completeness must be one of %s' % (COMPLETENESS,)) ids = list(event_ids) tags: list[list[str]] = [ ['d', shard_id], ['filter', crypto.canonical(filt).decode()], ] if since is not None: tags.append(['since', str(since)]) if until is not None: tags.append(['until', str(until)]) tags += [ ['completeness', completeness], ['fingerprint', replication.fingerprint(ids), str(now)], ['count', str(len(ids))], ] if expiration is not None: tags.append(['expiration', str(int(expiration))]) content = '' if handover is not None: chain = [dict(h) for h in ([handover] if isinstance(handover, Mapping) else list(handover))] tags.append(['handover', handover_id(chain[-1])]) content = crypto.canonical({'handover': chain}).decode() return store.make_event(pubkey, now, COVERAGE, tags, content)
[docs]def withdraw_coverage( claim: EventLike, now: int, lapses_in: int, ) -> UnsignedNostrEvent: """The claim that takes an asserted one back, for the same shard. A coverage claim is addressable, so a republication at the same `d` replaces what was there and relays drop the one it supersedes. The withdrawal is that republication, saying `best-effort` - which promises nothing and is not reportable - and carrying a NIP-40 expiration so it lapses rather than outliving the node that made it. A node that simply deleted its claim would leave every reader that already held one with no way to learn it had been taken back. The set is not restated: a withdrawal is about what this node will still stand behind, and the answer is nothing. It keeps the filter and the range so a reader can see which claim was withdrawn. """ d, filt = store.first_tag(claim, 'd'), store.first_tag(claim, 'filter') if d is None or len(d) < 2 or filt is None or len(filt) < 2: # Without both there is no shard to withdraw from: a claim that # passed check_coverage always has them, so this is a caller error. raise ValueError('a coverage claim carries a d and a filter tag') tags: list[list[str]] = [['d', d[1]], ['filter', filt[1]]] for name in ('since', 'until'): t = store.first_tag(claim, name) if t and len(t) > 1: tags.append([name, t[1]]) tags += [['completeness', 'best-effort'], ['expiration', str(int(now) + int(lapses_in))]] return store.make_event(claim['pubkey'], int(now), COVERAGE, tags, '')
[docs]def check_coverage(claim: EventLike, node_class: Optional[str] = None) -> list[str]: """Structural checks, plus the one class rule. Returns a list of problems.""" out: list[str] = [] if claim.get('kind') != COVERAGE: return ['not a kind 30803'] if node_class == 'leaf': out.append('a leaf MUST NOT publish coverage claims') for req in KINDS[str(COVERAGE)]['required_tags']: if not store.first_tag(claim, req): out.append('missing required tag: %s' % req) c = store.first_tag(claim, 'completeness') if c and (len(c) < 2 or c[1] not in COMPLETENESS): out.append('completeness must be asserted or best-effort') f = store.first_tag(claim, 'filter') if f: try: json.loads(f[1]) except (IndexError, ValueError): out.append('filter is not JSON') fp = store.first_tag(claim, 'fingerprint') if fp and (len(fp) < 2 or not spelling.is_hex(fp[1], 32)): out.append('fingerprint is not a 16-byte NIP-77 fingerprint') h = store.first_tag(claim, 'handover') if h is None: if claim.get('content'): out.append('content must be empty') return out if len(h) < 2: out.append('handover tag carries no id') return out chain = handover_chain(claim) if not chain: out.append('handover tag names %s and the content carries no chain ending in it' % h[1][:16]) return out if len(chain) > HANDOVER_MAX_CHAIN: out.append('a handover chain of %d is longer than %d' % (len(chain), HANDOVER_MAX_CHAIN)) return out for link in chain: out += check_handover(link) if out: return out out += check_chain(chain) if out: return out art = chain[-1] if c and len(c) > 1 and c[1] != 'asserted': out.append('a handover is only meaningful under an asserted claim') if art['to'] != claim.get('pubkey'): out.append('the handover names %s and this claim is by %s' % (art['to'][:16], (claim.get('pubkey') or '?')[:16])) d = store.first_tag(claim, 'd') if d and len(d) > 1 and d[1] != art['shard']: out.append('the handover is for shard %s and this claim is for %s' % (art['shard'], d[1])) if f and len(f) > 1 and _canonical_filter(f[1]) != art['filter']: out.append('the claim filter is not the handed-over filter') since, until = store.first_tag(claim, 'since'), store.first_tag(claim, 'until') if art.get('since') is None and since is not None: out.append('since narrows unbounded handover') if art.get('since') is not None and (not since or len(since) < 2 or store.ascii_int(since[1]) != art['since']): out.append('the claim range must begin where the handover does') if not until or len(until) < 2: out.append('a handed-over claim must carry until: the upstream signed for a bounded set') elif store.ascii_int(until[1]) != art['until']: out.append('the claim range must end where the handover does') if fp and len(fp) > 1 and fp[1] != art['fingerprint']: out.append('the claim fingerprint is not the handed-over fingerprint: ' 'a replica asserts exactly the set it was handed') cnt = store.first_tag(claim, 'count') if cnt and len(cnt) > 1 and store.ascii_int(cnt[1]) != art['count']: out.append('the claim count is not the handed-over count') return out
def _canonical_filter(text: str) -> str: try: return crypto.canonical(json.loads(text)).decode() except ValueError: return text # ------------------------------------------------------------- handover ---- HANDOVER_MAX_CHAIN: int = 8 HANDOVER_FIELDS: tuple[str, ...] = ('from', 'to', 'shard', 'claim', 'filter', 'since', 'until', 'fingerprint', 'count', 'at')
[docs]def handover_id(art: Mapping[str, Any]) -> str: """SHA-256 of the canonical form with `sig` excluded: the same construction as a receipt's chain link, so a handover is cited by the same 32 bytes whether it sits in a claim's content or in a report's evidence.""" from ..accounting import accounting return accounting.link(art)
[docs]def handover( sk_hex: str, to_pubkey: str, claim: EventLike, now: int, ) -> HandoverLink: """Sign over the set an asserted claim of ours covers, to one named peer. Everything a reader needs to check the replica's claim against is copied out of ours, so the artefact stands alone: filter, range, fingerprint and count. `until` is the range the set is complete over and `at` is when we said so; a claim that extends past `until` is asserting something we never signed for. A handover is signed by the node that holds the shard and names the node that will, and those may not be the same key. """ if claim.get('kind') != COVERAGE: raise ValueError('a handover is over a kind 30803') if not reportable(claim): raise ValueError('only an asserted claim can be handed over: ' 'best-effort promises nothing to pass on') frm = crypto.pubkey(sk_hex) if claim.get('pubkey') != frm: raise ValueError('a handover is signed by the claim publisher') if to_pubkey == frm: raise ValueError('a node cannot hand a shard to itself') f, d = store.first_tag(claim, 'filter'), store.first_tag(claim, 'd') fp, cnt = store.first_tag(claim, 'fingerprint'), store.first_tag(claim, 'count') since, until = store.first_tag(claim, 'since'), store.first_tag(claim, 'until') if not (f and d and fp and cnt): raise ValueError('the claim is missing filter, d, fingerprint or count') if not until or len(until) < 2: raise ValueError('a handover needs a bounded range: the claim carries no until') until_val = int(until[1]) at_val = int(now) body = { 'from': frm, 'to': to_pubkey, 'shard': d[1], 'claim': claim['id'], 'filter': _canonical_filter(f[1]), 'since': int(since[1]) if since and len(since) > 1 else None, 'until': until_val, 'fingerprint': fp[1], 'count': int(cnt[1]), 'at': at_val, } if until_val > at_val: raise ValueError('a handover cannot vouch for a range that has not closed') return cast(HandoverLink, dict(body, sig=crypto.schnorr_sign(bytes.fromhex(handover_id(body)), sk_hex)))
[docs]def handover_message( chain: Mapping[str, Any] | Sequence[Mapping[str, Any]], thid: str, mid: Optional[str] = None, ) -> Mapping[str, Any]: """The sync/1.0/handover that carries a chain to its replica. Sent after a done whose status is complete or in_sync, on that session's thread. The body is the chain and nothing else, ordered from the first-hand root to the link this message makes; the DIDComm from and to are the DIDs of that last link, so the message says nothing the signature does not already say. The receiver will carry the whole chain in its own claim, so the whole chain has to travel. """ links = [dict(h) for h in ([chain] if isinstance(chain, Mapping) else list(chain))] art = links[-1] return { 'id': mid or 'handover-%s-%s' % (art['shard'], art['to'][-8:]), 'type': 'https://fleetmesh.org/sync/1.0/handover', 'from': 'did:nostr:%s' % art['from'], 'to': ['did:nostr:%s' % art['to']], 'thid': thid, 'body': {'chain': links}, }
[docs]def verify_handover(art: Mapping[str, Any], signer: Optional[str] = None) -> bool: """The signature under `from`, and under `signer` if one is asked for.""" try: if signer is not None and art.get('from') != signer: return False return crypto.schnorr_verify(bytes.fromhex(handover_id(art)), art['from'], art['sig']) except (KeyError, ValueError, TypeError): return False
[docs]def check_handover(art: Mapping[str, Any]) -> list[str]: """Structural. Returns problems; an empty list is a well-formed signed handover.""" out: list[str] = [] for k in HANDOVER_FIELDS: if k not in art: out.append('handover is missing %s' % k) if out: return out if not spelling.is_hex(art['from'], 64) or not spelling.is_hex(art['to'], 64): out.append('handover from and to must be 32-byte hex pubkeys') if art['from'] == art['to']: out.append('a node cannot hand a shard to itself') if not isinstance(art['fingerprint'], str) or len(art['fingerprint']) != 32: out.append('handover fingerprint is not a 16-byte NIP-77 fingerprint') if not isinstance(art['until'], int) or not isinstance(art['at'], int): out.append('handover until and at must be integers') elif art['until'] > art['at']: out.append('a handover cannot vouch for a range that had not closed when it was signed') if art.get('since') is not None and not isinstance(art['since'], int): out.append('handover since must be an integer or null') if not isinstance(art['count'], int) or art['count'] < 0: out.append('handover count must be a non-negative integer') if out: return out if not verify_handover(art): out.append('handover signature does not verify under from') return out
[docs]def handover_chain(claim: EventLike) -> list[HandoverLink]: """The chain a claim carries, ordered from the first-hand root, or []. Held to the tag: the last link must hash to the id the tag names, or this is not the chain the tag is about. """ h = store.first_tag(claim, 'handover') body = claim.get('content') or '' if not h or len(h) < 2 or not body.startswith('{'): return [] try: doc = json.loads(body) except ValueError: return [] chain = doc.get('handover') if isinstance(chain, dict): # a chain of one, written as {id: artefact} chain = [chain.get(h[1])] if not isinstance(chain, list) or not chain or not all(isinstance(a, dict) for a in chain): return [] if handover_id(chain[-1]) != h[1]: return [] return [cast(HandoverLink, dict(a)) for a in chain]
[docs]def check_chain(chain: Sequence[Mapping[str, Any]]) -> list[str]: """Is this an unbroken chain about one set? Returns problems. Two rules and both are about what a reader can decide alone. Each link hands to the node that signs the next, so the chain is connected and a reader can follow it back to whoever held the shard first-hand. And every link names the same set, which is what makes the chain about one thing: an intermediary that had handed on something narrower or wider would be passing along a claim nobody above it ever made. """ out: list[str] = [] for i in range(len(chain) - 1): if chain[i]['to'] != chain[i + 1]['from']: out.append('the chain breaks: %s handed to %s and the next link is signed by %s' % (chain[i]['from'][:16], chain[i]['to'][:16], chain[i + 1]['from'][:16])) first = chain[0] for link in chain[1:]: if any(link[k] != first[k] for k in ('shard', 'filter', 'since', 'until', 'fingerprint', 'count')): out.append('a link hands on a different set from the one it was handed') break seen: list[str] = [] for link in chain: if link['from'] in seen: out.append('%s appears twice in the chain' % link['from'][:16]) break seen.append(link['from']) return out
[docs]def inline_handover(claim: EventLike) -> Optional[HandoverLink]: """The last link of the chain: the handover made to this claim's publisher.""" chain = handover_chain(claim) return chain[-1] if chain else None
[docs]def hole_in_handover( art: Mapping[str, Any], event: EventLike, held: Callable[[str], bool], ) -> Optional[list[str]]: """Evidence of a hole in a handover, against its signer. A handover is the signer's own asserted claim over the set it names, so the same event that is a hole in the replica's claim is a hole in the upstream's handover. `held` is what the *signer* holds, which only a reader syncing with the signer can answer; the replica's own shelves say nothing about it. """ if event.get('created_at', 0) >= art['until']: return None if art.get('since') is not None and event.get('created_at', 0) < art['since']: return None if not store.matches(event, json.loads(art['filter'])) or held(event.get('id', '')): return None return ['evidence', 'event', event.get('id', '')]
[docs]def barred_reporter(claim: EventLike, reporter: str) -> Optional[str]: """Why this reporter may not file a hole against this claim, or None. The signer of a claim's handover signed for the same set the claim asserts. A hole it files is an event it held and did not pass on, or one it never held and vouched for anyway; either way the fault is its own and the report is the mechanism for laundering it onto the replica. Decidable by any reader from the claim alone. """ for link in handover_chain(claim): if link.get('from') == reporter: return ('the reporter signed a handover this claim rests on: ' 'a hole in the handed set is the signer\'s own') return None
[docs]def reportable(claim: EventLike) -> bool: """Only an asserted claim can be reported against; best-effort promises nothing.""" c = store.first_tag(claim, 'completeness') return c is not None and len(c) > 1 and c[1] == 'asserted'
[docs]def hole( claim: EventLike, event: EventLike, held: Callable[[str], bool], ) -> Optional[list[str]]: """Evidence of a hole in an asserted claim, or None. An event inside the claim's filter and time range that the claimant does not hold is the evidence tag a report carries. Against best-effort there is no hole, because nothing was promised. """ if not reportable(claim): return None f_tag = store.first_tag(claim, 'filter') if not f_tag or len(f_tag) < 2: return None filt = json.loads(f_tag[1]) since, until = store.first_tag(claim, 'since'), store.first_tag(claim, 'until') if since and len(since) > 1 and event.get('created_at', 0) < int(since[1]): return None if until and len(until) > 1 and event.get('created_at', 0) >= int(until[1]): return None if not store.matches(event, filt) or held(event.get('id', '')): return None return ['evidence', 'event', event.get('id', '')]