Source code for fleetmesh_node.planes.store.blob_wire.core

"""The blob_wire compartment: serve pulls for what this node holds, take
offers within the operator's caps, drop replicas when the grant ends.
"""
from __future__ import annotations

import base64
import hashlib
from typing import Any, Callable, Mapping, Optional

from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, TROUBLE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.config import BlobWireConfig
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.compartment import Command, Handler, Need, Outbound, WireProtocol
from .client import TransportBlobClient
from .intake import IntakeStore
from .messages import (CAP_DEFAULTS, DEFAULT_CHUNK_BYTES, HTTP_SCHEMES, MAX_CHUNK_BYTES, PIURI, PULL_TIMEOUT, blossom_url, make_chunk,
                       make_have, make_pull, make_refuse)


[docs]class BlobWire(WireProtocol): """Answers have and pull for blobs this node holds; takes offers under policy.""" name = 'blob_wire' piuris: tuple[str, ...] = (PIURI,) reads = ('blob_wire',) config_cls = BlobWireConfig live = ('blob_wire',) SCHEMA = IntakeStore.SCHEMA STORE_CLS = IntakeStore # builds its own; the base opens nothing for it (B5) CHANNELS = ( Channel('blob_served', 'Blob chunk or have served to peer', level=INFO, say='blob chunk served to peer {peer}', tags=('blob', 'wire')), Channel('offer_refused', 'Blob offer refused under policy', level=TROUBLE, say='blob offer refused from peer {peer}: {reason}', tags=('blob', 'wire')), ) METRICS = ( Metric(name='blob_wire_chunks_served', description='Times blob chunks were served.', kind=COUNTER, counts='blob_served', tags=('blob', 'wire')), Metric(name='blob_wire_offers_refused', description='Times blob offers were refused.', kind=COUNTER, counts='offer_refused', tags=('blob', 'wire')), ) STORES = ( st.Store(key='_inflight', description='concurrent blob transfers per peer', kind=st.TABLE, shape=int, bound=1024, keyed_by='peer', evict=st.LRU), ) COMMANDS = ( Command( key='held', name='Held Replicas', description='replicas taken over the transport, per peer', tags=('blob_wire', 'held'), parameters=( Parameter(key='peer', name='peer', description='peer pubkey to filter by', type=str, default=None, required=False), ), permission='blob_wire/held', run='cmd_held', reply='{peer: [{sha256, size, at}]}', ), Command( key='drop', name='Drop Peer Replicas', description='drop every replica held for a peer now', tags=('blob_wire', 'drop'), parameters=( Parameter(key='peer', name='peer', description='peer pubkey', type=str, default='', required=True), ), permission='blob_wire/drop', run='cmd_drop', reply='{peer, dropped}', ), ) HANDLERS = ( Handler('have', run='handle_have', reads=('blobs',), doc='answer have with held hashes'), Handler('pull', run='handle_pull', reads=('blobs',), doc='serve requested chunk of held blob'), Handler('offer', run='handle_offer', writes=('blobs',), doc='evaluate offer and pull if accepted'), Handler('chunk', run='handle_chunk', doc='handle unprompted chunk'), Handler('refuse', run='handle_refuse', doc='handle unprompted refuse'), ) OUTBOUND = ( Outbound('have', doc='query peer for held hashes or reply with held hashes'), Outbound('offer', reply=('have', 'refuse'), doc='offer blob to peer'), Outbound('pull', reply=('chunk', 'refuse'), doc='request chunk from peer'), Outbound('chunk', doc='serve chunk bytes to pull requester'), Outbound('refuse', doc='refuse unheld or disallowed pull/offer'), ) REPLICAS_DB: str = 'blob_replicas.sqlite' NEEDS = ( Need('blobs', 'stores.blobs'), Need('transport', 'transport'), Need('db_path', path='REPLICAS_DB'), Need('cfg', 'cfg', getter=True), Need('grants', 'stores.grants'), )
[docs] def __init__( self, blobs: Any, transport: Any, db_path: str, cfg: Callable[[], Mapping[str, Any]], grantees: Optional[Callable[[], list[str]]] = None, clock: Optional[Callable[[], int]] = None, node: Any = None, grants: Optional[Any] = None, ) -> None: super().__init__(node, clock) self.blobs = blobs self.transport = transport self.db = IntakeStore(db_path) self._cfg = cfg if grantees is None and grants is not None: self._grantees = lambda: grants.peers() else: self._grantees = grantees or (lambda: []) self.client = TransportBlobClient(self) self._inflight: st.Table[str, int] self.taken = 0 self.dropped = 0 self.refused = 0 self._closed = False
# ---- configuration ----------------------------------------------------------------
[docs] def rc(self) -> BlobWireConfig: """The operator's `blob_wire` block as it is now; it reads live.""" return BlobWireConfig.from_dict((self._cfg() or {}).get('blob_wire') or {})
[docs] def settings(self) -> BlobWireConfig: # the Compartment hook, through the injected cfg return self.rc()
[docs] def cap(self, name: str) -> int: v = getattr(self.rc(), name) return max(0, int(v)) if v is not None else CAP_DEFAULTS[name]
[docs] def chunk_bytes(self) -> int: v = self.rc().chunk_bytes return max(1, min(int(v), MAX_CHUNK_BYTES)) if v is not None else DEFAULT_CHUNK_BYTES
[docs] def accepts(self, peer: str) -> bool: pol = self.rc().accept_from if isinstance(pol, tuple): return peer in pol if pol == 'grantees': return peer in self._grantees() return False
[docs] def why_not(self, peer: str, size: int, now: int) -> Optional[str]: """The first reason an offer from this peer may not be taken, or None.""" if not self.accepts(peer): return 'not accepting replicas from this peer' if size > self.cap('max_blob_bytes'): return 'blob larger than max_blob_bytes' if self.db.total_for(peer) + size > self.cap('max_total_bytes'): return 'would exceed max_total_bytes for this peer' if self.db.today(peer, now) + size > self.cap('bytes_per_day'): return 'would exceed bytes_per_day for this peer' if self._inflight.get(peer, 0) >= self.cap('max_concurrent'): return 'max_concurrent pulls already in flight for this peer' return None
# ---- serving: what I hold, on request ------------------------------------------------
[docs] def handle_have(self, inbound: Any, sender: str) -> tuple[Any, ...]: m = getattr(inbound, 'message', inbound) peer = str(sender).lower() body = m.get('body', {}) held = [h for h in body.get('sha256', []) if self.blobs.has(h)] self.wire_client.reply(inbound, make_have(held), sender=peer) return ('blob_wire', 'have', peer[:16], len(held))
[docs] def handle_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]: return self._serve_pull(inbound, sender)
[docs] def handle_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]: return self._take_offer(inbound, sender)
[docs] def handle_chunk(self, inbound: Any, sender: str) -> tuple[Any, ...]: return ('ignored', 'chunk')
[docs] def handle_refuse(self, inbound: Any, sender: str) -> tuple[Any, ...]: return ('ignored', 'refuse')
def _serve_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]: m = getattr(inbound, 'message', inbound) peer = str(sender).lower() body = m['body'] h = str(body['sha256']).lower() data = self.blobs.get(h) if data is None: self.wire_client.reply(inbound, make_refuse(h, 'not held'), sender=peer) return ('blob_wire', 'pull refused', peer[:16], h[:16], 'not held') off, ln = int(body['offset']), int(body['length']) if off < 0 or ln <= 0 or off > len(data): self.wire_client.reply(inbound, make_refuse(h, 'range out of bounds'), sender=peer) return ('blob_wire', 'pull refused', peer[:16], h[:16], 'range') ln = min(ln, self.chunk_bytes()) self.wire_client.reply(inbound, make_chunk(h, off, data[off:off + ln]), sender=peer) return ('blob_wire', 'chunk', peer[:16], h[:16], off) # ---- taking: the receiver's pull loop --------------------------------------------------- def _take_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]: m = getattr(inbound, 'message', inbound) peer = str(sender).lower() body = m['body'] h = str(body['sha256']).lower() size = int(body['size']) now = self.clock() if self.blobs.has(h): # Already here, whoever brought it. Say so; pin it for this peer too. self.blobs.pin(h) if not self.db.holds(h): self.db.took(h, peer, 0, now) self.wire_client.reply(inbound, make_have([h]), sender=peer) return ('blob_wire', 'already held', peer[:16], h[:16]) why = self.why_not(peer, size, now) if why is not None: self.refused += 1 self.refusals.refuse('blob_wire', 'offer refused', subject=peer, detail={'blob': h[:16], 'why': why}, at=now) self.wire_client.reply(inbound, make_refuse(h, why), sender=peer) return ('blob_wire', 'offer refused', peer[:16], h[:16], why) chunk = max(1, min(int(body.get('chunk_bytes', DEFAULT_CHUNK_BYTES)), self.chunk_bytes())) self._inflight[peer] = self._inflight.get(peer, 0) + 1 try: data = self._pull(peer, h, size, chunk) except Exception as e: why = '%s: %s' % (type(e).__name__, str(e)[:120]) self.refused += 1 self.refusals.refuse('blob_wire', 'pull failed', subject=peer, detail={'blob': h[:16], 'why': why}, at=now) self.wire_client.reply(inbound, make_refuse(h, why), sender=peer) return ('blob_wire', 'pull failed', peer[:16], h[:16], why) finally: self._inflight[peer] -= 1 if self._inflight[peer] <= 0: self._inflight.pop(peer, None) self.blobs.put(data, content_type=str(body.get('type') or 'application/octet-stream'), uploaded_by='replica:' + peer, now=now) self.blobs.pin(h) self.db.took(h, peer, size, now) self.taken += 1 self.wire_client.reply(inbound, make_have([h]), sender=peer) return ('blob_wire', 'took', peer[:16], h[:16], size) def _pull(self, peer: str, h: str, size: int, chunk: int) -> bytes: hasher = hashlib.sha256() out = bytearray() off = 0 while off < size: want = min(chunk, size - off) reply = self.wire_client.request(peer, make_pull(h, off, want), timeout=PULL_TIMEOUT) if reply is None: raise TimeoutError('no chunk at offset %d' % off) rm = reply.message k = self.wire_client.kind_of(rm) if k == 'refuse': raise ValueError('source refused: %s' % rm.get('body', {}).get('why', '?')) if k != 'chunk': raise ValueError('expected chunk, got %s' % (k or rm.get('type'))) rb = rm['body'] if str(rb.get('sha256', '')).lower() != h or int(rb.get('offset', -1)) != off: raise ValueError('chunk for the wrong blob or offset') piece = base64.b64decode(rb['data'], validate=True) if not piece or len(piece) > want: raise ValueError('chunk of the wrong size at offset %d' % off) hasher.update(piece) out += piece off += len(piece) if hasher.hexdigest() != h: raise ValueError('sha256 mismatch after %d bytes' % off) return bytes(out) # ---- the compartment contract ---------------------------------------------------------------
[docs] def drop_peer(self, peer: str, now: Optional[int] = None) -> Mapping[str, Any]: """Let go of every replica held for a peer: unpin, delete, forget.""" peer = peer.lower() gone = self.db.forget_peer(peer) for h in gone: try: self.blobs.unpin(h) stat = self.blobs.stat(h) if stat and str(stat.get('uploaded_by') or '') == 'replica:' + peer: self.blobs.delete(h) except Exception: pass self.dropped += len(gone) return {'peer': peer, 'dropped': len(gone)}
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: did: list[tuple[Any, ...]] = [] if self._closed: return did live = set(self._grantees()) for peer in self.db.peers(): if peer not in live: r = self.drop_peer(peer, now) if r['dropped']: did.append(('blob_wire', 'grant ended, dropped', peer[:16], r['dropped'])) self.db.prune_days(now) return did
[docs] def state(self) -> Mapping[str, Any]: if self._closed: return {'stopped': True, 'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused} peers = self.db.peers() pol = self.rc().accept_from return {'accept_from': list(pol) if isinstance(pol, tuple) else pol, 'caps': {k: self.cap(k) for k in CAP_DEFAULTS}, 'peers': len(peers), 'held_bytes': sum(self.db.total_for(p) for p in peers), 'inflight': dict(self._inflight), 'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused}
[docs] def trouble(self) -> list[str]: if self._closed: return [] pol = self.rc().accept_from if pol != 'none' and pol and not all(self.cap(k) for k in ('bytes_per_day', 'max_blob_bytes', 'max_total_bytes')): return ['blob_wire.accept_from is set but a cap is 0, so every offer is refused: ' 'set bytes_per_day, max_blob_bytes, max_total_bytes and max_concurrent'] return []
[docs] def cmd_held(self, args: Mapping[str, Any]) -> Mapping[str, Any]: peers = [str(args['peer']).lower()] if args.get('peer') else self.db.peers() return {p: self.db.held_for(p) for p in peers}
[docs] def cmd_drop(self, args: Mapping[str, Any]) -> Mapping[str, Any]: return self.drop_peer(str(args['peer']))
[docs] def stop(self) -> Any: self._closed = True try: self.db.close() except Exception: pass super().stop() return None
[docs]def partner_transport_peers(node: Any) -> list[str]: """`peer:<pk>` for every grantee that has no Blossom URL in its descriptor. Peers with an HTTP endpoint take the Blossom path; this is for the rest. """ out: list[str] = [] grants = getattr(node, 'grants', None) peers = getattr(node, 'peers', None) if grants is None or peers is None: return out for pk in grants.peers(): try: d = peers.descriptor(pk) except Exception: d = None has_http = False for t in (d or {}).get('tags', []): if t and t[0] == 'endpoint' and len(t) > 2 and t[1] in HTTP_SCHEMES and blossom_url(t[2]): has_http = True break if not has_http: out.append(TransportBlobClient.PREFIX + pk) return out