"""The blob_wire compartment: serve pulls for what this node holds, take
offers within the operator's caps, drop replicas when the grant ends.
"""
from __future__ import annotations
import base64
import hashlib
from typing import Any, Callable, Mapping, Optional
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, TROUBLE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.config import BlobWireConfig
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.compartment import Command, Handler, Need, Outbound, WireProtocol
from .client import TransportBlobClient
from .intake import IntakeStore
from .messages import (CAP_DEFAULTS, DEFAULT_CHUNK_BYTES, HTTP_SCHEMES, MAX_CHUNK_BYTES, PIURI, PULL_TIMEOUT, blossom_url, make_chunk,
make_have, make_pull, make_refuse)
[docs]class BlobWire(WireProtocol):
"""Answers have and pull for blobs this node holds; takes offers under policy."""
name = 'blob_wire'
piuris: tuple[str, ...] = (PIURI,)
reads = ('blob_wire',)
config_cls = BlobWireConfig
live = ('blob_wire',)
SCHEMA = IntakeStore.SCHEMA
STORE_CLS = IntakeStore # builds its own; the base opens nothing for it (B5)
CHANNELS = (
Channel('blob_served', 'Blob chunk or have served to peer', level=INFO, say='blob chunk served to peer {peer}', tags=('blob', 'wire')),
Channel('offer_refused', 'Blob offer refused under policy', level=TROUBLE, say='blob offer refused from peer {peer}: {reason}', tags=('blob', 'wire')),
)
METRICS = (
Metric(name='blob_wire_chunks_served', description='Times blob chunks were served.', kind=COUNTER, counts='blob_served', tags=('blob', 'wire')),
Metric(name='blob_wire_offers_refused', description='Times blob offers were refused.', kind=COUNTER, counts='offer_refused', tags=('blob', 'wire')),
)
STORES = (
st.Store(key='_inflight', description='concurrent blob transfers per peer',
kind=st.TABLE, shape=int, bound=1024, keyed_by='peer', evict=st.LRU),
)
COMMANDS = (
Command(
key='held',
name='Held Replicas',
description='replicas taken over the transport, per peer',
tags=('blob_wire', 'held'),
parameters=(
Parameter(key='peer', name='peer', description='peer pubkey to filter by', type=str, default=None, required=False),
),
permission='blob_wire/held',
run='cmd_held',
reply='{peer: [{sha256, size, at}]}',
),
Command(
key='drop',
name='Drop Peer Replicas',
description='drop every replica held for a peer now',
tags=('blob_wire', 'drop'),
parameters=(
Parameter(key='peer', name='peer', description='peer pubkey', type=str, default='', required=True),
),
permission='blob_wire/drop',
run='cmd_drop',
reply='{peer, dropped}',
),
)
HANDLERS = (
Handler('have', run='handle_have', reads=('blobs',), doc='answer have with held hashes'),
Handler('pull', run='handle_pull', reads=('blobs',), doc='serve requested chunk of held blob'),
Handler('offer', run='handle_offer', writes=('blobs',), doc='evaluate offer and pull if accepted'),
Handler('chunk', run='handle_chunk', doc='handle unprompted chunk'),
Handler('refuse', run='handle_refuse', doc='handle unprompted refuse'),
)
OUTBOUND = (
Outbound('have', doc='query peer for held hashes or reply with held hashes'),
Outbound('offer', reply=('have', 'refuse'), doc='offer blob to peer'),
Outbound('pull', reply=('chunk', 'refuse'), doc='request chunk from peer'),
Outbound('chunk', doc='serve chunk bytes to pull requester'),
Outbound('refuse', doc='refuse unheld or disallowed pull/offer'),
)
REPLICAS_DB: str = 'blob_replicas.sqlite'
NEEDS = (
Need('blobs', 'stores.blobs'),
Need('transport', 'transport'),
Need('db_path', path='REPLICAS_DB'),
Need('cfg', 'cfg', getter=True),
Need('grants', 'stores.grants'),
)
[docs] def __init__(
self,
blobs: Any,
transport: Any,
db_path: str,
cfg: Callable[[], Mapping[str, Any]],
grantees: Optional[Callable[[], list[str]]] = None,
clock: Optional[Callable[[], int]] = None,
node: Any = None,
grants: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
self.blobs = blobs
self.transport = transport
self.db = IntakeStore(db_path)
self._cfg = cfg
if grantees is None and grants is not None:
self._grantees = lambda: grants.peers()
else:
self._grantees = grantees or (lambda: [])
self.client = TransportBlobClient(self)
self._inflight: st.Table[str, int]
self.taken = 0
self.dropped = 0
self.refused = 0
self._closed = False
# ---- configuration ----------------------------------------------------------------
[docs] def rc(self) -> BlobWireConfig:
"""The operator's `blob_wire` block as it is now; it reads live."""
return BlobWireConfig.from_dict((self._cfg() or {}).get('blob_wire') or {})
[docs] def settings(self) -> BlobWireConfig: # the Compartment hook, through the injected cfg
return self.rc()
[docs] def cap(self, name: str) -> int:
v = getattr(self.rc(), name)
return max(0, int(v)) if v is not None else CAP_DEFAULTS[name]
[docs] def chunk_bytes(self) -> int:
v = self.rc().chunk_bytes
return max(1, min(int(v), MAX_CHUNK_BYTES)) if v is not None else DEFAULT_CHUNK_BYTES
[docs] def accepts(self, peer: str) -> bool:
pol = self.rc().accept_from
if isinstance(pol, tuple):
return peer in pol
if pol == 'grantees':
return peer in self._grantees()
return False
[docs] def why_not(self, peer: str, size: int, now: int) -> Optional[str]:
"""The first reason an offer from this peer may not be taken, or None."""
if not self.accepts(peer):
return 'not accepting replicas from this peer'
if size > self.cap('max_blob_bytes'):
return 'blob larger than max_blob_bytes'
if self.db.total_for(peer) + size > self.cap('max_total_bytes'):
return 'would exceed max_total_bytes for this peer'
if self.db.today(peer, now) + size > self.cap('bytes_per_day'):
return 'would exceed bytes_per_day for this peer'
if self._inflight.get(peer, 0) >= self.cap('max_concurrent'):
return 'max_concurrent pulls already in flight for this peer'
return None
# ---- serving: what I hold, on request ------------------------------------------------
[docs] def handle_have(self, inbound: Any, sender: str) -> tuple[Any, ...]:
m = getattr(inbound, 'message', inbound)
peer = str(sender).lower()
body = m.get('body', {})
held = [h for h in body.get('sha256', []) if self.blobs.has(h)]
self.wire_client.reply(inbound, make_have(held), sender=peer)
return ('blob_wire', 'have', peer[:16], len(held))
[docs] def handle_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]:
return self._serve_pull(inbound, sender)
[docs] def handle_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]:
return self._take_offer(inbound, sender)
[docs] def handle_chunk(self, inbound: Any, sender: str) -> tuple[Any, ...]:
return ('ignored', 'chunk')
[docs] def handle_refuse(self, inbound: Any, sender: str) -> tuple[Any, ...]:
return ('ignored', 'refuse')
def _serve_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]:
m = getattr(inbound, 'message', inbound)
peer = str(sender).lower()
body = m['body']
h = str(body['sha256']).lower()
data = self.blobs.get(h)
if data is None:
self.wire_client.reply(inbound, make_refuse(h, 'not held'), sender=peer)
return ('blob_wire', 'pull refused', peer[:16], h[:16], 'not held')
off, ln = int(body['offset']), int(body['length'])
if off < 0 or ln <= 0 or off > len(data):
self.wire_client.reply(inbound, make_refuse(h, 'range out of bounds'), sender=peer)
return ('blob_wire', 'pull refused', peer[:16], h[:16], 'range')
ln = min(ln, self.chunk_bytes())
self.wire_client.reply(inbound, make_chunk(h, off, data[off:off + ln]), sender=peer)
return ('blob_wire', 'chunk', peer[:16], h[:16], off)
# ---- taking: the receiver's pull loop ---------------------------------------------------
def _take_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]:
m = getattr(inbound, 'message', inbound)
peer = str(sender).lower()
body = m['body']
h = str(body['sha256']).lower()
size = int(body['size'])
now = self.clock()
if self.blobs.has(h):
# Already here, whoever brought it. Say so; pin it for this peer too.
self.blobs.pin(h)
if not self.db.holds(h):
self.db.took(h, peer, 0, now)
self.wire_client.reply(inbound, make_have([h]), sender=peer)
return ('blob_wire', 'already held', peer[:16], h[:16])
why = self.why_not(peer, size, now)
if why is not None:
self.refused += 1
self.refusals.refuse('blob_wire', 'offer refused', subject=peer, detail={'blob': h[:16], 'why': why}, at=now)
self.wire_client.reply(inbound, make_refuse(h, why), sender=peer)
return ('blob_wire', 'offer refused', peer[:16], h[:16], why)
chunk = max(1, min(int(body.get('chunk_bytes', DEFAULT_CHUNK_BYTES)), self.chunk_bytes()))
self._inflight[peer] = self._inflight.get(peer, 0) + 1
try:
data = self._pull(peer, h, size, chunk)
except Exception as e:
why = '%s: %s' % (type(e).__name__, str(e)[:120])
self.refused += 1
self.refusals.refuse('blob_wire', 'pull failed', subject=peer, detail={'blob': h[:16], 'why': why}, at=now)
self.wire_client.reply(inbound, make_refuse(h, why), sender=peer)
return ('blob_wire', 'pull failed', peer[:16], h[:16], why)
finally:
self._inflight[peer] -= 1
if self._inflight[peer] <= 0:
self._inflight.pop(peer, None)
self.blobs.put(data, content_type=str(body.get('type') or 'application/octet-stream'),
uploaded_by='replica:' + peer, now=now)
self.blobs.pin(h)
self.db.took(h, peer, size, now)
self.taken += 1
self.wire_client.reply(inbound, make_have([h]), sender=peer)
return ('blob_wire', 'took', peer[:16], h[:16], size)
def _pull(self, peer: str, h: str, size: int, chunk: int) -> bytes:
hasher = hashlib.sha256()
out = bytearray()
off = 0
while off < size:
want = min(chunk, size - off)
reply = self.wire_client.request(peer, make_pull(h, off, want), timeout=PULL_TIMEOUT)
if reply is None:
raise TimeoutError('no chunk at offset %d' % off)
rm = reply.message
k = self.wire_client.kind_of(rm)
if k == 'refuse':
raise ValueError('source refused: %s' % rm.get('body', {}).get('why', '?'))
if k != 'chunk':
raise ValueError('expected chunk, got %s' % (k or rm.get('type')))
rb = rm['body']
if str(rb.get('sha256', '')).lower() != h or int(rb.get('offset', -1)) != off:
raise ValueError('chunk for the wrong blob or offset')
piece = base64.b64decode(rb['data'], validate=True)
if not piece or len(piece) > want:
raise ValueError('chunk of the wrong size at offset %d' % off)
hasher.update(piece)
out += piece
off += len(piece)
if hasher.hexdigest() != h:
raise ValueError('sha256 mismatch after %d bytes' % off)
return bytes(out)
# ---- the compartment contract ---------------------------------------------------------------
[docs] def drop_peer(self, peer: str, now: Optional[int] = None) -> Mapping[str, Any]:
"""Let go of every replica held for a peer: unpin, delete, forget."""
peer = peer.lower()
gone = self.db.forget_peer(peer)
for h in gone:
try:
self.blobs.unpin(h)
stat = self.blobs.stat(h)
if stat and str(stat.get('uploaded_by') or '') == 'replica:' + peer:
self.blobs.delete(h)
except Exception:
pass
self.dropped += len(gone)
return {'peer': peer, 'dropped': len(gone)}
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
did: list[tuple[Any, ...]] = []
if self._closed:
return did
live = set(self._grantees())
for peer in self.db.peers():
if peer not in live:
r = self.drop_peer(peer, now)
if r['dropped']:
did.append(('blob_wire', 'grant ended, dropped', peer[:16], r['dropped']))
self.db.prune_days(now)
return did
[docs] def state(self) -> Mapping[str, Any]:
if self._closed:
return {'stopped': True, 'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused}
peers = self.db.peers()
pol = self.rc().accept_from
return {'accept_from': list(pol) if isinstance(pol, tuple) else pol,
'caps': {k: self.cap(k) for k in CAP_DEFAULTS},
'peers': len(peers),
'held_bytes': sum(self.db.total_for(p) for p in peers),
'inflight': dict(self._inflight),
'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused}
[docs] def trouble(self) -> list[str]:
if self._closed:
return []
pol = self.rc().accept_from
if pol != 'none' and pol and not all(self.cap(k) for k in ('bytes_per_day', 'max_blob_bytes', 'max_total_bytes')):
return ['blob_wire.accept_from is set but a cap is 0, so every offer is refused: '
'set bytes_per_day, max_blob_bytes, max_total_bytes and max_concurrent']
return []
[docs] def cmd_held(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
peers = [str(args['peer']).lower()] if args.get('peer') else self.db.peers()
return {p: self.db.held_for(p) for p in peers}
[docs] def cmd_drop(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
return self.drop_peer(str(args['peer']))
[docs] def stop(self) -> Any:
self._closed = True
try:
self.db.close()
except Exception:
pass
super().stop()
return None
[docs]def partner_transport_peers(node: Any) -> list[str]:
"""`peer:<pk>` for every grantee that has no Blossom URL in its descriptor.
Peers with an HTTP endpoint take the Blossom path; this is for the rest.
"""
out: list[str] = []
grants = getattr(node, 'grants', None)
peers = getattr(node, 'peers', None)
if grants is None or peers is None:
return out
for pk in grants.peers():
try:
d = peers.descriptor(pk)
except Exception:
d = None
has_http = False
for t in (d or {}).get('tags', []):
if t and t[0] == 'endpoint' and len(t) > 2 and t[1] in HTTP_SCHEMES and blossom_url(t[2]):
has_http = True
break
if not has_http:
out.append(TransportBlobClient.PREFIX + pk)
return out