Source code for fleetmesh_node.planes.store.blob_replication.compartment

"""Blossom replication: keep N copies of every pinned blob on other nodes.

Pinning keeps a blob off this node's LRU eviction; it does nothing if this
node's disk dies. Replication is the other half. For each pinned blob the
operator gives a target copy count, and on every tick this compartment:

  1. asks candidate servers without a copy to mirror it (BUD-04 PUT /mirror,
     signed kind 24242 by this node, source = this node's public Blossom URL),
  2. re-checks held copies on a slow cadence with HEAD, and
  3. forgets a copy that has gone, so step 1 asks again - on another server
     if there is one.

Where copies go: servers from config `blob_replication.servers`, then every peer
this node holds a live grant with that advertises an http(s)/ws(s) endpoint in
its descriptor (a node's face serves Blossom on the same port). A peering
partner is who this node already trusts with its traffic; a stranger found on
a relay is not asked to hold its data.

A partner with a grant but no http(s)/ws(s) endpoint - a node behind NAT -
takes the second path: blob/1.0 over the peering transport (blob_wire.py).
Such a candidate is named `peer:<pubkey>` and driven through the same
mirror_blob/head_blob calls, so run() and verify() below do not know which
path a copy took.

What it cannot do, and says so: a partner may accept a mirror and later drop
it; this is found by the verify pass, not prevented. A node with no public URL
(`blob_replication.source_url`, or a first http(s)/ws(s) endpoint) cannot be
mirrored from over Blossom, and `trouble` reports that instead of failing
silently; the transport path needs no URL.

State lives in SQLite beside the blob index (blob_replicas.sqlite).
"""
from __future__ import annotations

import urllib.error
from typing import Any, Callable, Mapping, Optional, cast

from fleetmesh_node.storage import SqliteStore
from fleetmesh_node.config import BlobReplicationConfig
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Every, Need
from fleetmesh_node.compartment.schema import Column, Table
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.protocols.nostr import blossom
from fleetmesh_node.transport import netfetch
from fleetmesh_node.planes.store.blob_wire.messages import HTTP_SCHEMES, blossom_url
from .types import (
    BlobReplicationStatus,
    BlobReplicationStateReport,
    PinResult,
    UnpinResult,
)

TICK_SECONDS: int = 300          # how often to try under-replicated blobs
VERIFY_SECONDS: int = 6 * 3600   # how often a held copy is re-checked
RETRY_SECONDS: int = 1800        # back-off for a server that just failed
MAX_ATTEMPTS_PER_TICK: int = 16  # bound the network work one tick does


[docs]class ReplicaStore(SqliteStore): SCHEMA = ( Table('wants', ( Column('sha256', 'TEXT', primary_key=True), Column('copies', 'INTEGER', not_null=True), Column('since', 'INTEGER', not_null=True), )), Table('replicas', ( Column('sha256', 'TEXT', not_null=True), Column('server', 'TEXT', not_null=True), Column('state', 'TEXT', not_null=True), Column('at', 'INTEGER', not_null=True), Column('why', 'TEXT'), ), primary_key=('sha256', 'server')), )
[docs] def want(self, h: str, copies: int, now: int) -> None: self.execute('INSERT INTO wants (sha256, copies, since) VALUES (?, ?, ?) ' 'ON CONFLICT(sha256) DO UPDATE SET copies = excluded.copies', (h, copies, now)) self.commit()
[docs] def unwant(self, h: str) -> bool: cur = self.execute('DELETE FROM wants WHERE sha256 = ?', (h,)) self.commit() return cur.rowcount > 0
[docs] def wants(self) -> list[tuple[str, int]]: return [(r[0], int(r[1])) for r in self._all('SELECT sha256, copies FROM wants ORDER BY since')]
[docs] def copies(self, h: str) -> int: row = self.want_row(h) return int(row[1]) if row else 0
[docs] def want_row(self, h: str) -> Optional[tuple[Any, ...]]: return self._one('SELECT sha256, copies FROM wants WHERE sha256 = ?', (h,))
[docs] def replicas(self, h: str) -> list[Mapping[str, Any]]: return [{'server': r[0], 'state': r[1], 'at': r[2], 'why': r[3]} for r in self._all('SELECT server, state, at, why FROM replicas WHERE sha256 = ? ORDER BY server', (h,))]
[docs] def mark(self, h: str, server: str, state: str, now: int, why: Optional[str] = None) -> None: self.execute('INSERT INTO replicas (sha256, server, state, at, why) VALUES (?, ?, ?, ?, ?) ' 'ON CONFLICT(sha256, server) DO UPDATE SET state = excluded.state, at = excluded.at, ' 'why = excluded.why', (h, server, state, now, why)) self.commit()
[docs] def forget(self, h: str, server: Optional[str] = None) -> None: if server is None: self.execute('DELETE FROM replicas WHERE sha256 = ?', (h,)) else: self.execute('DELETE FROM replicas WHERE sha256 = ? AND server = ?', (h, server)) self.commit()
[docs]class Replication(Compartment): requires: tuple[str, ...] = ('blob_wire',) """Keeps a target number of copies of each pinned blob on other servers.""" name = 'blob_replication' reads = ('blob_replication', 'endpoints') config_cls = BlobReplicationConfig live = ('blob_replication',) REPLICAS_DB = 'blob_replicas.sqlite' SCHEMA = ReplicaStore.SCHEMA STORE_CLS = ReplicaStore # builds its own; the base opens nothing for it (B5) STORES = ( st.Store(key='replicas', description='replicated blob records in sqlite', kind=st.RING, shape=dict, bound=1024), ) CHANNELS = ( Channel('blob_replicated', 'Blob successfully mirrored to remote partner', level=INFO, say='blob {sha256} replicated to {server}', tags=('blob', 'replication')), Channel('replication_failed', 'Blob mirroring attempt failed', level=NOTE, say='blob {sha256} replication failed on {server}: {why}', tags=('blob', 'replication')), ) METRICS = ( Metric(name='blob_replication_replicated', description='Times blobs were replicated.', kind=COUNTER, counts='blob_replicated', tags=('blob', 'replication')), Metric(name='blob_replication_failed', description='Times blob replications failed.', kind=COUNTER, counts='replication_failed', tags=('blob', 'replication')), ) NEEDS = ( Need('blobs', 'stores.blobs'), Need('db_path', path='REPLICAS_DB'), Need('signer_sk', 'ks.node_sk'), Need('cfg', 'cfg', getter=True), Need('blob_wire'), ) COMMANDS = ( Command( key='pin', name='Pin Blob', description='pin a blob here and keep that many copies of it on other servers', tags=('blob_replication', 'pin'), parameters=( Parameter(key='sha256', name='sha256', description='hex digest of blob', type=str, default='', required=True), Parameter(key='copies', name='copies', description='target copy count', type=int, default=None, required=False), ), permission='blob_replication/pin', run='pin_cmd', reply='{sha256, copies, pinned}', ), Command( key='unpin', name='Unpin Blob', description='stop replicating (and unpin) a blob; copies already made stay', tags=('blob_replication', 'unpin'), parameters=( Parameter(key='sha256', name='sha256', description='hex digest of blob', type=str, default='', required=True), ), permission='blob_replication/unpin', run='unpin_cmd', reply='{sha256, unpinned}', ), Command( key='status', name='Replication Status', description='target and held copies, per blob', tags=('blob_replication', 'status'), parameters=( Parameter(key='sha256', name='sha256', description='hex digest of blob to query', type=str, default=None, required=False), ), permission='blob_replication/status', run='status_cmd', reply='[{sha256, copies, held, servers}]', ), Command( key='run', name='Run Replication', description='mirror under-replicated blobs now', tags=('blob_replication', 'run'), permission='blob_replication/run', run='run_cmd', reply='[(replication, ...)]', ), Command( key='verify', name='Verify Copies', description='re-check every held copy now', tags=('blob_replication', 'verify'), permission='blob_replication/verify', run='verify_cmd', reply='[(replication, ...)]', ), ) EVERY = ( Every(seconds=VERIFY_SECONDS, run='verify', name='replication_verify'), Every(seconds=TICK_SECONDS, run='run', name='replication_run'), )
[docs] def __init__( self, blobs: Any, db_path: str, signer_sk: str, cfg: Callable[[], Mapping[str, Any]], candidates: Optional[Callable[[], list[str]]] = None, clock: Optional[Callable[[], int]] = None, client: Any = None, node: Any = None, transport_client: Any = None, blob_wire: Optional[Any] = None, ) -> None: Compartment.__init__(self, node, clock) self.blobs = blobs self.db = ReplicaStore(db_path) self.signer_sk = signer_sk self._cfg = cfg self.blob_wire = blob_wire if candidates is not None: self._candidates = candidates else: self._candidates = self._default_candidates self.client = client or blossom.BlossomClient # blob/1.0 over the peering transport, for `peer:<pk>` candidates. self.transport_client = transport_client if transport_client is not None else getattr(blob_wire, 'client', None) self.last_run: Optional[int] = None self.last_verify: Optional[int] = None self.mirrored = 0 self.lost = 0 self._closed = False
def _default_candidates(self) -> list[str]: if not self.node: return [] from fleetmesh_node.planes.store import blob_wire as blob_wire_mod return partner_blossom_servers(self.node) + blob_wire_mod.partner_transport_peers(self.node) # ---- configuration --------------------------------------------------------
[docs] def rc(self) -> BlobReplicationConfig: """The operator's `blob_replication` block as it is now; it reads live.""" return BlobReplicationConfig.from_dict((self._cfg() or {}).get('blob_replication') or {})
[docs] def settings(self) -> BlobReplicationConfig: # the Compartment hook, through the injected cfg return self.rc()
[docs] def default_copies(self) -> int: return max(0, self.rc().copies)
[docs] def allow_private(self) -> Any: """Address policy for partner Blossom servers. Partner URLs come out of peer descriptors, so a peer could name a loopback or LAN address and make this node mirror into it. netfetch refuses that by default; an operator running a LAN cluster sets ``blob_replication.allow_private`` to true or to a list of hosts. """ ap = self.rc().allow_private return list(ap) if isinstance(ap, tuple) else ap
[docs] def source_url(self) -> Optional[str]: """Where partners fetch from: this node's own public Blossom base URL.""" explicit = self.rc().source_url if explicit: return blossom_url(str(explicit)) eps = sorted((self._cfg() or {}).get('endpoints') or [], key=lambda e: int(e.get('priority', 50))) for e in eps: u = blossom_url(str(e.get('address') or '')) if u: return u return None
[docs] def servers(self) -> list[str]: out: list[str] = [] own = self.source_url() for s in list(self.rc().servers) + list(self._candidates() or []): s = str(s) if _is_peer(s): if self.transport_client is not None and s not in out: out.append(s) continue u = blossom_url(s) if u and u != own and u not in out: out.append(u) return out
[docs] def client_for(self, server: str) -> Any: return self.transport_client if _is_peer(server) else self.client
# ---- operator actions -----------------------------------------------------
[docs] def pin(self, h: str, copies: Optional[int] = None) -> PinResult: h = h.lower() if not self.blobs.has(h): raise ValueError('no blob %s on this node to replicate' % h[:16]) n = self.default_copies() if copies is None else max(0, int(copies)) self.blobs.pin(h) self.db.want(h, n, self.clock()) return {'sha256': h, 'copies': n, 'pinned': True}
[docs] def unpin(self, h: str) -> UnpinResult: h = h.lower() gone = self.db.unwant(h) self.db.forget(h) self.blobs.unpin(h) return {'sha256': h, 'unpinned': gone}
[docs] def status(self, h: Optional[str] = None) -> list[BlobReplicationStatus]: rows = self.db.wants() if h is None else [(h.lower(), self.db.copies(h.lower()))] out: list[BlobReplicationStatus] = [] for sha, n in rows: reps = self.db.replicas(sha) held = [r['server'] for r in reps if r['state'] == 'held'] out.append({'sha256': sha, 'copies': n, 'held': len(held), 'servers': held, 'failed': {r['server']: r['why'] for r in reps if r['state'] == 'failed'}}) return out
# ---- the work -------------------------------------------------------------
[docs] def run(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: """Ask servers to mirror every blob below its target copy count.""" now = int(now if now is not None else self.clock()) self.last_run = now did: list[tuple[Any, ...]] = [] src = self.source_url() servers = [s for s in self.servers() if src is not None or _is_peer(s)] if not servers: return did budget = MAX_ATTEMPTS_PER_TICK for h, want in self.db.wants(): if budget <= 0: break if not self.blobs.has(h): self.refusals.refuse('blob_replication', 'pinned blob is missing locally', subject=h[:16], at=now) continue reps = {r['server']: r for r in self.db.replicas(h)} held = sum(1 for r in reps.values() if r['state'] == 'held') for srv in servers: if held >= want or budget <= 0: break r = reps.get(srv) if r and (r['state'] == 'held' or now - int(r['at']) < RETRY_SECONDS): continue budget -= 1 try: res = self.client_for(srv).mirror_blob(srv, '%s/%s' % (src, h), h, signer_sk=self.signer_sk, allow_private=self.allow_private()) if str(res.get('sha256', '')).lower() != h: raise ValueError('server answered for a different blob') self.db.mark(h, srv, 'held', now) held += 1 self.mirrored += 1 did.append(('blob_replication', 'mirrored', h[:16], srv)) except Exception as e: why = _why(e) self.db.mark(h, srv, 'failed', now, why) self.refusals.refuse('blob_replication', 'mirror failed', subject=srv, detail={'blob': h[:16], 'why': why}, at=now) did.append(('blob_replication', 'mirror failed', h[:16], srv, why)) return did
[docs] def verify(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: """HEAD every held copy; forget the ones that are gone so run() re-asks.""" now = int(now if now is not None else self.clock()) self.last_verify = now did: list[tuple[Any, ...]] = [] for h, _want in self.db.wants(): for r in self.db.replicas(h): if r['state'] != 'held': continue try: meta = self.client_for(r['server']).head_blob(r['server'], h, allow_private=self.allow_private()) except Exception as e: # Unreachable is not gone: keep the copy, say so, look again later. did.append(('blob_replication', 'verify unreachable', h[:16], r['server'], _why(e))) continue if meta is None: self.db.forget(h, r['server']) self.lost += 1 self.refusals.refuse('blob_replication', 'copy lost', subject=r['server'], detail={'blob': h[:16]}, at=now) did.append(('blob_replication', 'copy lost', h[:16], r['server'])) else: self.db.mark(h, r['server'], 'held', now) return did
# ---- the compartment contract ------------------------------------------------- _tick_runs_every = True
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: if self._closed: return [] if not hasattr(self, '_every_last'): self._every_last = {} # not a store: Every job timing tracking if 'replication_verify' not in self._every_last: self._every_last['replication_verify'] = now self.last_verify = now return self.run_every(now)
[docs] def state(self) -> BlobReplicationStateReport: if self._closed: # health() is read after stop(); report the counters, not the store. return {'stopped': True, 'mirrored': self.mirrored, 'lost': self.lost, 'last_run': self.last_run, 'last_verify': self.last_verify} st = self.status() return {'blobs': len(st), 'under_replicated': sum(1 for s in st if s['held'] < s['copies']), 'servers': len(self.servers()), 'source_url': self.source_url(), 'mirrored': self.mirrored, 'lost': self.lost, 'last_run': self.last_run, 'last_verify': self.last_verify}
[docs] def trouble(self) -> list[str]: if self._closed: return [] st = self.status() if not st: return [] out: list[str] = [] servers = self.servers() if self.source_url() is None and not any(_is_peer(s) for s in servers): out.append('%d pinned blob(s) want copies, but this node advertises no http(s)/ws(s) ' 'endpoint (or blob_replication.source_url) for partners to mirror from, ' 'and no partner takes blob/1.0 over the transport' % len(st)) elif not servers: out.append('%d pinned blob(s) want copies, and there is no partner with a Blossom ' 'endpoint or a transport path and no blob_replication.servers to hold them' % len(st)) under = [s for s in st if s['held'] < s['copies']] if under: out.append('%d pinned blob(s) below their copy target' % len(under)) return out
[docs] def pin_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: return self.pin(str(args['sha256']), args.get('copies'))
[docs] def unpin_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]: return self.unpin(str(args['sha256']))
[docs] def status_cmd(self, args: Mapping[str, Any]) -> list[Mapping[str, Any]]: h = args.get('sha256') return cast(list[Mapping[str, Any]], self.status(str(h) if h is not None else None))
[docs] def run_cmd(self) -> list[tuple[Any, ...]]: return self.run()
[docs] def verify_cmd(self) -> list[tuple[Any, ...]]: return self.verify()
[docs] def stop(self) -> Any: self._closed = True try: self.db.close() except Exception: pass super().stop() return None
def _is_peer(server: str) -> bool: return isinstance(server, str) and server.startswith('peer:') def _why(e: Exception) -> str: if isinstance(e, (urllib.error.HTTPError, netfetch.FetchHTTPError)): return 'HTTP %d' % e.code return '%s: %s' % (type(e).__name__, str(e)[:120])
[docs]def partner_blossom_servers(node: Any) -> list[str]: """Blossom URLs of every peer this node holds a grant with, from their descriptors.""" out: list[str] = [] grants = getattr(node, 'grants', None) peers = getattr(node, 'peers', None) if grants is None or peers is None: return out for pk in grants.peers(): try: d = peers.descriptor(pk) except Exception: d = None if not d: continue eps = sorted((t for t in d.get('tags', []) if t and t[0] == 'endpoint' and len(t) > 2 and t[1] in HTTP_SCHEMES), key=lambda t: int(t[3]) if len(t) > 3 and str(t[3]).isdigit() else 50) for t in eps: u = blossom_url(t[2]) if u and u not in out: out.append(u) break return out