"""Blossom replication: keep N copies of every pinned blob on other nodes.
Pinning keeps a blob off this node's LRU eviction; it does nothing if this
node's disk dies. Replication is the other half. For each pinned blob the
operator gives a target copy count, and on every tick this compartment:
1. asks candidate servers without a copy to mirror it (BUD-04 PUT /mirror,
signed kind 24242 by this node, source = this node's public Blossom URL),
2. re-checks held copies on a slow cadence with HEAD, and
3. forgets a copy that has gone, so step 1 asks again - on another server
if there is one.
Where copies go: servers from config `blob_replication.servers`, then every peer
this node holds a live grant with that advertises an http(s)/ws(s) endpoint in
its descriptor (a node's face serves Blossom on the same port). A peering
partner is who this node already trusts with its traffic; a stranger found on
a relay is not asked to hold its data.
A partner with a grant but no http(s)/ws(s) endpoint - a node behind NAT -
takes the second path: blob/1.0 over the peering transport (blob_wire.py).
Such a candidate is named `peer:<pubkey>` and driven through the same
mirror_blob/head_blob calls, so run() and verify() below do not know which
path a copy took.
What it cannot do, and says so: a partner may accept a mirror and later drop
it; this is found by the verify pass, not prevented. A node with no public URL
(`blob_replication.source_url`, or a first http(s)/ws(s) endpoint) cannot be
mirrored from over Blossom, and `trouble` reports that instead of failing
silently; the transport path needs no URL.
State lives in SQLite beside the blob index (blob_replicas.sqlite).
"""
from __future__ import annotations
import urllib.error
from typing import Any, Callable, Mapping, Optional, cast
from fleetmesh_node.storage import SqliteStore
from fleetmesh_node.config import BlobReplicationConfig
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Every, Need
from fleetmesh_node.compartment.schema import Column, Table
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.protocols.nostr import blossom
from fleetmesh_node.transport import netfetch
from fleetmesh_node.planes.store.blob_wire.messages import HTTP_SCHEMES, blossom_url
from .types import (
BlobReplicationStatus,
BlobReplicationStateReport,
PinResult,
UnpinResult,
)
TICK_SECONDS: int = 300 # how often to try under-replicated blobs
VERIFY_SECONDS: int = 6 * 3600 # how often a held copy is re-checked
RETRY_SECONDS: int = 1800 # back-off for a server that just failed
MAX_ATTEMPTS_PER_TICK: int = 16 # bound the network work one tick does
[docs]class ReplicaStore(SqliteStore):
SCHEMA = (
Table('wants', (
Column('sha256', 'TEXT', primary_key=True),
Column('copies', 'INTEGER', not_null=True),
Column('since', 'INTEGER', not_null=True),
)),
Table('replicas', (
Column('sha256', 'TEXT', not_null=True),
Column('server', 'TEXT', not_null=True),
Column('state', 'TEXT', not_null=True),
Column('at', 'INTEGER', not_null=True),
Column('why', 'TEXT'),
), primary_key=('sha256', 'server')),
)
[docs] def want(self, h: str, copies: int, now: int) -> None:
self.execute('INSERT INTO wants (sha256, copies, since) VALUES (?, ?, ?) '
'ON CONFLICT(sha256) DO UPDATE SET copies = excluded.copies', (h, copies, now))
self.commit()
[docs] def unwant(self, h: str) -> bool:
cur = self.execute('DELETE FROM wants WHERE sha256 = ?', (h,))
self.commit()
return cur.rowcount > 0
[docs] def wants(self) -> list[tuple[str, int]]:
return [(r[0], int(r[1])) for r in self._all('SELECT sha256, copies FROM wants ORDER BY since')]
[docs] def copies(self, h: str) -> int:
row = self.want_row(h)
return int(row[1]) if row else 0
[docs] def want_row(self, h: str) -> Optional[tuple[Any, ...]]:
return self._one('SELECT sha256, copies FROM wants WHERE sha256 = ?', (h,))
[docs] def replicas(self, h: str) -> list[Mapping[str, Any]]:
return [{'server': r[0], 'state': r[1], 'at': r[2], 'why': r[3]}
for r in self._all('SELECT server, state, at, why FROM replicas WHERE sha256 = ? ORDER BY server', (h,))]
[docs] def mark(self, h: str, server: str, state: str, now: int, why: Optional[str] = None) -> None:
self.execute('INSERT INTO replicas (sha256, server, state, at, why) VALUES (?, ?, ?, ?, ?) '
'ON CONFLICT(sha256, server) DO UPDATE SET state = excluded.state, at = excluded.at, '
'why = excluded.why', (h, server, state, now, why))
self.commit()
[docs] def forget(self, h: str, server: Optional[str] = None) -> None:
if server is None:
self.execute('DELETE FROM replicas WHERE sha256 = ?', (h,))
else:
self.execute('DELETE FROM replicas WHERE sha256 = ? AND server = ?', (h, server))
self.commit()
[docs]class Replication(Compartment):
requires: tuple[str, ...] = ('blob_wire',)
"""Keeps a target number of copies of each pinned blob on other servers."""
name = 'blob_replication'
reads = ('blob_replication', 'endpoints')
config_cls = BlobReplicationConfig
live = ('blob_replication',)
REPLICAS_DB = 'blob_replicas.sqlite'
SCHEMA = ReplicaStore.SCHEMA
STORE_CLS = ReplicaStore # builds its own; the base opens nothing for it (B5)
STORES = (
st.Store(key='replicas', description='replicated blob records in sqlite',
kind=st.RING, shape=dict, bound=1024),
)
CHANNELS = (
Channel('blob_replicated', 'Blob successfully mirrored to remote partner', level=INFO, say='blob {sha256} replicated to {server}', tags=('blob', 'replication')),
Channel('replication_failed', 'Blob mirroring attempt failed', level=NOTE, say='blob {sha256} replication failed on {server}: {why}', tags=('blob', 'replication')),
)
METRICS = (
Metric(name='blob_replication_replicated', description='Times blobs were replicated.', kind=COUNTER, counts='blob_replicated', tags=('blob', 'replication')),
Metric(name='blob_replication_failed', description='Times blob replications failed.', kind=COUNTER, counts='replication_failed', tags=('blob', 'replication')),
)
NEEDS = (
Need('blobs', 'stores.blobs'),
Need('db_path', path='REPLICAS_DB'),
Need('signer_sk', 'ks.node_sk'),
Need('cfg', 'cfg', getter=True),
Need('blob_wire'),
)
COMMANDS = (
Command(
key='pin',
name='Pin Blob',
description='pin a blob here and keep that many copies of it on other servers',
tags=('blob_replication', 'pin'),
parameters=(
Parameter(key='sha256', name='sha256', description='hex digest of blob', type=str, default='', required=True),
Parameter(key='copies', name='copies', description='target copy count', type=int, default=None, required=False),
),
permission='blob_replication/pin',
run='pin_cmd',
reply='{sha256, copies, pinned}',
),
Command(
key='unpin',
name='Unpin Blob',
description='stop replicating (and unpin) a blob; copies already made stay',
tags=('blob_replication', 'unpin'),
parameters=(
Parameter(key='sha256', name='sha256', description='hex digest of blob', type=str, default='', required=True),
),
permission='blob_replication/unpin',
run='unpin_cmd',
reply='{sha256, unpinned}',
),
Command(
key='status',
name='Replication Status',
description='target and held copies, per blob',
tags=('blob_replication', 'status'),
parameters=(
Parameter(key='sha256', name='sha256', description='hex digest of blob to query', type=str, default=None, required=False),
),
permission='blob_replication/status',
run='status_cmd',
reply='[{sha256, copies, held, servers}]',
),
Command(
key='run',
name='Run Replication',
description='mirror under-replicated blobs now',
tags=('blob_replication', 'run'),
permission='blob_replication/run',
run='run_cmd',
reply='[(replication, ...)]',
),
Command(
key='verify',
name='Verify Copies',
description='re-check every held copy now',
tags=('blob_replication', 'verify'),
permission='blob_replication/verify',
run='verify_cmd',
reply='[(replication, ...)]',
),
)
EVERY = (
Every(seconds=VERIFY_SECONDS, run='verify', name='replication_verify'),
Every(seconds=TICK_SECONDS, run='run', name='replication_run'),
)
[docs] def __init__(
self,
blobs: Any,
db_path: str,
signer_sk: str,
cfg: Callable[[], Mapping[str, Any]],
candidates: Optional[Callable[[], list[str]]] = None,
clock: Optional[Callable[[], int]] = None,
client: Any = None,
node: Any = None,
transport_client: Any = None,
blob_wire: Optional[Any] = None,
) -> None:
Compartment.__init__(self, node, clock)
self.blobs = blobs
self.db = ReplicaStore(db_path)
self.signer_sk = signer_sk
self._cfg = cfg
self.blob_wire = blob_wire
if candidates is not None:
self._candidates = candidates
else:
self._candidates = self._default_candidates
self.client = client or blossom.BlossomClient
# blob/1.0 over the peering transport, for `peer:<pk>` candidates.
self.transport_client = transport_client if transport_client is not None else getattr(blob_wire, 'client', None)
self.last_run: Optional[int] = None
self.last_verify: Optional[int] = None
self.mirrored = 0
self.lost = 0
self._closed = False
def _default_candidates(self) -> list[str]:
if not self.node:
return []
from fleetmesh_node.planes.store import blob_wire as blob_wire_mod
return partner_blossom_servers(self.node) + blob_wire_mod.partner_transport_peers(self.node)
# ---- configuration --------------------------------------------------------
[docs] def rc(self) -> BlobReplicationConfig:
"""The operator's `blob_replication` block as it is now; it reads live."""
return BlobReplicationConfig.from_dict((self._cfg() or {}).get('blob_replication') or {})
[docs] def settings(self) -> BlobReplicationConfig: # the Compartment hook, through the injected cfg
return self.rc()
[docs] def default_copies(self) -> int:
return max(0, self.rc().copies)
[docs] def allow_private(self) -> Any:
"""Address policy for partner Blossom servers.
Partner URLs come out of peer descriptors, so a peer could name a
loopback or LAN address and make this node mirror into it. netfetch
refuses that by default; an operator running a LAN cluster sets
``blob_replication.allow_private`` to true or to a list of hosts.
"""
ap = self.rc().allow_private
return list(ap) if isinstance(ap, tuple) else ap
[docs] def source_url(self) -> Optional[str]:
"""Where partners fetch from: this node's own public Blossom base URL."""
explicit = self.rc().source_url
if explicit:
return blossom_url(str(explicit))
eps = sorted((self._cfg() or {}).get('endpoints') or [], key=lambda e: int(e.get('priority', 50)))
for e in eps:
u = blossom_url(str(e.get('address') or ''))
if u:
return u
return None
[docs] def servers(self) -> list[str]:
out: list[str] = []
own = self.source_url()
for s in list(self.rc().servers) + list(self._candidates() or []):
s = str(s)
if _is_peer(s):
if self.transport_client is not None and s not in out:
out.append(s)
continue
u = blossom_url(s)
if u and u != own and u not in out:
out.append(u)
return out
[docs] def client_for(self, server: str) -> Any:
return self.transport_client if _is_peer(server) else self.client
# ---- operator actions -----------------------------------------------------
[docs] def pin(self, h: str, copies: Optional[int] = None) -> PinResult:
h = h.lower()
if not self.blobs.has(h):
raise ValueError('no blob %s on this node to replicate' % h[:16])
n = self.default_copies() if copies is None else max(0, int(copies))
self.blobs.pin(h)
self.db.want(h, n, self.clock())
return {'sha256': h, 'copies': n, 'pinned': True}
[docs] def unpin(self, h: str) -> UnpinResult:
h = h.lower()
gone = self.db.unwant(h)
self.db.forget(h)
self.blobs.unpin(h)
return {'sha256': h, 'unpinned': gone}
[docs] def status(self, h: Optional[str] = None) -> list[BlobReplicationStatus]:
rows = self.db.wants() if h is None else [(h.lower(), self.db.copies(h.lower()))]
out: list[BlobReplicationStatus] = []
for sha, n in rows:
reps = self.db.replicas(sha)
held = [r['server'] for r in reps if r['state'] == 'held']
out.append({'sha256': sha, 'copies': n, 'held': len(held), 'servers': held,
'failed': {r['server']: r['why'] for r in reps if r['state'] == 'failed'}})
return out
# ---- the work -------------------------------------------------------------
[docs] def run(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
"""Ask servers to mirror every blob below its target copy count."""
now = int(now if now is not None else self.clock())
self.last_run = now
did: list[tuple[Any, ...]] = []
src = self.source_url()
servers = [s for s in self.servers() if src is not None or _is_peer(s)]
if not servers:
return did
budget = MAX_ATTEMPTS_PER_TICK
for h, want in self.db.wants():
if budget <= 0:
break
if not self.blobs.has(h):
self.refusals.refuse('blob_replication', 'pinned blob is missing locally', subject=h[:16], at=now)
continue
reps = {r['server']: r for r in self.db.replicas(h)}
held = sum(1 for r in reps.values() if r['state'] == 'held')
for srv in servers:
if held >= want or budget <= 0:
break
r = reps.get(srv)
if r and (r['state'] == 'held' or now - int(r['at']) < RETRY_SECONDS):
continue
budget -= 1
try:
res = self.client_for(srv).mirror_blob(srv, '%s/%s' % (src, h), h, signer_sk=self.signer_sk,
allow_private=self.allow_private())
if str(res.get('sha256', '')).lower() != h:
raise ValueError('server answered for a different blob')
self.db.mark(h, srv, 'held', now)
held += 1
self.mirrored += 1
did.append(('blob_replication', 'mirrored', h[:16], srv))
except Exception as e:
why = _why(e)
self.db.mark(h, srv, 'failed', now, why)
self.refusals.refuse('blob_replication', 'mirror failed', subject=srv,
detail={'blob': h[:16], 'why': why}, at=now)
did.append(('blob_replication', 'mirror failed', h[:16], srv, why))
return did
[docs] def verify(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
"""HEAD every held copy; forget the ones that are gone so run() re-asks."""
now = int(now if now is not None else self.clock())
self.last_verify = now
did: list[tuple[Any, ...]] = []
for h, _want in self.db.wants():
for r in self.db.replicas(h):
if r['state'] != 'held':
continue
try:
meta = self.client_for(r['server']).head_blob(r['server'], h, allow_private=self.allow_private())
except Exception as e:
# Unreachable is not gone: keep the copy, say so, look again later.
did.append(('blob_replication', 'verify unreachable', h[:16], r['server'], _why(e)))
continue
if meta is None:
self.db.forget(h, r['server'])
self.lost += 1
self.refusals.refuse('blob_replication', 'copy lost', subject=r['server'],
detail={'blob': h[:16]}, at=now)
did.append(('blob_replication', 'copy lost', h[:16], r['server']))
else:
self.db.mark(h, r['server'], 'held', now)
return did
# ---- the compartment contract -------------------------------------------------
_tick_runs_every = True
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
if self._closed:
return []
if not hasattr(self, '_every_last'):
self._every_last = {} # not a store: Every job timing tracking
if 'replication_verify' not in self._every_last:
self._every_last['replication_verify'] = now
self.last_verify = now
return self.run_every(now)
[docs] def state(self) -> BlobReplicationStateReport:
if self._closed:
# health() is read after stop(); report the counters, not the store.
return {'stopped': True, 'mirrored': self.mirrored, 'lost': self.lost,
'last_run': self.last_run, 'last_verify': self.last_verify}
st = self.status()
return {'blobs': len(st),
'under_replicated': sum(1 for s in st if s['held'] < s['copies']),
'servers': len(self.servers()),
'source_url': self.source_url(),
'mirrored': self.mirrored, 'lost': self.lost,
'last_run': self.last_run, 'last_verify': self.last_verify}
[docs] def trouble(self) -> list[str]:
if self._closed:
return []
st = self.status()
if not st:
return []
out: list[str] = []
servers = self.servers()
if self.source_url() is None and not any(_is_peer(s) for s in servers):
out.append('%d pinned blob(s) want copies, but this node advertises no http(s)/ws(s) '
'endpoint (or blob_replication.source_url) for partners to mirror from, '
'and no partner takes blob/1.0 over the transport' % len(st))
elif not servers:
out.append('%d pinned blob(s) want copies, and there is no partner with a Blossom '
'endpoint or a transport path and no blob_replication.servers to hold them' % len(st))
under = [s for s in st if s['held'] < s['copies']]
if under:
out.append('%d pinned blob(s) below their copy target' % len(under))
return out
[docs] def pin_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
return self.pin(str(args['sha256']), args.get('copies'))
[docs] def unpin_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
return self.unpin(str(args['sha256']))
[docs] def status_cmd(self, args: Mapping[str, Any]) -> list[Mapping[str, Any]]:
h = args.get('sha256')
return cast(list[Mapping[str, Any]], self.status(str(h) if h is not None else None))
[docs] def run_cmd(self) -> list[tuple[Any, ...]]:
return self.run()
[docs] def verify_cmd(self) -> list[tuple[Any, ...]]:
return self.verify()
[docs] def stop(self) -> Any:
self._closed = True
try:
self.db.close()
except Exception:
pass
super().stop()
return None
def _is_peer(server: str) -> bool:
return isinstance(server, str) and server.startswith('peer:')
def _why(e: Exception) -> str:
if isinstance(e, (urllib.error.HTTPError, netfetch.FetchHTTPError)):
return 'HTTP %d' % e.code
return '%s: %s' % (type(e).__name__, str(e)[:120])
[docs]def partner_blossom_servers(node: Any) -> list[str]:
"""Blossom URLs of every peer this node holds a grant with, from their descriptors."""
out: list[str] = []
grants = getattr(node, 'grants', None)
peers = getattr(node, 'peers', None)
if grants is None or peers is None:
return out
for pk in grants.peers():
try:
d = peers.descriptor(pk)
except Exception:
d = None
if not d:
continue
eps = sorted((t for t in d.get('tags', []) if t and t[0] == 'endpoint' and len(t) > 2
and t[1] in HTTP_SCHEMES),
key=lambda t: int(t[3]) if len(t) > 3 and str(t[3]).isdigit() else 50)
for t in eps:
u = blossom_url(t[2])
if u and u not in out:
out.append(u)
break
return out