Source code for fleetmesh_node.planes.store.retention.compartment

"""What this node stops keeping, and the one thing it must never stop keeping.

An open relay face collects what an open face invites. On the alpha mesh one
node was holding 6,025 events to every other node's 38, of which 5,625 were
NIP-59 gift wraps between strangers using it as an ordinary DM relay - about
5,200 arriving a day, 62 MB and climbing, on a box with 113 MB of memory free.
Nothing bounded it: NIP-40 expiry is enforced when an event is read and never on
disk, and only 68 of those 6,025 carried an expiration tag at all, so leaning on
other people's compliance would have reclaimed nothing.

So retention is the operator's, stated per kind, and **the default keeps
everything**. A relay that quietly began deleting what it was handed would be a
worse thing than one that fills up, and the specification prices retrieval
rather than retention precisely because a standing promise to hold something is
the one product this protocol declines to sell. Nothing here is protocol. It is
a disk, and whose disk it is decides.

The guard is the part that is not a preference. A kind 30803 coverage claim
names a shard as a filter and carries a negentropy fingerprint over the set;
evicting an event inside a claimed shard would leave this node advertising a
fingerprint it can no longer reproduce, and a peer reconciling against it would
find the difference and be right. So an event matching any shard this node
claims is never evicted, whatever the operator asked for, and the refusal is
recorded rather than performed silently.
"""
from __future__ import annotations

import dataclasses
from typing import Any, Iterator, Mapping, Optional
from fleetmesh_core.storage import store as store_mod  # noqa: E402
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Every, Need  # noqa: E402
from fleetmesh_node.config.parameter import Parameter, ParamBlock
from .types import (
    RetentionStateReport,
)

# A sweep walks the store, so it does not run on every tick. An hour is far
# finer than any age worth setting and coarse enough to cost nothing.
SWEEP_SECONDS: int = 3600
# How many events one page of a sweep holds. The sweep reads events to decide
# whether a shard claim protects them, and the backlog it reads on the first
# sweep after a policy is set is unbounded by nature - days of an open face.
SWEEP_PAGE: int = 500


[docs]@dataclasses.dataclass(frozen=True) class RetentionConfig(ParamBlock): key = 'retention' name = 'retention' DERIVED: bool = False PARAMS = ( Parameter(key='sweep_seconds', name='Sweep Cadence', description='Seconds between retention sweeps', type=int, default=SWEEP_SECONDS, tags=('retention', 'sweep')), Parameter(key='sweep_page', name='Sweep Page Size', description='Number of events examined per sweep page', type=int, default=SWEEP_PAGE, tags=('retention', 'sweep')), ) sweep_seconds: int = SWEEP_SECONDS sweep_page: int = SWEEP_PAGE
[docs]class Retention(Compartment): """Per-kind ages, swept on a slow cadence, with the shards held back.""" name = 'retention' reads = ('retention', 'coverage') live = ('retention',) config_cls = RetentionConfig CHANNELS = ( Channel('events_evicted', 'Expired events evicted during sweep', level=NOTE, say='retention sweep evicted {count} expired events', tags=('retention', 'sweep')), Channel('eviction_refused', 'Event eviction refused because it matches a claimed shard', level=NOTE, say='retention eviction refused for event {id}', tags=('retention', 'sweep')), ) METRICS = ( Metric(name='retention_events_evicted', description='Times expired events were evicted.', kind=COUNTER, counts='events_evicted', tags=('retention', 'sweep')), Metric(name='retention_evictions_refused', description='Times evictions were refused.', kind=COUNTER, counts='eviction_refused', tags=('retention', 'sweep')), ) NEEDS = ( Need('store', 'stores.events'), Need('cfg', 'cfg', getter=True), ) EVERY = (Every(seconds=SWEEP_SECONDS, run='sweep'),) COMMANDS = ( Command( key='show', name='Retention', description='the ages this operator set, what they protect, and what has been evicted', tags=('retention', 'status'), permission='retention/show', run='state', reply='{ages, protected, swept_at, evicted, held_back}', ), Command( key='sweep', name='Retention sweep', description='run the sweep now rather than waiting for the cadence', tags=('retention',), permission='retention/sweep', run='sweep', reply='{evicted: {kind: n}, held_back: n, scanned: n}', ), ) STORES = ( st.Store(key='evicted', description='total ever evicted, per event kind', kind=st.TABLE, shape=int, bound=1024, keyed_by='kind', evict=st.LRU), )
[docs] def __init__(self, store: Any, cfg: Any, clock: Any, node: Any = None) -> None: Compartment.__init__(self, node, clock) self.store = store self._cfg = cfg # a callable returning the live config self.swept_at: Optional[int] = None self.evicted: st.Table[int, int] # kind -> total ever evicted here self.held_back: int = 0 # events an age would have taken and a shard kept
# ---- what the operator asked for ----------------------------------------
[docs] def ages(self) -> dict[int, int]: """{kind: seconds}. Absent, empty or unreadable all mean keep everything.""" raw = (self._cfg() or {}).get('retention') or {} if not isinstance(raw, dict): return {} out: dict[int, int] = {} for k, v in raw.items(): try: kind, age = int(k), int(v) except (TypeError, ValueError): self.refusals.refuse('retention', 'not a kind and a number of seconds', at=self.clock(), detail={'kind': str(k)[:16], 'age': str(v)[:16]}) continue if age > 0: out[kind] = age return out
[docs] def protected(self) -> list[Mapping[str, Any]]: """The shard filters a coverage claim or ProjectedNodeStore covers. Never evicted.""" cov = (self._cfg() or {}).get('coverage') or [] out: list[Mapping[str, Any]] = [ c['filter'] for c in cov if isinstance(c, dict) and isinstance(c.get('filter'), dict) ] if hasattr(self.store, 'protected_filters'): for f in self.store.protected_filters(): if f not in out: out.append(f) return out
# ---- the sweep ----------------------------------------------------------
[docs] def sweep(self, now: Optional[int] = None) -> Mapping[str, Any]: now = now if now is not None else self.clock() ages, guard = self.ages(), self.protected() evicted: dict[int, int] = {} held_back = 0 scanned = 0 if not ages: self.swept_at = now return {'evicted': evicted, 'held_back': held_back, 'scanned': scanned} for kind, age in sorted(ages.items()): cutoff = now - age for ev in self._older(kind, cutoff): scanned += 1 if any(store_mod.matches(ev, f) for f in guard): # Not a preference and not overridable: the fingerprint this # node published over that shard has to stay reproducible. held_back += 1 self.held_back += 1 self.refusals.refuse('retention', 'inside a shard this node claims coverage for', at=now, detail={'id': ev['id'][:16], 'kind': kind}) continue self.store.forget(ev['id']) evicted[kind] = evicted.get(kind, 0) + 1 self.evicted[kind] = self.evicted.get(kind, 0) + 1 if sum(evicted.values()) > 0 and hasattr(self.store, 'vacuum'): try: self.store.vacuum(pages=50) except Exception: pass self.swept_at = now return {'evicted': evicted, 'held_back': held_back, 'scanned': scanned}
def _older(self, kind: int, cutoff: int) -> Iterator[Mapping[str, Any]]: """One page at a time, oldest first, so the sweep costs a page and not the backlog. The first sweep after a policy is set is the big one.""" after: Optional[tuple[int, str]] = None while True: page = self.store.by_kind_before(kind, cutoff, SWEEP_PAGE, after) if not page: return for ev in page: yield ev if len(page) < SWEEP_PAGE: return after = (page[-1]['created_at'], page[-1]['id']) # ---- the contract ------------------------------------------------------- def _report_sweep(self, r: Mapping[str, Any]) -> list[tuple[Any, ...]]: did: list[tuple[Any, ...]] = [] if r.get('evicted'): did.append(('evicted', sum(r['evicted'].values()), r['evicted'])) if r.get('held_back'): did.append(('held back', r['held_back'], 'inside a claimed shard')) return did
[docs] def state(self) -> RetentionStateReport: return {'ages': {str(k): v for k, v in self.ages().items()}, 'protected': len(self.protected()), 'swept_at': self.swept_at, 'evicted': {str(k): v for k, v in self.evicted.items()}, 'held_back': self.held_back}
[docs] def trouble(self) -> list[str]: out: list[str] = [] if self.held_back: out.append('%d event(s) an age would have taken are inside a shard this node claims, and ' 'were kept: a coverage fingerprint has to stay reproducible' % self.held_back) return out