Source code for fleetmesh_node.compartment.stores

"""What a compartment holds: its in-memory collections, declared and bounded.

The sixth thing a compartment declares, beside PARAMS (what it can be told),
CHANNELS (what it says), METRICS (what it measures), COMMANDS (what it can be
asked to do) and LIFECYCLE (how it fails): STORES, what it keeps.

Before this a compartment's memory was whatever attributes its __init__
happened to assign. Nothing outside it knew they existed, so nobody could ask
how much announce was holding, which collections grow with the number of
peers, or what a command reads. Some were trimmed by hand (`del self.log
[:-200]` in capacity) and some were not trimmed at all, and the difference was
visible only to someone reading the code.

A Store declares one collection:

    key        the attribute it is reached under, and its name everywhere else
    kind       RING      a log, newest last, oldest dropped at `bound`
               TABLE     a mapping, keyed by `keyed_by`
               SET       members, keyed by `keyed_by`
               COUNTERS  a fixed set of named integers, the fields of `shape`
    shape      the type of one entry (one row, one member, one record). A
               TypedDict is checked on every write: required keys present, no
               key it does not name. A class is checked with isinstance. A
               write that does not fit raises TypeError - a programming error,
               and the lifecycle degrades the compartment that made it.
    bound      how many entries it may hold. Required: a collection that grows
               with the outside world and says nothing about how far is the
               failure this exists to prevent. COUNTERS are bounded by their
               shape and take no bound.
    evict      what happens at the bound. OLDEST drops the earliest written,
               LRU the least recently read or written, REFUSE refuses the
               write (StoreFull, or False from `put`) - for state that must
               not be silently lost, where the caller has to decide.
    keyed_by   what makes it grow: 'peer', 'event', 'url'. Required on a
               TABLE and a SET, because that is the question an operator asks.
    persist    MEMORY (gone with the process) or SNAPSHOT (carried across a
               process restart by the lifecycle, fleetmesh_node/lifecycle phase 4)
    sensitive  its entries are never shown in a report, only its size

and the compartment base class builds a container for each one - Ring, Table,
Members, Counters - that enforces the bound and the shape, and counts itself:
inserts, evictions, refusals and its high-water mark. So the declaration and
the behaviour cannot drift apart, and the node meters every compartment's
memory without the compartment writing a metric.

What is NOT a store: a list of callbacks (wiring), a config cache, a small
fixed lookup table, a per-packet buffer on a hot path. The rule is whether
the collection grows with the outside world.

Durable data (the event store, grants, the change log) lives in sqlite and is
not declared here yet. A persist value for it arrives with the code that
honours it.

A package rather than a root module for the reason fleetmesh_node/channels is one:
fleetmesh_node/*.py is held to a monotone baseline (fleetmesh.org/scripts/check-root.py).
Standard library only.
"""
from __future__ import annotations

import collections
import dataclasses
import re
import threading
from typing import (
    Any,
    Generic,
    Hashable,
    Iterable,
    Iterator,
    Mapping,
    MutableMapping,
    Optional,
    Sequence,
    TypedDict,
    TypeVar,
    Union,
    is_typeddict,
)

# ---- the vocabulary -------------------------------------------------------------
RING = 'ring'
TABLE = 'table'
SET = 'set'
COUNTERS = 'counters'
KINDS: tuple[str, ...] = (RING, TABLE, SET, COUNTERS)

OLDEST = 'oldest'
LRU = 'lru'
REFUSE = 'refuse'
EVICTIONS: tuple[str, ...] = (OLDEST, LRU, REFUSE)

MEMORY = 'memory'
SNAPSHOT = 'snapshot'
PERSISTS: tuple[str, ...] = (MEMORY, SNAPSHOT)

# A REFUSE store this full is trouble before it is refusing: the operator
# hears about it while there is still room to act.
NEARLY_FULL = 0.9

_KEY = re.compile(r'[a-z_][a-z0-9_]*$')


[docs]class StoreFull(Exception): """A REFUSE store at its bound was asked to take a new entry."""
[docs]def shape_problem(shape: Optional[type], value: object) -> str: """Why `value` is not one `shape`, or '' when it is (or nothing is declared). Shallow by design: a TypedDict's keys, not its values' types. The keys are where drift happens - a field renamed in one writer and not another - and checking them costs a set comparison per write. """ if shape is None: return '' if is_typeddict(shape): if not isinstance(value, dict): return 'a %s is a dict, not %s' % (shape.__name__, type(value).__name__) required: frozenset[str] = getattr(shape, '__required_keys__', frozenset()) known = required | getattr(shape, '__optional_keys__', frozenset()) missing = sorted(required - value.keys()) extra = sorted(k for k in value if k not in known) if missing or extra: return 'a %s %s' % (shape.__name__, '; '.join( (['is missing %s' % ', '.join(missing)] if missing else []) + (['has no field %s' % ', '.join(extra)] if extra else []))) return '' if isinstance(shape, type) and not isinstance(value, shape): return 'expected %s, got %s' % (shape.__name__, type(value).__name__) return ''
[docs]class StoreRecord(TypedDict, total=False): """A store as the census, `meshnode subsystems` and the docs read it.""" key: str description: str kind: str shape: str bound: int evict: str keyed_by: str persist: str sensitive: bool tags: list[str] compartment: str
[docs]class StoreLive(TypedDict): """What a store is holding now, for a compartment's report.""" kind: str size: int bound: int high_water: int inserts: int evictions: int refused: int
[docs]@dataclasses.dataclass(frozen=True) class Store: """One collection a compartment keeps. See the module docstring.""" key: str description: str kind: str shape: Optional[type] = None bound: int = 0 evict: str = OLDEST keyed_by: str = '' persist: str = MEMORY sensitive: bool = False tags: tuple[str, ...] = () @property def limit(self) -> int: """The bound in force: declared, or a COUNTERS store's field count.""" if self.kind == COUNTERS: return len(_fields(self.shape)) return self.bound
[docs] def problems(self) -> list[str]: """Everything wrong with this declaration, in sentences, collected.""" out: list[str] = [] if not _KEY.match(self.key or ''): out.append('store %r: a key is lowercase letters, digits and underscores' % self.key) if not self.description: out.append('store %s: say what it holds (description)' % self.key) if self.kind not in KINDS: out.append('store %s: kind must be one of %s' % (self.key, ', '.join(KINDS))) if self.persist not in PERSISTS: out.append('store %s: persist must be one of %s' % (self.key, ', '.join(PERSISTS))) if self.kind == COUNTERS: if not is_typeddict(self.shape): out.append('store %s: COUNTERS name their fields with a TypedDict shape' % self.key) if self.bound: out.append('store %s: COUNTERS are bounded by their shape; drop bound' % self.key) return out if self.bound <= 0: out.append('store %s: a %s must declare its bound - how far it may grow' % (self.key, self.kind)) if self.evict not in EVICTIONS: out.append('store %s: evict must be one of %s' % (self.key, ', '.join(EVICTIONS))) if self.kind == RING and self.evict != OLDEST: out.append('store %s: a ring drops its oldest entry; evict is OLDEST' % self.key) if self.kind in (TABLE, SET) and not self.keyed_by: out.append('store %s: a %s says what it grows with (keyed_by: peer, event, url ...)' % (self.key, self.kind)) return out
[docs] def describe(self) -> StoreRecord: out: StoreRecord = {'key': self.key, 'description': self.description, 'kind': self.kind, 'bound': self.limit, 'persist': self.persist, 'tags': list(self.tags)} if self.shape is not None: out['shape'] = getattr(self.shape, '__name__', str(self.shape)) if self.kind != COUNTERS: out['evict'] = self.evict if self.keyed_by: out['keyed_by'] = self.keyed_by if self.sensitive: out['sensitive'] = True return out
[docs] def build(self) -> 'Container': """The container that keeps this declaration's promises.""" match self.kind: case 'ring': return Ring(self) case 'table': return Table(self) case 'set': return Members(self) case _: return Counters(self)
def _fields(shape: Optional[type]) -> tuple[str, ...]: if shape is None or not is_typeddict(shape): return () return tuple(getattr(shape, '__annotations__', {}))
[docs]def declared(obj: Any) -> tuple[Store, ...]: """The stores a compartment declares, or () for one not yet converted.""" return tuple(getattr(obj, 'STORES', ()) or ())
[docs]def every_store(compartments: Sequence[Any]) -> list[StoreRecord]: """Every declared store: what the node keeps, beside what it says (every_channel), measures (every_metric) and can be told (every_parameter).""" out: list[StoreRecord] = [] for c in compartments: for s in declared(c): row = s.describe() row['compartment'] = getattr(c, 'name', '') or getattr(c, '__name__', str(c)) out.append(row) return out
# ---- the containers --------------------------------------------------------------- class _Container: """What every container shares: the declaration, a lock, and its counts. Locked because the tick writes on one thread while the control socket reads `state` on another; iterating a dict while it is resized raises. Every read that iterates takes a copy under the lock. """ def __init__(self, store: Store) -> None: self.store: Store = store self._lock: threading.RLock = threading.RLock() self.inserts: int = 0 self.evictions: int = 0 self.refused: int = 0 self.high_water: int = 0 # Each container says how big it is; the counts are shared. def __len__(self) -> int: raise NotImplementedError def _check(self, value: object) -> None: why = shape_problem(self.store.shape, value) if why: raise TypeError('store %s: %s' % (self.store.key, why)) def _grew(self) -> None: self.inserts += 1 n = len(self) if n > self.high_water: self.high_water = n def live(self) -> StoreLive: return {'kind': self.store.kind, 'size': len(self), 'bound': self.store.limit, 'high_water': self.high_water, 'inserts': self.inserts, 'evictions': self.evictions, 'refused': self.refused} def nearly_full(self) -> bool: lim = self.store.limit return bool(lim) and len(self) >= NEARLY_FULL * lim # ---- carried across a process restart (persist=SNAPSHOT) ----------------- def dump(self) -> object: """The contents as JSON-able data.""" raise NotImplementedError def check(self, data: object) -> None: """Raise ValueError unless `data` is a dump this store can load whole. Separate from load() so a compartment's restore is all or nothing.""" raise NotImplementedError def load(self, data: object) -> None: """Replace the contents with a checked dump. Not counted as inserts.""" raise NotImplementedError V = TypeVar('V') K = TypeVar('K', bound=Hashable)
[docs]class Ring(_Container, Generic[V]): """A log: newest last, the oldest dropped at the bound."""
[docs] def __init__(self, store: Store) -> None: super().__init__(store) self._items: collections.deque[V] = collections.deque()
def __len__(self) -> int: return len(self._items)
[docs] def append(self, item: V) -> None: self._check(item) with self._lock: self._items.append(item) while len(self._items) > self.store.bound: self._items.popleft() self.evictions += 1 self._grew()
[docs] def extend(self, items: Iterable[V]) -> None: for item in items: self.append(item)
[docs] def last(self, n: int) -> list[V]: with self._lock: return list(self._items)[-n:] if n > 0 else []
[docs] def discard(self, item: V) -> bool: """Take back the newest entry equal to `item`; False if none is kept. For a log written before the thing it records, and undone when that thing failed (sync records a handover, then sends it). The newest, because the entry being undone is the one just written. Not an eviction: nothing was dropped for room, so the counts stay as they are. """ with self._lock: for i in range(len(self._items) - 1, -1, -1): if self._items[i] == item: del self._items[i] return True return False
def __getitem__(self, index: Union[int, slice]) -> Any: with self._lock: if isinstance(index, slice): return list(self._items)[index] return self._items[index] def __iter__(self) -> Iterator[V]: with self._lock: return iter(list(self._items))
[docs] def clear(self) -> None: with self._lock: self._items.clear()
[docs] def dump(self) -> object: with self._lock: return list(self._items)
[docs] def check(self, data: object) -> None: if not isinstance(data, list): raise ValueError('store %s: a ring dump is a list' % self.store.key) for item in data: why = shape_problem(self.store.shape, item) if why: raise ValueError('store %s: %s' % (self.store.key, why))
[docs] def load(self, data: object) -> None: self.check(data) assert isinstance(data, list) with self._lock: self._items = collections.deque(data[-self.store.bound:]) self.high_water = max(self.high_water, len(self._items))
[docs]class Table(_Container, MutableMapping[K, V]): """A bounded mapping. Reads refresh an entry's place when evict is LRU."""
[docs] def __init__(self, store: Store) -> None: super().__init__(store) self._rows: collections.OrderedDict[K, V] = collections.OrderedDict()
def __len__(self) -> int: return len(self._rows) def __getitem__(self, key: K) -> V: with self._lock: v = self._rows[key] if self.store.evict == LRU: self._rows.move_to_end(key) return v def __setitem__(self, key: K, value: V) -> None: if not self.put(key, value): raise StoreFull('store %s is full (%d %s); it refuses new entries rather than drop one' % (self.store.key, self.store.bound, self.store.keyed_by or 'entries'))
[docs] def put(self, key: K, value: V) -> bool: """Write one entry. False - and counted - when a REFUSE store is full.""" self._check(value) with self._lock: if key in self._rows: self._rows[key] = value if self.store.evict == LRU: self._rows.move_to_end(key) return True if len(self._rows) >= self.store.bound: if self.store.evict == REFUSE: self.refused += 1 return False self._rows.popitem(last=False) self.evictions += 1 self._rows[key] = value self._grew() return True
def __delitem__(self, key: K) -> None: with self._lock: del self._rows[key] def __iter__(self) -> Iterator[K]: with self._lock: return iter(list(self._rows)) def __contains__(self, key: object) -> bool: with self._lock: return key in self._rows def __repr__(self) -> str: return 'Table(%s, %d/%d)' % (self.store.key, len(self), self.store.bound)
[docs] def dump(self) -> object: # Pairs, not an object: a JSON object's keys are strings, and order # is what LRU and OLDEST evict by. with self._lock: return [[k, v] for k, v in self._rows.items()]
[docs] def check(self, data: object) -> None: if not isinstance(data, list) or not all(isinstance(p, list) and len(p) == 2 for p in data): raise ValueError('store %s: a table dump is a list of [key, value] pairs' % self.store.key) if len(data) > self.store.bound: raise ValueError('store %s: %d entries, more than its bound of %d' % (self.store.key, len(data), self.store.bound)) for k, v in data: if not isinstance(k, (str, int)): raise ValueError('store %s: key %r is not a string or an integer' % (self.store.key, k)) why = shape_problem(self.store.shape, v) if why: raise ValueError('store %s: %s' % (self.store.key, why))
[docs] def load(self, data: object) -> None: self.check(data) assert isinstance(data, list) with self._lock: self._rows = collections.OrderedDict((k, v) for k, v in data) self.high_water = max(self.high_water, len(self._rows))
[docs]class Members(_Container, Generic[K]): """A bounded set. Evicts in the order members were added (or touched, LRU)."""
[docs] def __init__(self, store: Store) -> None: super().__init__(store) self._m: collections.OrderedDict[K, None] = collections.OrderedDict()
def __len__(self) -> int: return len(self._m)
[docs] def add(self, member: K) -> bool: """Add one. False - and counted - when a REFUSE set is full.""" self._check(member) with self._lock: if member in self._m: if self.store.evict == LRU: self._m.move_to_end(member) return True if len(self._m) >= self.store.bound: if self.store.evict == REFUSE: self.refused += 1 return False self._m.popitem(last=False) self.evictions += 1 self._m[member] = None self._grew() return True
[docs] def discard(self, member: K) -> None: with self._lock: self._m.pop(member, None)
[docs] def remove(self, member: K) -> None: with self._lock: if member not in self._m: raise KeyError(member) del self._m[member]
[docs] def clear(self) -> None: with self._lock: self._m.clear()
def __contains__(self, member: object) -> bool: with self._lock: return member in self._m def __iter__(self) -> Iterator[K]: with self._lock: return iter(list(self._m))
[docs] def dump(self) -> object: with self._lock: return list(self._m)
[docs] def check(self, data: object) -> None: if not isinstance(data, list) or len(data) > self.store.bound: raise ValueError('store %s: a set dump is a list within its bound' % self.store.key) for m in data: why = shape_problem(self.store.shape, m) if why: raise ValueError('store %s: %s' % (self.store.key, why))
[docs] def load(self, data: object) -> None: self.check(data) assert isinstance(data, list) with self._lock: self._m = collections.OrderedDict((m, None) for m in data) self.high_water = max(self.high_water, len(self._m))
[docs]class Counters(_Container): """A fixed set of named integers - the fields of the store's TypedDict."""
[docs] def __init__(self, store: Store) -> None: super().__init__(store) self._n: dict[str, int] = {f: 0 for f in _fields(store.shape)} self.high_water = len(self._n)
def __len__(self) -> int: return len(self._n) def __getitem__(self, field: str) -> int: return self._n[field] def __setitem__(self, field: str, value: int) -> None: if field not in self._n: raise KeyError('store %s has no counter %r; its fields are %s' % (self.store.key, field, ', '.join(self._n))) if not isinstance(value, int) or isinstance(value, bool): raise TypeError('store %s: %s is a count, not %s' % (self.store.key, field, type(value).__name__)) with self._lock: self._n[field] = value
[docs] def incr(self, field: str, n: int = 1) -> int: with self._lock: self[field] = self._n.get(field, 0) + n self.inserts += 1 return self._n[field]
[docs] def as_dict(self) -> dict[str, int]: with self._lock: return dict(self._n)
def __iter__(self) -> Iterator[str]: with self._lock: return iter(list(self._n))
[docs] def items(self) -> list[tuple[str, int]]: with self._lock: return list(self._n.items())
[docs] def keys(self) -> list[str]: with self._lock: return list(self._n.keys())
[docs] def values(self) -> list[int]: with self._lock: return list(self._n.values())
[docs] def get(self, field: str, default: Optional[int] = None) -> Optional[int]: with self._lock: return self._n.get(field, default)
[docs] def dump(self) -> object: return self.as_dict()
[docs] def check(self, data: object) -> None: if not isinstance(data, dict): raise ValueError('store %s: a counters dump is an object' % self.store.key) for f in self._n: v = data.get(f) if not isinstance(v, int) or isinstance(v, bool) or v < 0: raise ValueError('store %s: %s is %r, not a count' % (self.store.key, f, v))
[docs] def load(self, data: object) -> None: self.check(data) assert isinstance(data, dict) with self._lock: self._n = {f: int(data[f]) for f in self._n}
Container = Union[Ring[Any], Table[Any, Any], Members[Any], Counters]
[docs]class Holdings: """One compartment's containers, built from its STORES. Owned by the base."""
[docs] def __init__(self, stores: Sequence[Store]) -> None: self._by_key: dict[str, Container] = {s.key: s.build() for s in stores}
def __getitem__(self, key: str) -> Container: return self._by_key[key] def __iter__(self) -> Iterator[str]: return iter(self._by_key) def __len__(self) -> int: return len(self._by_key)
[docs] def items(self) -> list[tuple[str, Container]]: return list(self._by_key.items())
[docs] def live(self) -> dict[str, StoreLive]: return {k: c.live() for k, c in self._by_key.items()}
[docs] def persisted(self) -> dict[str, Container]: return {k: c for k, c in self._by_key.items() if c.store.persist == SNAPSHOT}
[docs] def snapshot(self) -> Mapping[str, Any]: return {k: c.dump() for k, c in self.persisted().items()}
[docs] def restore(self, state: Mapping[str, Any]) -> None: """All or nothing: every persisted store checks its dump, then all load.""" mine = self.persisted() for k, c in mine.items(): if k not in state: raise ValueError('the snapshot has no store %s' % k) c.check(state[k]) for k, c in mine.items(): c.load(state[k])
__all__ = [ 'COUNTERS', 'Container', 'Counters', 'EVICTIONS', 'Holdings', 'KINDS', 'LRU', 'MEMORY', 'Members', 'NEARLY_FULL', 'OLDEST', 'PERSISTS', 'REFUSE', 'RING', 'Ring', 'SET', 'SNAPSHOT', 'Store', 'StoreFull', 'StoreLive', 'StoreRecord', 'TABLE', 'Table', 'declared', 'every_store', 'shape_problem', ]