Source code for fleetmesh_node.planes.peering.capacity.compartment

"""The node's own ceiling, held across ticks, and the shed ladder it drives.

`fleetmesh_core/capacity.py` is the arithmetic and states why there is no default total.
This is the part that remembers: which rungs are engaged and since when, what a
grant's limits were before a shed so they can be handed back, and the largest
window this machine has actually carried so an operator has a number to start
from rather than a blank.

The total is read from config under `capacity` and is live. An operator may set
it, raise it or delete it while the node runs and the next tick follows the
file, which is the same contract every other live key has: the file is the
interface and the daemon follows it.
"""
from __future__ import annotations

from typing import Any, Callable, Mapping, Optional, cast

from fleetmesh_core.accounting import capacity as ref_capacity, window
from fleetmesh_node import channels as ch_mod
from fleetmesh_node.storage import grants as GRANTS
from fleetmesh_node.compartment import lifecycle as lifecycle_mod
from fleetmesh_node import metrics as mx_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.config import CapacityConfig
from fleetmesh_node.compartment import Command, Compartment, Need


from .types import RungChange, CapacityStateReport


[docs]class Capacity(Compartment): requires: tuple[str, ...] = ('budget',) """The node's own ceiling, as a compartment the operator can watch. What it declares: the config key `capacity`, live; one control command; a state an operator reads; the trouble it is in; and every peering it turned away, with the reason, in the refusal channel every compartment has. Before that channel existed a full node declined a stranger and recorded nothing, so the only trace of the first shed rung was a line in somebody's journal. """ name: str = 'capacity' reads: tuple[str, ...] = ('capacity',) config_cls = CapacityConfig live: tuple[str, ...] = ('capacity',) NEEDS = ( Need('budget'), Need('total', block=CapacityConfig, field='total'), Need('grants', 'stores.grants'), Need('amend', 'amend'), ) COMMANDS = ( Command( key='show', name='Capacity', description="the declared total, the pressure on it, the rungs engaged and the largest window carried", tags=('capacity', 'status'), permission='capacity/show', run='show', reply='{declared, total, pressure, binding, rungs, held, since, measured, peak, suggested, note}', ), ) # What it keeps. The ladder's own state (`held`, `since`) is a handful of # rung names and is not a store: it does not grow with the outside world. STORES = ( st.Store(key='log', description='rungs engaged and released, newest last', kind=st.RING, shape=RungChange, bound=200), # REFUSE, not OLDEST: dropping a peer's remembered limits would leave # it shed for good when the pressure lifts. A peer that cannot be # remembered is not cut instead (shed_grants). # # SNAPSHOT: a restart in the middle of a shed came back with an empty # ladder and an empty table - the peers stayed cut, and nothing knew # what to give back. Carried, it pairs with the reconcile in tick(): # whatever is still held back while amend_down is off is returned. st.Store(key='before', description='the limits each peer held before a shed, to give back', kind=st.TABLE, shape=dict, bound=4096, evict=st.REFUSE, keyed_by='peer', persist=st.SNAPSHOT), # Carried across a restart: "the largest window this machine has # actually carried" is the number an operator sizes a total from, # and an upgrade used to reset it to nothing. st.Store(key='peak', description='the largest window carried, per counter', kind=st.TABLE, shape=int, bound=64, evict=st.REFUSE, keyed_by='counter', persist=st.SNAPSHOT), ) CHANNELS = ( ch_mod.Channel('rung_engaged', 'A capacity shed ladder rung engaged under pressure', level=ch_mod.TROUBLE, say='capacity shed rung {rung} engaged at {pressure}% pressure on {binding}', tags=('capacity', 'shed')), ch_mod.Channel('rung_released', 'A capacity shed ladder rung released as pressure eased', level=ch_mod.NOTE, say='capacity shed rung {rung} released', tags=('capacity', 'shed')), ) METRICS = ( mx_mod.Metric(name='capacity_rungs_engaged', description='Times a capacity shed rung engaged.', kind=mx_mod.COUNTER, counts='rung_engaged', tags=('capacity', 'shed')), mx_mod.Metric(name='capacity_rungs_released', description='Times a capacity shed rung released.', kind=mx_mod.COUNTER, counts='rung_released', tags=('capacity', 'shed')), ) # v1 is `peak` and `before`. `before` joined v1 before v1 ever left this # repository, so no deployed snapshot has the shape without it; a local # one that does is refused by the restore ("no store before") and the # compartment starts fresh, which is the honest outcome. LIFECYCLE = lifecycle_mod.Lifecycle(snapshot=1)
[docs] def __init__( self, budget: Any, total: Optional[Mapping[str, float | int]] = None, clock: Optional[Callable[[], int]] = None, node: Optional[Any] = None, grants: Optional[Any] = None, amend: Optional[Callable[..., tuple[Any, Any]]] = None, ) -> None: super().__init__(node, clock) self.budget: Any = budget # The grant store a shed reads live grants from and the node call # that amends one; a fake node in a suite supplies both. self.grants: Optional[Any] = grants if grants is not None else getattr(node, 'grants', None) self.amend: Callable[..., tuple[Any, Any]] = amend or (lambda *a: getattr(node, '_amend')(*a)) self.total: dict[str, float | int] = dict(total or {}) # not a store: the operator's declared ceilings, from config self.held: list[str] = [] # not a store: the rungs engaged, at most the four of the ladder self.since: dict[str, int] = {} # not a store: when each held rung engaged, one per rung # The stores (`log`, `before`, `peak`) were built and bound by the base. self.log: st.Ring[RungChange] self.before: st.Table[str, Mapping[str, Any]] self.peak: st.Table[str, int]
# ---- the number the operator declares -------------------------------------
[docs] def set_total(self, total: Optional[Mapping[str, Any]]) -> bool: """Take up a new declaration. Returns True if it changed.""" new = {k: v for k, v in (total or {}).items() if isinstance(v, (int, float)) and v > 0} if new == self.total: return False self.total = new return True
[docs] def declared(self) -> bool: return bool(self.total)
# ---- what is happening ----------------------------------------------------
[docs] def measure(self, now: Optional[int] = None) -> dict[str, int]: """The open window, summed across every peer. See Budget.aggregate.""" m: dict[str, int] = self.budget.aggregate(now) for k, v in m.items(): if v > self.peak.get(k, 0): self.peak[k] = v return m
[docs] def pressure(self, now: Optional[int] = None) -> tuple[float, Optional[str]]: return ref_capacity.pressure(self.measure(now), self.total)
[docs] def transitions(self, now: Optional[int] = None) -> list[tuple[str, str]]: """Recompute and return the rungs that changed, as (action, what). The caller acts on the transitions rather than on the state, because a shed and a restore each cost the peers an amend and neither should happen twice for one crossing. """ now_val = now if now is not None else self.clock() if not self.declared(): was, self.held = self.held, [] return [(a, 'released') for a in was] p, _binding = self.pressure(now_val) want = ref_capacity.engaged(p, self.held) changed = ([(a, 'engaged') for a in want if a not in self.held] + [(a, 'released') for a in self.held if a not in want]) for action, what in changed: self.log.append({'at': now_val, 'action': action, 'what': what, 'pressure': round(p, 4)}) if what == 'engaged': self.since[action] = now_val self.say('rung_engaged', rung=action, pressure=round(p * 100), binding=_binding or 'capacity') else: self.since.pop(action, None) self.say('rung_released', rung=action) self.held = want return changed
# ---- what each rung means to a caller -------------------------------------
[docs] def refusing(self) -> bool: return 'refuse_peerings' in self.held
[docs] def narrowing(self) -> bool: return 'narrow_scope' in self.held
[docs] def amending(self) -> bool: return 'amend_down' in self.held
[docs] def leafing(self) -> bool: return 'leaf_behaviour' in self.held
[docs] def why(self) -> str: """The message a decline carries. Names the binding counter and nothing else: a stranger learns this node is full, never who is filling it.""" p, binding = self.pressure() return 'at %d%% of this node\'s declared %s' % (round(p * 100), binding or 'capacity')
# ---- the limits a shed took, so they can be given back ---------------------
[docs] def remember(self, peer: str, limits: Mapping[str, Any]) -> bool: """Keep what a shed is about to take. False when the table is full: the caller must not take what it could not promise to give back.""" if peer in self.before: return True return self.before.put(peer, dict(limits))
[docs] def restore_for(self, peer: str, tier: str) -> Optional[Mapping[str, Any]]: """What this peer should get back, or None if nothing was taken. Held inside the ceilings of the tier it holds NOW, because the ladder may have lowered it while the shed was on and a restore must never undo a remedy. """ was = self.before.pop(peer, None) if was is None: return None return window.within(was, tier)
[docs] def forget(self, peer: str) -> None: self.before.pop(peer, None)
# ---- the compartment contract -------------------------------------------------
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: """Recompute pressure and act on the rungs that changed. Transitions and not state: a shed and a restore each cost the peers an amend, and neither should happen twice for one crossing of a threshold. Then one invariant, checked every tick rather than on a transition: nothing is held back while amend_down is off. The transition alone misses the restart - the ladder comes back empty, so a shed that was on is never seen to release, and `before` (carried) would sit there with the peers still cut. Normally `before` is empty here and this is one length check. """ self.set_total(self.settings().total) # live did: list[tuple[Any, ...]] = [] for action, what in self.transitions(now): p, binding = self.pressure(now) did.append(('capacity', action, what, round(p, 3), binding)) if action == 'amend_down' and what == 'engaged': did += self.shed_grants(now) elif action == 'amend_down' and what == 'released': did += self.restore_grants(now) if self.before and not self.amending(): did += self.restore_grants(now) return did
[docs] def trouble(self) -> list[str]: if not self.held: return [] p, binding = self.pressure() return ['shedding at %d%% of the declared %s: %s' % (round(p * 100), binding or 'total', ', '.join(self.held))]
[docs] def refuse_peering(self, peer: str) -> str: """A stranger turned away because this node is full. Recorded rather than only answered: a decline is not a finding, but a node that is refusing peerings and cannot say how many is a node whose operator finds out from somebody else. """ self.refusals.refuse('peering', 'this node is at its declared ceiling', subject=peer, detail={'pressure': self.why()}, at=self.clock()) return self.why()
# ---- the grants a shed moves ------------------------------------------------
[docs] def shed_grants(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: """Reduce every live grant because this node is full, not because anyone misbehaved. What each peer held is remembered so it can be given back.""" now_val = now if now is not None else self.clock() floor = ref_capacity.shed_floor(window.ceilings('probation')) did: list[tuple[Any, ...]] = [] if self.grants is None: return did # no grant store: nothing to shed for peer in self.grants.peers(): row = self.grants.get(peer, GRANTS.ISSUED) if row is None or row['status'] != 'live': continue if peer in self.before: # Shed and not yet given back. Only a restart reaches this - # the ladder came back empty, `before` came back carried, and # amend_down engaged again - and cutting the already-cut # limits would squeeze the peer twice for one shed. continue limits = row['payload'].get('limits') or {} cut = ref_capacity.reduce(limits, floor) if cut == limits: continue # already on the floor; nothing to take if not self.remember(peer, limits): self.refusals.refuse('grant', 'not reduced: no room to remember what it held', subject=peer, at=now_val) continue env, _ = self.amend(peer, dict(row['payload'], limits=cut), 'this node is shedding load', 'capacity', now_val) self.refusals.refuse('grant', 'reduced because this node is full', subject=peer, detail={'was': limits.get('events_in'), 'now': cut.get('events_in')}, at=now_val) did.append(('shed', peer, env['id'])) return did
[docs] def restore_grants(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: """Give back what a shed took, held inside the tier each peer holds now.""" now_val = now if now is not None else self.clock() did: list[tuple[Any, ...]] = [] if self.grants is None: return did for peer in list(self.before): row = self.grants.get(peer, GRANTS.ISSUED) if row is None or row['status'] != 'live': self.forget(peer) continue back = self.restore_for(peer, row['payload'].get('tier', 'probation')) if back is None or back == (row['payload'].get('limits') or {}): continue env, _ = self.amend(peer, dict(row['payload'], limits=back), 'this node has room again', 'capacity', now_val) did.append(('restored', peer, env['id'])) return did
# ---- the operator's view ---------------------------------------------------
[docs] def show(self) -> Mapping[str, Any]: st = self.state() st['suggested'] = self.suggested() return st
[docs] def suggested(self) -> Mapping[str, Any]: return ref_capacity.suggest([dict(self.peak)]) if self.peak else {}
[docs] def state(self, now: Optional[int] = None) -> CapacityStateReport: # The ref summary is a closed TypedDict; the operator's view adds the # node-side memory on top, so widen it to a plain dict before extending. s = dict(ref_capacity.summary(self.measure(now), self.total)) s['total'] = dict(self.total) s['held'] = list(self.held) s['since'] = {a: self.since[a] for a in self.held if a in self.since} s['peak'] = dict(self.peak) if not s['declared']: s['note'] = ('no aggregate ceiling is declared, so this node will accept load ' 'until it fails; meshnode capacity prints what it has carried') return cast(CapacityStateReport, s)