"""How a compartment lives: built, wired, running, degraded, quarantined, retired.
The fifth thing a compartment declares, beside PARAMS (what it can be told),
CHANNELS (what it says), METRICS (what it measures) and COMMANDS (what it can
be asked to do): LIFECYCLE, how it behaves when it fails.
Before this a compartment had no state. It was built or it was not, and once
built it was ticked forever. A tick that raised was recorded as a refusal and
then raised again on the next tick, and the one after, for as long as the node
ran - a broken compartment cost a traceback per cycle, filled its refusal
ring with one reason, and could not tell an operator anything they did not
already know after the first one. Nothing distinguished "failed once and
recovered" from "has been failing for a day".
So the Registry now carries one `Tracker` per compartment and moves it through
CONFIGURED -> WIRED -> RUNNING <-> DEGRADED -> QUARANTINED -> RUNNING
| ^ (breaker) (trial / resume)
| |
v | restart (only when declared)
DRAINING
|
v
RETIRED start() raised: -> FAILED, or
-> QUARANTINED when restartable
and each move is a declared channel, so the journal, the control socket and
the metrics all see the same transition.
CONFIGURED built from its config block; not yet reaching its neighbours
WIRED back-edges reached (`wire()` ran)
RUNNING started and ticking cleanly
DEGRADED the last tick raised; still ticked, and back to RUNNING after
one clean tick. A note, not trouble: one failure is weather.
QUARANTINED `Breaker.failures` ticks raised within `Breaker.window`
seconds. Not ticked until the backoff passes, then given one
trial tick: clean returns it to RUNNING, a raise sends it back
with the backoff doubled (to `Breaker.backoff[1]` at most).
Trouble, because something is wrong now. `resume` lets an
operator end it early.
FAILED an optional compartment's start() raised and it does not
declare `restart`, so nothing here can safely try again. Not
ticked, not stopped. Trouble. (A required compartment whose
start() raises still stops the node - see Registry.start.)
DRAINING being stopped - for good, or for a restart
RETIRED stopped; never ticked again
`Lifecycle(restart=True)` is a promise the compartment makes: its stop() and
then start() on the SAME instance leave it as good as new. Only a compartment
that makes it is restarted - by the `restart` control command, by a config
reload that changes its own block, by a quarantine's trial (which restarts it
before the trial tick), and after a start() that raised (which quarantines it
rather than failing it). The default is False because most compartments were
written to start once: a restart nobody designed for would read as resilience
the node does not have.
`Lifecycle(snapshot=N)` is the other promise, and it is about the PROCESS
restarting rather than one compartment: a core upgrade, a digest change, a
`systemctl restart`. The compartment implements `snapshot()` - a JSON-able
mapping of the state worth carrying - and `restore(state)`, and N is the
version of that mapping's shape. On a clean stop the node writes every
started compartment's snapshot to SNAPSHOT_FILE; the next start hands each one
back BEFORE start() runs, but only when the version still matches, the file
is recent (SNAPSHOT_MAX_AGE), and restore() returns. Anything else is a
refusal recorded in the compartment's lifecycle (`restored`), and it starts
fresh - which is what it did before snapshots existed, so a refused restore
costs continuity and nothing else. The file is removed once a start
succeeds: a snapshot is handed over once.
restore() must be all-or-nothing: validate first, then assign. A raise is
caught, but a half-applied restore is not something the node can see.
A package rather than a root module for the reason fleetmesh_node/channels is one:
fleetmesh_node/*.py is held to a monotone baseline (fleetmesh.org/scripts/check-root.py).
Standard library only, beside the two surfaces it emits through.
"""
from __future__ import annotations
import collections
import dataclasses
import json
import os
from typing import Any, Callable, Deque, Mapping, NamedTuple, Optional, TypedDict
from ..channels import NOTE, TROUBLE, Channel
from ..metrics import COUNTER, Metric
# ---- the states ---------------------------------------------------------------
CONFIGURED = 'configured'
WIRED = 'wired'
RUNNING = 'running'
DEGRADED = 'degraded'
QUARANTINED = 'quarantined'
FAILED = 'failed'
DRAINING = 'draining'
RETIRED = 'retired'
STATES: tuple[str, ...] = (CONFIGURED, WIRED, RUNNING, DEGRADED, QUARANTINED, FAILED, DRAINING, RETIRED)
# The states in which a compartment is ticked on the ordinary cadence. A
# quarantined one is ticked only for its trial; a retired one never.
TICKED: frozenset[str] = frozenset({CONFIGURED, WIRED, RUNNING, DEGRADED})
[docs]@dataclasses.dataclass(frozen=True)
class Breaker:
"""When repeated failure stops being weather and becomes a fault.
`failures` ticks that raise within `window` seconds trip it. `backoff` is
(first, ceiling) in seconds: the first quarantine lasts `first`, each
failed trial doubles it, and it never exceeds `ceiling`. `failures=0`
turns the breaker off for a compartment that must be ticked whatever
happens - the old behaviour, chosen rather than defaulted into.
"""
failures: int = 5
window: int = 300
backoff: tuple[int, int] = (30, 600)
[docs] def delay(self, trips: int) -> int:
"""Seconds a compartment stays quarantined on its `trips`-th trip."""
first, ceiling = self.backoff
return int(min(first * (2 ** max(trips - 1, 0)), ceiling))
[docs] def problems(self) -> list[str]:
out: list[str] = []
if self.failures < 0:
out.append('breaker failures must be 0 (off) or more')
if self.window <= 0:
out.append('breaker window must be positive')
first, ceiling = self.backoff
if first <= 0 or ceiling < first:
out.append('breaker backoff is (first, ceiling) with 0 < first <= ceiling')
return out
[docs]@dataclasses.dataclass(frozen=True)
class Lifecycle:
"""How a compartment behaves when it fails. The default suits nearly all.
`restart`: stop() then start() on the same instance is safe and leaves it
as good as new. `snapshot`: the version of what snapshot() returns and
restore() reads; 0 carries nothing across a process restart. See the
module docstring for everything that acts on either.
"""
breaker: Breaker = Breaker()
restart: bool = False
snapshot: int = 0
[docs] def problems(self) -> list[str]:
out = self.breaker.problems()
if self.snapshot < 0:
out.append('snapshot is a version: 0 (none) or more')
return out
[docs] def describe(self) -> 'LifecycleDecl':
return {'breaker': {'failures': self.breaker.failures, 'window': self.breaker.window,
'backoff': list(self.breaker.backoff)},
'restart': self.restart, 'snapshot': self.snapshot}
DEFAULT: Lifecycle = Lifecycle()
[docs]class BreakerDecl(TypedDict):
failures: int
window: int
backoff: list[int]
[docs]class LifecycleDecl(TypedDict):
breaker: BreakerDecl
restart: bool
snapshot: int
[docs]class LifecycleRecord(TypedDict):
"""One compartment's lifecycle as `meshnode lifecycle` and health read it."""
state: str
since: int
failures: int # ticks that raised inside the breaker's window
trips: int # times the breaker has tripped since the last clean run
until: int # quarantined until (0 when not quarantined)
last_error: str
started: Optional[bool] # None: start() not run yet; False: it raised; True: it returned
restarts: int
restored: str # '' nothing offered; 'restored (vN, from <at>)'; or why it was refused
transitions: dict[str, int]
declared: LifecycleDecl
[docs]class Transition(NamedTuple):
compartment: str
was: str
now: str
at: int
why: str
[docs]class Tracker:
"""One compartment's place in the lifecycle. Owned by the Registry.
Not thread-safe on its own: the Registry ticks from one thread and the
control socket's `resume` goes through the same lock the tick holds.
"""
[docs] def __init__(self, name: str, lifecycle: Lifecycle, at: int = 0) -> None:
self.name: str = name
self.lifecycle: Lifecycle = lifecycle
self.state: str = CONFIGURED
self.since: int = at
self.fails: Deque[int] = collections.deque()
self.trips: int = 0
self.until: int = 0
self.last_error: str = ''
# Whether start() has returned. Registry.stop() skips a compartment
# whose start() raised: a stop() for something that never began is how
# a node that never served sent its peers a farewell (audit L1).
self.started: Optional[bool] = None
self.restarts: int = 0
# What happened to the snapshot the last process left for it.
self.restored: str = ''
self.transitions: collections.Counter[str] = collections.Counter()
[docs] def move(self, to: str, at: int, why: str = '') -> Optional[Transition]:
"""Go to `to`. Returns the transition, or None when already there."""
if to == self.state:
return None
t = Transition(self.name, self.state, to, at, why)
self.state, self.since = to, at
self.transitions[to] += 1
return t
[docs] def due(self, now: int) -> bool:
"""Whether the Registry should tick this compartment now."""
if self.state in TICKED:
return True
return self.state == QUARANTINED and now >= self.until
[docs] def succeeded(self, now: int) -> Optional[Transition]:
"""A clean tick. Clears the failure record and recovers if it was down."""
self.fails.clear()
if self.state in (DEGRADED, QUARANTINED):
self.trips, self.until = 0, 0
return self.move(RUNNING, now, 'a clean tick')
return None
[docs] def failed(self, now: int, error: str) -> Optional[Transition]:
"""A tick that raised. Degrades, or trips the breaker."""
b = self.lifecycle.breaker
self.last_error = error[:200]
trial = self.state == QUARANTINED
self.fails.append(now)
while self.fails and self.fails[0] <= now - b.window:
self.fails.popleft()
if b.failures and (trial or len(self.fails) >= b.failures):
self.trips += 1
self.until = now + b.delay(self.trips)
why = ('the trial tick raised: %s' % self.last_error if trial else
'%d ticks raised within %ds: %s' % (len(self.fails), b.window, self.last_error))
if trial:
# Still quarantined; a new transition record keeps the
# operator's count honest about how many trials failed.
self.transitions[QUARANTINED] += 1
return Transition(self.name, QUARANTINED, QUARANTINED, now, why)
return self.move(QUARANTINED, now, why)
return self.move(DEGRADED, now, self.last_error)
[docs] def resume(self, now: int, why: str = 'resumed by the operator') -> Optional[Transition]:
"""End a quarantine now. The next tick is an ordinary one."""
if self.state != QUARANTINED:
return None
self.fails.clear()
self.trips, self.until = 0, 0
return self.move(RUNNING, now, why)
[docs] def start_failed(self, now: int, error: str) -> Optional[Transition]:
"""start() raised. Quarantined when a restart can try again, else FAILED.
From QUARANTINED (a trial restart that raised) this is a failed trial:
the breaker's `failed` already doubles the backoff and says so.
"""
self.started = False
self.last_error = error[:200]
if self.state == QUARANTINED:
return self.failed(now, error)
if not self.lifecycle.restart:
return self.move(FAILED, now, 'start raised: %s' % self.last_error)
self.trips += 1
self.until = now + self.lifecycle.breaker.delay(self.trips)
return self.move(QUARANTINED, now, 'start raised: %s' % self.last_error)
[docs] def restarted(self, now: int, why: str) -> Optional[Transition]:
"""stop() and start() both returned: as good as new, so a clean record."""
self.started = True
self.restarts += 1
self.fails.clear()
self.trips, self.until, self.last_error = 0, 0, ''
return self.move(RUNNING, now, 'restarted: %s' % why)
[docs] def describe(self) -> LifecycleRecord:
return {'state': self.state, 'since': self.since, 'failures': len(self.fails),
'trips': self.trips, 'until': self.until, 'last_error': self.last_error,
'started': self.started, 'restarts': self.restarts, 'restored': self.restored,
'transitions': dict(self.transitions),
'declared': self.lifecycle.describe()}
# ---- carrying state across a process restart ------------------------------------
SNAPSHOT_FILE = 'snapshots.json'
SNAPSHOT_FORMAT = 1
# A snapshot carries a compartment across a restart - an upgrade, a reload -
# which takes seconds. One an hour old is from a node that was down, and the
# state in it describes a world that has moved on (peers learned, links held)
# while nobody was counting. Starting fresh is the honest answer then.
SNAPSHOT_MAX_AGE = 3600
[docs]class SnapshotEntry(TypedDict):
version: int
state: Mapping[str, Any]
[docs]class SnapshotFile(TypedDict):
format: int
at: int
compartments: dict[str, SnapshotEntry]
[docs]def write_snapshots(directory: str, at: int, compartments: Mapping[str, SnapshotEntry]) -> str:
"""Write the snapshot file atomically. Returns its path."""
path = os.path.join(directory, SNAPSHOT_FILE)
body: SnapshotFile = {'format': SNAPSHOT_FORMAT, 'at': at, 'compartments': dict(compartments)}
tmp = path + '.tmp'
with open(tmp, 'w', encoding='utf-8') as f:
json.dump(body, f, indent=1, sort_keys=True)
os.replace(tmp, path)
return path
[docs]def read_snapshots(directory: str, now: int) -> tuple[dict[str, SnapshotEntry], int, str]:
"""(compartments, written at, why none) from the snapshot file.
An empty mapping with a reason when there is a file and it cannot be
used - unreadable, another format, or older than SNAPSHOT_MAX_AGE - and
with '' when there is simply no file. Never raises: a node must start
whatever the last one left behind.
"""
path = os.path.join(directory, SNAPSHOT_FILE)
if not os.path.exists(path):
return {}, 0, ''
try:
with open(path, encoding='utf-8') as f:
body = json.load(f)
except (OSError, ValueError) as e:
return {}, 0, 'the snapshot file is unreadable (%s)' % type(e).__name__
if not isinstance(body, dict) or body.get('format') != SNAPSHOT_FORMAT:
return {}, 0, 'the snapshot file is not format %d' % SNAPSHOT_FORMAT
at = body.get('at')
comps = body.get('compartments')
if not isinstance(at, int) or not isinstance(comps, dict):
return {}, 0, 'the snapshot file is malformed'
if now - at > SNAPSHOT_MAX_AGE:
return {}, at, 'the snapshot is %ds old (more than %ds): starting fresh' % (now - at, SNAPSHOT_MAX_AGE)
out: dict[str, SnapshotEntry] = {}
for name, entry in comps.items():
if isinstance(entry, dict) and isinstance(entry.get('version'), int) and isinstance(entry.get('state'), dict):
out[str(name)] = {'version': entry['version'], 'state': entry['state']}
return out, at, ''
[docs]def discard_snapshots(directory: str) -> None:
"""A snapshot is handed over once: gone after the start that used it."""
try:
os.unlink(os.path.join(directory, SNAPSHOT_FILE))
except FileNotFoundError:
pass
# ---- what the lifecycle says, and counts ---------------------------------------
# Declared here, beside the states, rather than on each compartment: every
# compartment moves through the same machine, so its transitions are the
# Registry's to say. The payload names the compartment, so one channel per
# kind of move is enough and a reader filters by field.
def _payload(t: Transition) -> Mapping[str, Any]:
return {'compartment': t.compartment, 'was': t.was, 'now': t.now, 'why': t.why}
CHANGED = 'lifecycle.changed'
DEGRADED_CH = 'lifecycle.degraded'
QUARANTINED_CH = 'lifecycle.quarantined'
RECOVERED_CH = 'lifecycle.recovered'
RESUMED_CH = 'lifecycle.resumed'
RESTARTED_CH = 'lifecycle.restarted'
FAILED_CH = 'lifecycle.failed'
CHANNELS: tuple[Channel, ...] = (
Channel(name=CHANGED, description='A compartment moved through its lifecycle '
'(configured, wired, running, draining, retired).',
wire=_payload, say='{compartment} {was} -> {now}', tags=('lifecycle',)),
Channel(name=DEGRADED_CH, description='A compartment tick raised; it is still ticked and '
'recovers after one clean tick.',
level=NOTE, wire=_payload, say='{compartment} degraded: {why}', tags=('lifecycle',)),
Channel(name=QUARANTINED_CH, description='A compartment tripped its breaker and is not ticked '
'until its backoff passes.',
level=TROUBLE, wire=_payload, say='{compartment} quarantined: {why}', tags=('lifecycle',)),
Channel(name=RECOVERED_CH, description='A degraded or quarantined compartment ticked cleanly again.',
wire=_payload, say='{compartment} recovered ({was} -> {now})', tags=('lifecycle',)),
Channel(name=RESUMED_CH, description='An operator ended a quarantine early.',
wire=_payload, say='{compartment} resumed: {why}', tags=('lifecycle',)),
Channel(name=RESTARTED_CH, description='A compartment that declares Lifecycle(restart=True) was '
'stopped and started again in place.',
wire=_payload, say='{compartment} {why}', tags=('lifecycle',)),
Channel(name=FAILED_CH, description='An optional compartment\'s start() raised and it cannot be '
'restarted in place; it is not ticked until the node restarts.',
level=TROUBLE, wire=_payload, say='{compartment} failed: {why}', tags=('lifecycle',)),
)
METRICS: tuple[Metric, ...] = (
Metric(name='lifecycle_degraded', description='Times a compartment went from running to degraded. '
'Every raise is in its refusal ring as tick: raised.',
kind=COUNTER, counts=DEGRADED_CH, tags=('lifecycle',)),
Metric(name='lifecycle_quarantined', description='Times a compartment breaker tripped, '
'including failed trial ticks.',
kind=COUNTER, counts=QUARANTINED_CH, tags=('lifecycle',)),
Metric(name='lifecycle_recovered', description='Times a compartment recovered to running.',
kind=COUNTER, counts=RECOVERED_CH, tags=('lifecycle',)),
Metric(name='lifecycle_restarted', description='Times a compartment was restarted in place.',
kind=COUNTER, counts=RESTARTED_CH, tags=('lifecycle',)),
Metric(name='lifecycle_failed', description='Times an optional compartment failed to start.',
kind=COUNTER, counts=FAILED_CH, tags=('lifecycle',)),
)
[docs]def channel_for(t: Transition) -> str:
"""Which declared channel a transition is said on."""
if t.now == QUARANTINED:
return QUARANTINED_CH
if t.now == DEGRADED:
return DEGRADED_CH
if t.now == FAILED:
return FAILED_CH
if t.now == RUNNING and t.why.startswith('restarted'):
return RESTARTED_CH
if t.now == RUNNING and t.was in (DEGRADED, QUARANTINED):
return RESUMED_CH if t.why.startswith('resumed') else RECOVERED_CH
return CHANGED
[docs]def gauge(count: Callable[[], int], name: str, description: str) -> Metric:
"""A sampled gauge over the Registry's trackers, built where they live."""
return Metric(name=name, description=description, sample=count, tags=('lifecycle',))
__all__ = [
'BreakerDecl', 'Breaker', 'CHANGED', 'CHANNELS', 'CONFIGURED', 'DEFAULT', 'DEGRADED', 'DEGRADED_CH',
'DRAINING', 'FAILED', 'FAILED_CH', 'Lifecycle', 'LifecycleDecl', 'LifecycleRecord', 'METRICS',
'QUARANTINED', 'QUARANTINED_CH', 'RECOVERED_CH', 'RESTARTED_CH', 'RESUMED_CH', 'RETIRED', 'RUNNING',
'SNAPSHOT_FILE', 'SNAPSHOT_FORMAT', 'SNAPSHOT_MAX_AGE', 'STATES', 'SnapshotEntry', 'SnapshotFile',
'TICKED', 'Tracker', 'Transition', 'WIRED', 'channel_for', 'discard_snapshots', 'gauge',
'read_snapshots', 'write_snapshots',
]