"""The Registry: the compartments a node has, and the uniform ways to reach them.
Split out of fleetmesh_node/compartment.py when it became a package.
"""
from __future__ import annotations
import collections
import json
import threading
import time
from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, TypeVar
from . import lifecycle as lifecycle_mod
from . import stores as stores_mod
from ..interfaces import CommandSpec
from ..metrics import COUNTER, Metric
from .base import Compartment, CompartmentProtocol
from .problem import Note, NoteList, Trouble, TroubleList
from .refusals import Refusal, why_detail
T = TypeVar('T', bound=CompartmentProtocol)
[docs]class Registry:
"""The compartments a node has, and the uniform ways to reach all of them.
It also owns where each one is in its lifecycle (fleetmesh_node/lifecycle): every
verb below moves a compartment's Tracker, a tick that raises degrades it,
enough of them trip its breaker, and a quarantined compartment is left
alone until its backoff passes. `say`, when given, is the node log's
`say`, and every transition goes out on a declared lifecycle channel.
"""
[docs] def __init__(self, say: Optional[Callable[..., Any]] = None,
clock: Optional[Callable[[], int]] = None) -> None:
self._by_name: collections.OrderedDict[str, CompartmentProtocol] = collections.OrderedDict()
self._life: dict[str, lifecycle_mod.Tracker] = {}
self.say: Optional[Callable[..., Any]] = say
self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
# The tick and an operator's `resume` arrive on different threads.
self._life_lock: threading.Lock = threading.Lock()
# And a `restart` must never stop a compartment in the middle of its
# tick: held around each compartment's tick and around each restart.
# Re-entrant because a restart from inside a tick (a quarantine trial)
# already holds it.
self._busy: threading.RLock = threading.RLock()
# name -> error, for each compartment whose last report() raised.
self._report_failed: dict[str, str] = {}
[docs] def add(self, sub: T) -> T:
if not sub.name:
raise ValueError('a compartment must have a name')
if sub.name in self._by_name:
raise ValueError('two compartments called %r' % sub.name)
self._by_name[sub.name] = sub
decl = getattr(sub, 'LIFECYCLE', None)
if not isinstance(decl, lifecycle_mod.Lifecycle):
decl = lifecycle_mod.DEFAULT
problems = decl.problems()
if decl.snapshot and not (callable(getattr(sub, 'snapshot', None)) and callable(getattr(sub, 'restore', None))):
# The declaration is the promise; the two methods are the code that
# keeps it. One without the other would read as continuity the
# node does not have.
problems.append('Lifecycle(snapshot=%d) needs snapshot() and restore(state)' % decl.snapshot)
# The base bound each declared store's container as `self.<key>`. One
# that is not there any more was replaced in __init__ - by a plain
# list or dict, which is the unbounded collection the declaration
# promised was not there.
holdings = getattr(sub, 'holdings', None)
declared_stores: tuple[object, ...] = tuple(getattr(sub, 'STORES', None) or ())
if holdings is None and declared_stores:
# Declared but never built: an __init__ that did not call
# Compartment.__init__. The compartment would run without one of
# its containers and fail at its first write - or at shutdown,
# when snapshot() asks for holdings that are not there.
problems.append('it declares STORES but has no holdings - its __init__ never called '
'Compartment.__init__, which builds them')
for key, container in (holdings.items() if holdings is not None else ()):
if getattr(sub, key, None) is not container:
problems.append('store %s was replaced after the base built it (self.%s = ...); '
'write into the container instead' % (key, key))
if problems:
del self._by_name[sub.name]
raise ValueError('%s declares what it cannot keep: %s' % (sub.name, '; '.join(problems)))
self._life[sub.name] = lifecycle_mod.Tracker(sub.name, decl, at=self.clock())
return sub
# ---- lifecycle ----------------------------------------------------------
def _moved(self, t: Optional[lifecycle_mod.Transition]) -> None:
"""Say a transition, if there was one and there is a log to say it to."""
if t is None or self.say is None:
return
try:
self.say(lifecycle_mod.channel_for(t), t)
except Exception:
pass # a log must never fail the lifecycle it records
def _move(self, name: Optional[str], to: str, at: Optional[int] = None, why: str = '') -> None:
tr = self._life.get(name or '')
if tr is None:
return
with self._life_lock:
t = tr.move(to, self.clock() if at is None else at, why)
self._moved(t)
[docs] def state_of(self, name: str) -> Optional[str]:
tr = self._life.get(name)
return tr.state if tr is not None else None
[docs] def lifecycle(self) -> dict[str, lifecycle_mod.LifecycleRecord]:
"""Where every compartment is, for health and `meshnode lifecycle`."""
return {n: tr.describe() for n, tr in self._life.items()}
[docs] def resume(self, name: str, why: str = 'resumed by the operator') -> str:
"""End a compartment's quarantine now. Returns the state it is left in.
Raises KeyError for a compartment this node does not have. Resuming
one that is not quarantined is not an error - it is already ticking -
and the state it is in is returned unchanged.
"""
tr = self._life.get(name)
if tr is None:
raise KeyError('no compartment called %r; compartments: %s' % (name, ', '.join(self._life)))
s = self._by_name.get(name)
if s is not None and tr.state == lifecycle_mod.QUARANTINED and tr.started is False:
# Quarantined because its start() raised: it never began, so
# ending the quarantine means starting it, not ticking it.
with self._busy:
self._restart(s, tr, 'resumed: %s' % why)
return tr.state
with self._life_lock:
t = tr.resume(self.clock(), why)
self._moved(t)
return tr.state
[docs] def quarantined(self) -> list[str]:
return [n for n, tr in self._life.items() if tr.state == lifecycle_mod.QUARANTINED]
[docs] def failed(self) -> list[str]:
return [n for n, tr in self._life.items() if tr.state == lifecycle_mod.FAILED]
[docs] def restartable(self, name: str) -> bool:
tr = self._life.get(name)
return tr is not None and tr.lifecycle.restart
[docs] def restart(self, name: str, why: str = 'by the operator') -> str:
"""Stop and start one compartment in place. Returns the state it is left in.
Only for a compartment that declares `Lifecycle(restart=True)`; any
other is refused with ValueError, because a stop() and start() nobody
designed to run twice could leave threads or sockets behind and call
it a recovery. KeyError for a name this node does not have.
The same instance is stopped and started: every other compartment
that reached it in `wire()` still holds it, and a fresh instance would
leave them holding the old one. A start() that raises quarantines it
with backoff; the trial that follows restarts it again.
"""
tr = self._life.get(name)
s = self._by_name.get(name)
if tr is None or s is None:
raise KeyError('no compartment called %r; compartments: %s' % (name, ', '.join(self._life)))
if not tr.lifecycle.restart:
raise ValueError('%s does not declare Lifecycle(restart=True): its stop() and start() are not '
'known to be safe to run twice on one instance, so only a node restart '
'restarts it' % name)
if tr.state in (lifecycle_mod.DRAINING, lifecycle_mod.RETIRED):
raise ValueError('%s is %s; the node is stopping' % (name, tr.state))
with self._busy:
self._restart(s, tr, why)
return tr.state
def _restart(self, s: CompartmentProtocol, tr: lifecycle_mod.Tracker, why: str,
trial: bool = False, now: Optional[int] = None) -> bool:
"""stop() then start(). True when start() returned. Holds `_busy`.
A trial restart (from quarantine) moves nothing on success: the trial
tick that follows decides whether it recovered, and says so.
"""
now = self.clock() if now is None else now
if tr.started and not trial:
self._move(s.name, lifecycle_mod.DRAINING, now, 'restart: %s' % why)
if tr.started:
try:
s.stop()
except Exception as e:
# Said and gone past: a stop() that raised has still stopped
# as far as it is going to, and start() is the recovery.
s.refusals.refuse('restart', 'stop raised', detail=why_detail(e))
finally:
if hasattr(s, 'stop_workers'):
try:
s.stop_workers()
except Exception:
pass
tr.started = False
try:
if hasattr(s, 'start_workers'):
s.start_workers()
s.start()
except Exception as e:
s.refusals.refuse('restart', 'start raised', detail=why_detail(e))
with self._life_lock:
t = tr.start_failed(now, '%s: %s' % (type(e).__name__, e))
self._moved(t)
return False
if trial:
tr.started = True
tr.restarts += 1
return True
with self._life_lock:
t = tr.restarted(now, why)
self._moved(t)
return True
# ---- carrying state across a process restart ----------------------------
[docs] def snapshot(self) -> dict[str, lifecycle_mod.SnapshotEntry]:
"""What every started compartment that declares `snapshot` hands on.
One whose snapshot() raises, or returns something that is not a
JSON object, is a refusal against itself and is left out: the next
process starts it fresh, and every other compartment's state still
goes across.
"""
out: dict[str, lifecycle_mod.SnapshotEntry] = {}
for s in self:
tr = self._life.get(s.name or '')
if tr is None or not tr.lifecycle.snapshot or tr.started is not True:
continue
try:
state = getattr(s, 'snapshot')()
if not isinstance(state, Mapping):
raise TypeError('snapshot() returned %s, not a mapping' % type(state).__name__)
state = json.loads(json.dumps(dict(state)))
except Exception as e:
s.refusals.refuse('snapshot', 'raised', detail=why_detail(e))
continue
out[s.name or ''] = {'version': tr.lifecycle.snapshot, 'state': state}
return out
[docs] def restore(self, saved: Mapping[str, lifecycle_mod.SnapshotEntry], at: int = 0) -> dict[str, str]:
"""Hand each compartment the state the last process left for it.
Before start(), so a compartment starts from where it was.
Returns {name: what happened} for every compartment that declares
`snapshot`, and records the same in its lifecycle. Refused - and so
started fresh - when there is no entry, the entry's version is not
the one the compartment reads now, or restore() raises. Never raises
itself: continuity is worth having, not worth refusing to start for.
"""
out: dict[str, str] = {}
for s in self:
tr = self._life.get(s.name or '')
if tr is None or not tr.lifecycle.snapshot:
continue
e = saved.get(s.name or '')
if e is None:
tr.restored = ''
continue
if e['version'] != tr.lifecycle.snapshot:
tr.restored = ('refused: the snapshot is v%d and this %s reads v%d; started fresh'
% (e['version'], s.name, tr.lifecycle.snapshot))
else:
try:
getattr(s, 'restore')(e['state'])
tr.restored = 'restored (v%d, written at %d)' % (e['version'], at)
except Exception as exc:
s.refusals.refuse('restore', 'raised', detail=why_detail(exc))
tr.restored = 'refused: restore raised %s: %s; started fresh' % (type(exc).__name__, exc)
out[s.name or ''] = tr.restored
return out
[docs] def lifecycle_metrics(self) -> tuple[Any, ...]:
"""The lifecycle's counters, and gauges sampled from these trackers."""
return lifecycle_mod.METRICS + (
lifecycle_mod.gauge(lambda: len(self.quarantined()), 'compartments_quarantined',
'Compartments whose breaker is open now.'),
lifecycle_mod.gauge(
lambda: sum(1 for tr in self._life.values() if tr.state == lifecycle_mod.DEGRADED),
'compartments_degraded', 'Compartments whose last tick raised.'),
lifecycle_mod.gauge(lambda: len(self.failed()), 'compartments_failed',
'Optional compartments whose start() raised and that cannot restart.'),
)
# ---- what the compartments keep -------------------------------------------
def _containers(self) -> list[tuple[str, str, stores_mod.Container]]:
out: list[tuple[str, str, stores_mod.Container]] = []
for s in self:
holdings = getattr(s, 'holdings', None)
for key, c in (holdings.items() if holdings is not None else ()):
out.append((s.name or '', key, c))
return out
[docs] def store_metrics(self) -> tuple[Any, ...]:
"""Every declared store, metered without its compartment writing a
metric: size, bound and high-water as gauges, and the writes, the
evictions and the refusals as counters. One series per (compartment,
store), sampled from the containers when /metrics is read."""
def per(read: Callable[[stores_mod.Container], int]) -> Callable[[], dict[tuple[str, str], int]]:
return lambda: {(n, k): read(c) for n, k, c in self._containers()}
labels = ('compartment', 'store')
return (
Metric(name='store_entries', description='Entries a compartment store holds now.',
labels=labels, sample=per(len), tags=('stores',)),
Metric(name='store_bound', description='The most entries a store may hold, as declared.',
labels=labels, sample=per(lambda c: c.store.limit), tags=('stores',)),
Metric(name='store_high_water', description='The most entries a store has held since start.',
labels=labels, sample=per(lambda c: c.high_water), tags=('stores',)),
Metric(name='store_writes', description='Entries written to a store (and counters incremented).',
kind=COUNTER, labels=labels, sample=per(lambda c: c.inserts), tags=('stores',)),
Metric(name='store_evictions', description='Entries a store dropped at its bound (OLDEST or LRU).',
kind=COUNTER, labels=labels, sample=per(lambda c: c.evictions), tags=('stores',)),
Metric(name='store_refused', description='Writes a full REFUSE store turned away.',
kind=COUNTER, labels=labels, sample=per(lambda c: c.refused), tags=('stores',)),
)
[docs] def store_trouble(self) -> list[str]:
"""A REFUSE store is trouble once it is nearly full, and while it has
refused anything: the state it holds is the kind that must not be
lost, so the write that failed was a decision the compartment had to
make without it. A ring at its bound is a log doing its job."""
out: list[str] = []
for n, k, c in self._containers():
st = c.store
if st.kind == stores_mod.COUNTERS or st.evict != stores_mod.REFUSE:
continue
if c.refused or c.nearly_full():
out.append('%s: store %s holds %d of %d %s%s' % (
n, k, len(c), st.limit, st.keyed_by or 'entries',
'; %d write(s) refused' % c.refused if c.refused else ' and refuses new ones when full'))
return out
[docs] def store_notes(self) -> list[str]:
"""A TABLE or SET that has evicted is dropping state it was keeping
on purpose. Not a fault - the bound is the decision - but worth
reading when something it tracked is unexpectedly gone."""
out: list[str] = []
for n, k, c in self._containers():
if c.store.kind in (stores_mod.TABLE, stores_mod.SET) and c.evictions:
out.append('%s: store %s has dropped %d %s at its bound of %d (%s first)' % (
n, k, c.evictions, c.store.keyed_by or 'entries', c.store.limit, c.store.evict))
return out
def __iter__(self) -> Iterator[CompartmentProtocol]:
return iter(self._by_name.values())
def __contains__(self, name: object) -> bool:
return name in self._by_name
[docs] def get(self, name: str) -> Optional[CompartmentProtocol]:
return self._by_name.get(name)
[docs] def names(self) -> list[str]:
return list(self._by_name)
# ---- assembly -----------------------------------------------------------
[docs] def assemble(self, ctx: Any, manifest: Sequence[type[Any]],
enabled: Optional[Callable[[str], bool]] = None) -> list[str]:
"""Build every compartment in the manifest from one context, in the
manifest's order, and register each as it is built.
The manifest is the node's declared tick order. It is *validated*
against each class's `requires` rather than re-sorted from it, because
tick order is a reviewed decision with reasons the dependency graph
does not carry (the relay journal ticks first so a dropped
subscription is said before anything that depends on it; the
operator's decisions tick before the mechanisms they outrank). A
manifest that lists a compartment before one it requires, or omits
one it requires, is a programming error and is refused at startup
with the two names in the message - not discovered three ticks in.
Optional compartments are built only when `enabled(name)` says so;
the default asks whether `enabled_by` is a present config key. A
compartment whose `api` major differs from this registry's, or whose
`protocol` is set and differs from the digest in `ctx`, is refused
the same way: the node would rather not start than start wrong.
Returns the names built, in order. The context's `node` (when it is
a NodeApp) gets each compartment set on it under its name, which is
what the rest of the node and the tests reach for today.
"""
is_on: Callable[[str], bool] = enabled or (lambda key: bool(ctx.cfg.get(key)))
digest = getattr(ctx, 'digest', None)
setattr(ctx, '_registry', self)
built: list[str] = []
seen: list[str] = [] # every manifest name passed, built or not
for cls in manifest:
name = getattr(cls, 'name', None)
if not name:
raise ValueError('%s declares no name' % cls.__name__)
for dep in getattr(cls, 'uses', ()):
if dep not in seen:
raise ValueError('%s uses %s, which the manifest does not list before it (manifest order: %s)'
% (name, dep, ' '.join(seen) or '(nothing yet)'))
seen.append(name)
api = getattr(cls, 'api', Compartment.API)
if api.split('/')[0] != Compartment.API.split('/')[0] or \
api.split('/')[1].split('.')[0] != Compartment.API.split('/')[1].split('.')[0]:
raise ValueError('%s is written against %s; this node speaks %s' % (name, api, Compartment.API))
proto = getattr(cls, 'protocol', '')
if proto and digest and proto != digest:
raise ValueError('%s was built against schema %s; this node speaks %s' % (name, proto[:12], digest[:12]))
if getattr(cls, 'optional', False):
key = getattr(cls, 'enabled_by', None) or name
if not is_on(key):
continue
for dep in getattr(cls, 'requires', ()):
if dep not in self._by_name:
raise ValueError('%s requires %s, which is not built before it (manifest order: %s)'
% (name, dep, ' '.join(built) or '(nothing yet)'))
sub = cls.from_context(ctx)
self.add(sub)
refs = self.store_ref_problems(sub)
if refs:
del self._by_name[name]
self._life.pop(name, None)
raise ValueError('%s reads what is not there: %s' % (name, '; '.join(refs)))
built.append(name)
if ctx.node is not None:
setattr(ctx.node, getattr(cls, 'attr', None) or name, sub)
return built
[docs] def store_ref_problems(self, sub: Any) -> list[str]:
"""Every `other.key` a command of `sub` reads, where `other` is built
and declares no store `key`.
The class lint (Compartment._command_data_problems) holds the ref to
`requires`/`uses`; only here do both sides exist, so only here can
the store itself be checked. The manifest puts every required and
used compartment before `sub`, so all that will be built already is.
One that is not built is an absent `uses` - optional by definition,
and the command must cope without it - and is skipped.
"""
out: list[str] = []
refs = getattr(type(sub), 'store_refs', None)
for cmd, other, key in (refs() if callable(refs) else ()):
target = self._by_name.get(other)
if target is None:
continue
declared = {getattr(s, 'key', None) for s in getattr(target, 'STORES', ())}
if key not in declared:
out.append('command %s reads %s.%s, and %s declares no store %r (it declares: %s)'
% (cmd, other, key, other, key, ', '.join(sorted(str(k) for k in declared)) or 'none'))
return out
# ---- the uniform verbs --------------------------------------------------
[docs] def wire(self, ctx: Any) -> None:
"""Every compartment's back-edges, in manifest order, once all are built."""
for s in self:
for need in getattr(type(s), 'NEEDS', ()) or ():
if getattr(need, 'late', False):
val = need.resolve_late(ctx, s, registry=self)
setattr(s, need.key, val)
s.wire(self, ctx)
self._move(s.name, lifecycle_mod.WIRED)
[docs] def route(self, router: Any) -> list[tuple[str, str]]:
"""Register every compartment's declared protocols on the router.
Returns [(compartment, piuri)] as registered. A compartment that
declares a protocol and has no `handle` is a programming error and
is refused here, at startup, with its name.
"""
out: list[tuple[str, str]] = []
for s in self:
piuris = tuple(getattr(s, 'piuris', ()) or ())
exact = tuple(getattr(s, 'exact_types', ()) or ())
if not piuris and not exact:
continue
handle = getattr(s, 'handle', None)
is_base = getattr(handle, '__func__', handle) is Compartment.handle
if not callable(handle) or (is_base and not getattr(s, 'HANDLERS', ())):
raise ValueError('%s declares protocols %s but has no handle(inbound)'
% (s.name, ', '.join(piuris + exact)))
for p in piuris:
router.register(p, handle)
out.append((s.name or '', p))
for t in exact:
router.register_exact(t, handle)
out.append((s.name or '', t))
return out
[docs] def start(self) -> None:
"""Start every compartment, in manifest order, each isolated.
One that raises is a refusal against itself, and what happens next
depends on whether the node can run without it (audit L1):
required (not `optional`) every compartment already started is
stopped, newest first, and the raise goes
on and stops the node. A node that cannot
begin is a configuration the operator
must hear about at boot - and it must not
have half-begun, with a farewell sent by
something that never served.
optional the rest still start. It is QUARANTINED
when it declares restart (its trial
restarts it), FAILED when it does not.
"""
begun: list[CompartmentProtocol] = []
for s in self:
tr = self._life.get(s.name or '')
try:
if hasattr(s, 'start_workers'):
s.start_workers()
s.start()
except Exception as e:
s.refusals.refuse('start', 'raised', detail=why_detail(e))
err = '%s: %s' % (type(e).__name__, e)
if not getattr(s, 'optional', False):
for o in reversed(begun):
self._move(o.name, lifecycle_mod.DRAINING, why='%s could not start' % s.name)
try:
o.stop()
except Exception as e2:
o.refusals.refuse('stop', 'raised', detail=why_detail(e2))
self._move(o.name, lifecycle_mod.RETIRED)
if tr is not None:
tr.started = False
self._move(s.name, lifecycle_mod.FAILED, why='start raised: %s' % err)
raise
if tr is not None:
with self._life_lock:
t = tr.start_failed(self.clock(), err)
self._moved(t)
continue
if tr is not None:
tr.started = True
begun.append(s)
self._move(s.name, lifecycle_mod.RUNNING)
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
"""Every compartment's periodic work. One that raises does not stop the
others, and its failure is a refusal against itself rather than a
traceback nobody reads.
A raise also moves the compartment: to DEGRADED, or to QUARANTINED once
its breaker trips. A quarantined compartment is skipped until its
backoff passes, then given one trial tick - restarted first when it
declares restart, because "turn it off and on again" is the recovery a
stuck connection or a wedged worker needs. A retired or failed one is
skipped.
"""
did: list[tuple[Any, ...]] = []
for s in self:
tr = self._life.get(s.name or '')
if tr is not None and not tr.due(now):
continue
with self._busy:
if tr is not None and tr.state == lifecycle_mod.QUARANTINED and tr.lifecycle.restart:
if not self._restart(s, tr, 'trial after quarantine', trial=True, now=now):
if tr.state == lifecycle_mod.QUARANTINED:
s.refusals.refuse('tick', 'quarantined', detail={'until': tr.until, 'trips': tr.trips})
did.append((s.name, 'restart failed', tr.last_error))
continue
try:
did += list(s.run_tick(now) if hasattr(s, 'run_tick') else (s.tick(now) or []))
except Exception as e:
s.refusals.refuse('tick', 'raised', detail=why_detail(e))
did.append((s.name, 'tick failed', '%s: %s' % (type(e).__name__, e)))
if tr is not None:
with self._life_lock:
t = tr.failed(now, '%s: %s' % (type(e).__name__, e))
self._moved(t)
if tr.state == lifecycle_mod.QUARANTINED:
s.refusals.refuse('tick', 'quarantined', detail={'until': tr.until, 'trips': tr.trips})
continue
if tr is not None:
with self._life_lock:
t = tr.succeeded(now)
self._moved(t)
return did
[docs] def stop(self) -> list[tuple[Optional[str], Any]]:
"""Stop every compartment, and hand back what each had to say.
Returns [(name, whatever stop returned)]. One that raises does not stop
the others: a node that cannot say goodbye must still stop. One whose
start() raised is not stopped - it never began - and returns None.
"""
out: list[tuple[Optional[str], Any]] = []
for s in self:
tr = self._life.get(s.name or '')
if tr is not None and tr.started is False:
out.append((s.name, None))
self._move(s.name, lifecycle_mod.RETIRED, why='never started')
continue
self._move(s.name, lifecycle_mod.DRAINING)
try:
out.append((s.name, s.stop()))
except Exception as e:
s.refusals.refuse('stop', 'raised', detail=why_detail(e))
out.append((s.name, None))
finally:
if hasattr(s, 'stop_workers'):
try:
s.stop_workers()
except Exception:
pass
self._move(s.name, lifecycle_mod.RETIRED)
return out
[docs] def report(self) -> dict[str, Mapping[str, Any]]:
"""Every compartment's report, each isolated as trouble() and notes()
already were.
This is what health(), status() and status.json are made of, and the
tick writes status.json. One compartment whose report() raised used to
raise here, so the node did its work and then failed the tick, every
tick, and `meshnode health` failed with it (audit 2026-10-15, B2). A
report that raises is now that compartment's entry, an `error`, and a
`report_error` trouble until a later report() succeeds - the others
still report.
"""
out: dict[str, Mapping[str, Any]] = {}
failed: dict[str, str] = {}
for s in self:
try:
out[s.name or ''] = s.report()
except Exception as e:
failed[s.name or ''] = '%s: %s' % (type(e).__name__, e)
out[s.name or ''] = {'error': failed[s.name or '']}
self._report_failed = failed
return out
[docs] def trouble(self) -> list[str]:
out = TroubleList()
for s in self:
try:
for t in (s.trouble() or []):
if isinstance(t, Trouble):
comp = t.compartment or s.name or ''
msg = '%s: %s' % (comp, t.message)
out.append(Trouble(
msg,
code=t.code,
compartment=comp,
severity=t.severity,
since=t.since,
hint=t.hint,
**t.metadata,
))
else:
out.append(Trouble(
'%s: %s' % (s.name, t),
code='fault',
compartment=s.name or '',
severity='trouble',
))
except Exception as e:
out.append(Trouble(
'%s: could not report trouble (%s)' % (s.name, e),
code='report_error',
compartment=s.name or '',
severity='fatal',
))
# A quarantine is trouble whatever the compartment itself reports:
# while its breaker is open it is not ticking, and it may not be in a
# position to say so.
for n in self.quarantined():
tr = self._life[n]
msg = ('%s: quarantined - not ticking until %s (trip %d; last: %s); '
'the control command `resume` with name=%s ends it now'
% (n, time.strftime('%H:%M:%SZ', time.gmtime(tr.until)), tr.trips,
tr.last_error or 'unknown', n))
out.append(Trouble(msg, code='quarantined', compartment=n, severity='fault'))
for n in self.failed():
tr = self._life[n]
msg = ('%s: failed to start (%s) and does not declare restart; it is not running '
'until the node restarts' % (n, tr.last_error or 'unknown'))
out.append(Trouble(msg, code='failed_start', compartment=n, severity='fatal'))
# As of the last report(): health() takes the report first, and a
# second report() here would double the cost of every health read.
for n, err in self._report_failed.items():
out.append(Trouble('%s: could not report its state (%s)' % (n, err),
code='report_error', compartment=n, severity='fatal'))
for st in self.store_trouble():
out.append(Trouble(st, code='store_trouble', severity='trouble'))
return out
[docs] def trouble_items(self) -> list[Trouble]:
"""All active troubles as structured Trouble instances."""
res = self.trouble()
return [t if isinstance(t, Trouble) else Trouble(str(t)) for t in res]
[docs] def notes(self) -> NoteList:
out = NoteList()
for s in self:
try:
for t in (s.notes() or []):
if isinstance(t, Note):
out.append(Note(
'%s: %s' % (s.name, t),
topic=t.topic,
compartment=s.name or '',
level=t.level,
since=t.since,
**t.metadata,
))
else:
out.append(Note('%s: %s' % (s.name, t), compartment=s.name or ''))
except Exception as e:
out.append(Note('%s: could not report notes (%s)' % (s.name, e), compartment=s.name or '', level='notice'))
for n, tr in self._life.items():
if tr.state == lifecycle_mod.DEGRADED:
out.append(Note(
'%s: degraded - the last tick raised (%s); it recovers after one clean tick'
% (n, tr.last_error or 'unknown'),
compartment=n,
topic='degraded',
level='notice',
))
for sn in self.store_notes():
out.append(Note(sn, topic='store', level='info'))
return out
[docs] def note_items(self) -> list[Note]:
"""All active notes as structured Note instances."""
res = self.notes()
return [n if isinstance(n, Note) else Note(str(n)) for n in res]
[docs] def refusals(self, n: int = 5) -> list[tuple[Optional[str], Refusal]]:
"""Every no this node said, across every compartment, newest last."""
out: list[tuple[Optional[str], Refusal]] = []
for s in self:
for r in s.refusals.last(n):
out.append((s.name, r))
return sorted(out, key=lambda p: p[1].at)
[docs] def live_keys(self) -> tuple[str, ...]:
keys: list[str] = []
for s in self:
keys += [k for k in s.live if k not in keys]
return tuple(keys)
[docs] def commands(self) -> collections.OrderedDict[str, CommandSpec]:
"""name -> spec, prefixed by subsystem, for the control socket."""
out: collections.OrderedDict[str, CommandSpec] = collections.OrderedDict()
for s in self:
for cmd, spec in (s.commands or {}).items():
named: CommandSpec = {**spec, 'subsystem': s.name or ''}
out['%s.%s' % (s.name, cmd)] = named
return out
[docs] def declared_commands(self) -> tuple[Any, ...]:
"""Every Command declared across compartments."""
out: list[Any] = []
for s in self:
out.extend(getattr(s, 'COMMANDS', ()))
return tuple(out)
[docs] def dispatch(self, name: str, args: Optional[Mapping[str, Any]]) -> Any:
# All three refusals below are the caller naming something that does
# not exist - a typo, not a fault in this node. They were plain
# KeyErrors, which the control socket's boundary treats as a bug: it
# prefixes the type name and appends to its 200-entry ring, and
# health reports the last five of that ring. So five `meshctl` typos
# put five errors in the node's own health and pushed any real bug
# out of the window (audit 2026-10-07, I14). ControlError is the
# declared way to answer the caller and leave no trace. The import is
# function-local to keep compartment/ off app/ at module scope, which
# is how the compartments that already raise it do it.
from ..app import socket as control_mod
sub, _, cmd = name.partition('.')
s = self.get(sub)
if s is None:
raise control_mod.ControlError("unknown compartment %r in command %r" % (sub, name))
if not cmd:
raise control_mod.ControlError("missing command in %r" % (name,))
if s.commands and cmd not in s.commands:
raise control_mod.ControlError(
"unknown command %r on compartment %r; available: %s" % (cmd, sub, list(s.commands)))
try:
return s.command(cmd, args or {})
except KeyError as e:
# `Compartment.command` raises KeyError(name) for a name it does
# not declare - the same caller mistake as the three above, one
# level down, reached when a compartment publishes no `commands`
# mapping for the check above to screen against. Only that exact
# shape is converted: a KeyError raised from inside a command's
# own body is a real bug and must keep reaching the error ring.
if e.args == (cmd,):
raise control_mod.ControlError(
"unknown command %r on compartment %r" % (cmd, sub)) from None
raise