"""Sidecars: fronting off-the-shelf projects the node does not reimplement.
This compartment does not speak git smart-http, the OCI distribution
protocol or Magpie's media API itself - ngit-grasp, an ncontainer-compatible
gateway and magpie already do, and reimplementing any of them (Option B,
considered and rejected each time) buys nothing a sidecar does not already
give for less code and less to keep current with upstream. What this
compartment owns is the bridge:
* which capabilities this node declares, and the endpoint tags a leaf
reads to find the sidecar - all opt-in, all carried on the kind 11801
descriptor the same way mint/price already are (see
PaidFetch.descriptor_tags).
* whether the sidecar actually speaks its protocol. Every sidecar ships
its own self-tests (a git `info/refs`, an OCI `/v2/`, a magpie files
route): they run when the compartment starts, on the usual cadence after
that, and on demand through the `sidecar_test` verb. A capability is
declared ONLY while the tests pass - a config flag alone declares
nothing - and when the verified set changes the compartment rebuilds the
descriptor there and then, so a kind 11801 never advertises a protocol
nothing behind this node can answer. A failing sidecar is `trouble()`
the way every other mechanism here is.
* nothing about payment: PaidFetch.quote_session (GRASP, coarse per
clone/fetch session) and PaidFetch.quote (ncontainer and magpie, per blob -
a resolved OCI digest or a Magpie file sha256 is priced exactly like any
other Blossom fetch) are the payment surface, called from the reverse
proxy in front of each sidecar, not from here.
What a sidecar IS lives in a manifest (fleetmesh_node/sidecar_manifest.py), not here:
the node ships three in fleetmesh_node/planes/services/sidecars/manifests/ (grasp, ncontainer, magpie);
an operator adds a fourth with `sidecars.<name>.manifest`; a plugin may
declare more. This compartment reads the `sidecars` config block, resolves
each entry to a manifest, runs the manifest's checks and declares what
passes. Adding a sidecar is a JSON file - not a change to this one.
A node with no `sidecars` block behaves exactly as before: no capability
tag, no endpoint tag, nothing to test, nothing in `trouble()`.
"""
from __future__ import annotations
import urllib.parse
from typing import Any, Callable, Mapping, MutableMapping, NamedTuple, Optional
from fleetmesh_node.transport import netfetch
from . import manifest as manifest_mod
from fleetmesh_node import compartment as compartment_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Every, Need
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_core.wire import validate
from fleetmesh_node.config import SidecarEntry, sidecars_of, SidecarsConfig
from .manifest import Check, ManifestError, Sidecar # noqa: F401 - the compartment's own vocabulary
# The origin whose capability identifiers the schema digest governs: only a
# shipped manifest may declare under it. Derived from the schema, never
# written down here (fleetmesh_node/ names no host; see test_node's unplug scan).
RESERVED_ORIGIN = validate.FLEETMESH_ORIGIN.rstrip('/')
# The manifests the node ships, in descriptor order. Operator and plugin
# manifests are appended behind them, in config order.
BUILTINS: tuple[Sidecar, ...] = manifest_mod.builtins()
_BUILTIN_CAPS = {s.name: s.capability for s in BUILTINS}
CAP_GRASP = _BUILTIN_CAPS['grasp']
CAP_NCONTAINER = _BUILTIN_CAPS['ncontainer']
CAP_MAGPIE = _BUILTIN_CAPS['magpie']
SIDECARS = BUILTINS # the historical name; the live set is Sidecars.registry()
_BUILTIN_BY_NAME: dict[str, Sidecar] = {s.name: s for s in BUILTINS}
from .types import SidecarStatusReport
[docs]def resolve_manifest(name: str, entry: Mapping[str, Any],
extra: Mapping[str, Sidecar] = {}) -> Sidecar:
"""The manifest a config entry stands for: inline or `@file` when given,
else a built-in or plugin-declared manifest of that name. A manifest's
`name` must match the entry key it is configured under - the key is the
systemd unit and the Caddy route, the manifest the protocol."""
raw = entry.get('manifest')
if raw is None:
s = _BUILTIN_BY_NAME.get(name) or extra.get(name)
if s is None:
raise ManifestError('sidecars.%s: not a built-in (%s) and no manifest given' % (
name, ', '.join(_BUILTIN_BY_NAME)))
return s
if isinstance(raw, str):
if not raw.startswith('@'):
raise ManifestError('sidecars.%s.manifest must be "@/path/to/manifest.json" or an object' % name)
s = manifest_mod.load_file(raw[1:], origin=RESERVED_ORIGIN)
else:
s = manifest_mod.parse(raw, origin=RESERVED_ORIGIN)
if s.name != name:
raise ManifestError('sidecars.%s: the manifest calls itself %r; the key and the name must agree' % (name, s.name))
return s
# How often a tick re-runs the checks - matching the cadence other
# compartments use for anything that costs a network round trip. The first
# run is at start(), not a cadence later.
_CHECK_EVERY_SECONDS = 300
_PROBE_TIMEOUT = 3.0
[docs]class ProbeResult(NamedTuple):
"""What one check said. `status` is the HTTP status when there was one."""
name: str
ok: bool
status: Optional[int]
why: str
Fetcher = Callable[[str, str], tuple[int, Mapping[str, str]]]
def _http(url: str, method: str, timeout: float = _PROBE_TIMEOUT) -> tuple[int, Mapping[str, str]]:
"""One request to the sidecar: (status, headers). Only http/https are
ever dialled - the URL is operator config, but urlopen would otherwise
honour file:// and ftp:// too - and it goes through netfetch so redirects
stay inside the same policy. allow_private=True because sidecars are
local by design. Raises on anything that is not an HTTP answer."""
parts = urllib.parse.urlsplit(url)
if parts.scheme not in ('http', 'https'):
raise ValueError('not an http(s) URL')
try:
_body, headers = netfetch.fetch(url, max_bytes=4096, timeout=timeout, allow_private=True, method=method)
return 200, headers
except netfetch.FetchHTTPError as e:
return e.code, e.headers or {}
def _reachable(url: str, timeout: float = _PROBE_TIMEOUT) -> bool:
"""HEAD the sidecar. Reachable means it answered HTTP at all: a 4xx is a
live process that disliked HEAD, not an absent one."""
try:
status, _ = _http(url, 'HEAD', timeout)
return status < 500
except Exception:
return False
[docs]def run_check(fetch: Fetcher, endpoint: str, check: Check) -> ProbeResult:
"""One check against one endpoint, never raising."""
url = endpoint.rstrip('/') + check.path
try:
status, headers = fetch(url, check.method)
except Exception as e:
return ProbeResult(check.name, False, None, '%s: %s' % (type(e).__name__, str(e)[:80] or 'no answer'))
if status not in check.expect:
return ProbeResult(check.name, False, status, 'HTTP %d, wanted one of %s' % (status, sorted(check.expect)))
if status == 200 and check.content_type:
ctype = next((v for k, v in headers.items() if k.lower() == 'content-type'), '')
if not str(ctype).lower().startswith(check.content_type):
return ProbeResult(check.name, False, status, 'content-type %r is not %s' % (ctype, check.content_type))
return ProbeResult(check.name, True, status, '')
[docs]class Sidecars(compartment_mod.Compartment):
"""Declares one capability per sidecar whose self-tests pass, and keeps
testing the sidecars behind them. See module docstring for what it does
not do."""
name = 'sidecars'
reads = ('sidecars',) + tuple(s.name for s in BUILTINS) # the old top-level keys, one release more
live = ('sidecars',) + tuple(s.name for s in BUILTINS)
config_cls = SidecarsConfig
STORES = (
st.Store(key='probes', description='recent sidecar probe results',
kind=st.RING, shape=dict, bound=256),
)
CHANNELS = (
Channel('sidecar_verified', 'Sidecar self-tests passed and capability declared', level=INFO, say='sidecar {name} verified and declared', tags=('sidecars', 'probe')),
Channel('sidecar_failed', 'Sidecar self-test failed and capability retracted', level=NOTE, say='sidecar {name} self-test failed: {why}', tags=('sidecars', 'probe')),
)
METRICS = (
Metric(name='sidecars_verified_total', description='Times sidecars verified.', kind=COUNTER, counts='sidecar_verified', tags=('sidecars', 'probe')),
Metric(name='sidecars_failed_total', description='Times sidecar probes failed.', kind=COUNTER, counts='sidecar_failed', tags=('sidecars', 'probe')),
)
NEEDS = (
Need('cfg', 'cfg', getter=True),
Need('descriptor', 'descriptor'),
Need('redescribe', 'redescribe'),
)
EVERY = (Every(seconds=_CHECK_EVERY_SECONDS, run='_check_endpoints'),)
COMMANDS = (
Command(
key='sidecars',
name='Configured Sidecars',
description='each configured sidecar, its endpoint, whether it is verified and its last self-test',
tags=('sidecars', 'status'),
permission='sidecars/sidecars',
run='list_sidecars',
reply='dict',
),
Command(
key='sidecar_test',
name='Sidecar Self-Test',
description='run the self-tests of one sidecar (or all) now and redeclare if the result changed',
tags=('sidecars', 'test'),
parameters=(
Parameter(key='name', name='name', description='test: only this sidecar (default all enabled)', type=str, default=None, required=False),
),
permission='sidecars/sidecar_test',
run='run_sidecar_test',
reply='{name: {verified, checks: [{name, ok, status, why}], at}}',
),
)
[docs] def __init__(self, cfg: Callable[[], Mapping[str, Any]], clock: Optional[Callable[[], int]] = None,
checker: Optional[Callable[[str], bool]] = None, node: Any = None,
fetch: Fetcher = _http,
descriptor: Optional[Callable[[], Optional[Mapping[str, Any]]]] = None,
redescribe: Optional[Callable[[], Any]] = None) -> None:
super().__init__(node, clock)
self._cfg = cfg
# `checker(url) -> bool` is the older, coarser injection: when given it
# stands in for every check of a sidecar. `fetch` is the real seam.
self._checker = checker
self._fetch = fetch
self._descriptor = descriptor or (lambda: None)
self._redescribe = redescribe
self._last_checked: dict[str, int] = {} # not a store: local configured sidecars probe timestamps
self._results: dict[str, list[ProbeResult]] = {} # not a store: local configured sidecars probe results
self._verified: dict[str, bool] = {} # not a store: local configured sidecars verified flags
self._declared: dict[str, Sidecar] = {} # not a store: plugin-declared manifests, by name
self._manifest_errors: dict[str, str] = {} # not a store: manifest errors for configured sidecars
self.redeclared = 0
self._closed = False
# -- config -----------------------------------------------------------
def _entries(self) -> dict[str, SidecarEntry]:
"""The `sidecars` block as it is now (old top-level keys folded in);
every sidecar reads live."""
out = {}
for name, e in sidecars_of(self._cfg()).items():
e = e or {}
out[name] = SidecarEntry(enabled=bool(e.get('enabled', False)), endpoint=e.get('endpoint'),
manifest=e.get('manifest'))
return out
[docs] def block(self, name: str) -> SidecarEntry:
return self._entries().get(name, SidecarEntry())
[docs] def enabled(self, name: str) -> bool:
return self.block(name).on
[docs] def registry(self) -> list[Sidecar]:
"""Every sidecar this node knows of, in descriptor order: the
built-ins, then plugin-declared ones, then any the operator named
only by manifest. A configured entry whose manifest does not resolve
is remembered as a manifest problem (trouble()) and skipped - a
bad manifest declares nothing, same as a failing check."""
self._manifest_errors = {} # not a store: reset on registry evaluation
out: list[Sidecar] = list(BUILTINS)
seen = {s.name for s in out}
for name, s in self._declared.items():
if name not in seen:
out.append(s); seen.add(name)
for name, e in self._entries().items():
if e.manifest is None and name in seen:
continue
try:
s = resolve_manifest(name, {'manifest': e.manifest}, self._declared)
except (ManifestError, OSError) as err:
self._manifest_errors[name] = str(err)
continue
if name in seen: # operator manifest overrides a plugin's, never a built-in
if name in _BUILTIN_BY_NAME:
self._manifest_errors[name] = 'sidecars.%s: %s is a built-in; its manifest is not overridable' % (name, name)
continue
out = [s if x.name == name else x for x in out]
else:
out.append(s); seen.add(name)
return out
[docs] def manifest(self, name: str) -> Sidecar:
for s in self.registry():
if s.name == name:
return s
raise KeyError('no sidecar named %r (have: %s)' % (name, ', '.join(x.name for x in self.registry())))
[docs] def declare(self, sidecar: Sidecar) -> None:
"""A plugin (or test) adds a manifest the node did not ship. Refused
under the reserved origin, like any other third-party manifest."""
manifest_mod.parse(sidecar.to_dict(), origin=RESERVED_ORIGIN)
self._declared[sidecar.name] = sidecar
# Readable names for the individual sidecars; enabled(name) is the
# generic form and what the table-driven paths use.
[docs] def grasp_enabled(self) -> bool:
return self.enabled('grasp')
[docs] def ncontainer_enabled(self) -> bool:
return self.enabled('ncontainer')
[docs] def magpie_enabled(self) -> bool:
return self.enabled('magpie')
def _endpoints(self) -> dict[str, str]:
"""Enabled entries whose manifest resolved. One with a manifest
problem is not an endpoint: it is neither tested nor declared."""
known = {s.name for s in self.registry()} - set(self._manifest_errors)
return {n: str(e.endpoint) for n, e in self._entries().items() if e.on and n in known}
[docs] def verified(self, name: str) -> bool:
"""Enabled now AND its last self-test passed."""
return self.enabled(name) and self._verified.get(name, False)
# -- what the descriptor says ------------------------------------------
[docs] def capability_protocols(self) -> list[str]:
"""Extra `c` protocol identifiers this node's descriptor should carry,
alongside the required ones `descriptor.build` always adds. Only the
sidecars whose self-tests pass: enabling one declares nothing until
it answers."""
return [s.capability for s in self.registry() if self.verified(s.name)]
def _declared_on_descriptor(self) -> set[str]:
desc = self._descriptor()
if not desc:
return set()
caps = {t[1] for t in desc.get('tags', []) if len(t) > 1 and t[0] == 'c'}
return {s.name for s in self.registry() if s.capability in caps}
# -- the self-tests ------------------------------------------------------
[docs] def test(self, name: str, now: Optional[int] = None) -> list[ProbeResult]:
"""Run one sidecar's checks now. Records the verdict; does not redeclare."""
s = self.manifest(name)
at = now if now is not None else self.clock()
url = self._endpoints().get(name)
if url is None:
results = [ProbeResult('enabled', False, None, 'not enabled')]
elif self._checker is not None:
up = self._checker(url)
results = [ProbeResult(c.name, up, 200 if up else None, '' if up else 'no answer') for c in s.checks]
else:
results = [run_check(self._fetch, url, c) for c in s.checks]
self._last_checked[name] = at
self._results[name] = results
self._verified[name] = url is not None and all(r.ok for r in results)
for r in results:
if not r.ok:
self.refusals.refuse('sidecar', 'self-test %s failed' % r.name, subject=name,
detail={'endpoint': url or '', 'status': r.status, 'why': r.why}, at=at)
return results
[docs] def test_all(self, now: Optional[int] = None, only: Optional[str] = None) -> dict[str, list[ProbeResult]]:
names = [only] if only else list(self._endpoints())
return {n: self.test(n, now) for n in names}
[docs] def redeclare_if_changed(self) -> bool:
"""Rebuild the descriptor when what should be declared differs from
what the current one says. True when it was rebuilt."""
want = {s.name for s in self.registry() if self.verified(s.name)}
if want == self._declared_on_descriptor():
return False
if self._redescribe is None:
return False
self._redescribe()
self.redeclared += 1
return True
# -- lifecycle -----------------------------------------------------------
[docs] def start(self) -> None:
"""Test every enabled sidecar before anything is declared, and bring
the descriptor into agreement at once rather than a cadence later."""
if self._endpoints():
self.test_all(self.clock())
self._every_last['_check_endpoints'] = self.clock()
self.redeclare_if_changed()
def _check_endpoints(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
now = now if now is not None else self.clock()
out: list[tuple[Any, ...]] = []
changed = False
for name in self._endpoints():
was = self._verified.get(name)
self.test(name, now)
is_ok = self._verified[name]
if was is not None and was != is_ok:
out.append(('sidecar', name, 'verified' if is_ok else 'failing'))
changed = changed or (was != is_ok)
if changed and self.redeclare_if_changed():
out.append(('sidecar', 'descriptor', 'redeclared'))
return out
_tick_runs_every = True
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
out: list[tuple[Any, ...]] = []
if any(name not in self._last_checked for name in self._endpoints()):
self._every_last['_check_endpoints'] = now
out += self._check_endpoints(now)
else:
out += self.run_every(now)
changed = False
# A sidecar removed from the config stops being verified without a probe.
for name in [n for n in self._verified if n not in self._endpoints()]:
del self._verified[name]
changed = True
if changed and self.redeclare_if_changed():
out.append(('sidecar', 'descriptor', 'redeclared'))
return out
def _report(self, name: str) -> SidecarStatusReport:
return {'verified': self.verified(name), 'last_checked': self._last_checked.get(name),
'checks': [r._asdict() for r in self._results.get(name, [])]}
[docs] def state(self) -> Mapping[str, Any]:
if self._closed:
return {'stopped': True}
out: MutableMapping[str, Any] = {}
entries = self._entries()
for s in self.registry():
if s.name in entries:
out[s.name] = {'enabled': self.enabled(s.name), 'endpoint': self.block(s.name).endpoint,
'capability': s.capability, 'builtin': s.builtin,
'reachable': self._verified.get(s.name), **self._report(s.name)}
else:
out[s.name] = {'enabled': False, 'capability': s.capability, 'builtin': s.builtin}
for name, why in self._manifest_errors.items():
out[name] = {'enabled': entries[name].enabled, 'manifest_error': why}
out['redeclared'] = self.redeclared
return out
[docs] def trouble(self) -> list[str]:
out = []
eps = self._endpoints()
self.registry()
for name, why in self._manifest_errors.items():
out.append('%s sidecar has no usable manifest: %s' % (name, why))
for name, url in eps.items():
if self._verified.get(name) is False:
failed = [r for r in self._results.get(name, []) if not r.ok]
out.append('%s sidecar is configured at %s but not verified: %s' % (
name, url, '; '.join('%s (%s)' % (r.name, r.why) for r in failed) or 'untested'))
return out
[docs] def list_sidecars(self) -> Mapping[str, Any]:
return {n: {'endpoint': url, **self._report(n)} for n, url in self._endpoints().items()}
[docs] def run_sidecar_test(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
only = args.get('name')
if only is not None:
self.manifest(only) # KeyError names the ones there are
if only is not None and not self.enabled(only):
raise KeyError('%s is not enabled' % only)
now = self.clock()
results = self.test_all(now, only)
redeclared = self.redeclare_if_changed()
return {n: {'verified': all(r.ok for r in rs), 'checks': [r._asdict() for r in rs], 'at': now}
for n, rs in results.items()} | {'redeclared': redeclared}
[docs] def stop(self) -> Any:
self._closed = True