Source code for fleetmesh_node.planes.services.sidecars.compartment

"""Sidecars: fronting off-the-shelf projects the node does not reimplement.

This compartment does not speak git smart-http, the OCI distribution
protocol or Magpie's media API itself - ngit-grasp, an ncontainer-compatible
gateway and magpie already do, and reimplementing any of them (Option B,
considered and rejected each time) buys nothing a sidecar does not already
give for less code and less to keep current with upstream. What this
compartment owns is the bridge:

  * which capabilities this node declares, and the endpoint tags a leaf
    reads to find the sidecar - all opt-in, all carried on the kind 11801
    descriptor the same way mint/price already are (see
    PaidFetch.descriptor_tags).
  * whether the sidecar actually speaks its protocol. Every sidecar ships
    its own self-tests (a git `info/refs`, an OCI `/v2/`, a magpie files
    route): they run when the compartment starts, on the usual cadence after
    that, and on demand through the `sidecar_test` verb. A capability is
    declared ONLY while the tests pass - a config flag alone declares
    nothing - and when the verified set changes the compartment rebuilds the
    descriptor there and then, so a kind 11801 never advertises a protocol
    nothing behind this node can answer. A failing sidecar is `trouble()`
    the way every other mechanism here is.
  * nothing about payment: PaidFetch.quote_session (GRASP, coarse per
    clone/fetch session) and PaidFetch.quote (ncontainer and magpie, per blob -
    a resolved OCI digest or a Magpie file sha256 is priced exactly like any
    other Blossom fetch) are the payment surface, called from the reverse
    proxy in front of each sidecar, not from here.

What a sidecar IS lives in a manifest (fleetmesh_node/sidecar_manifest.py), not here:
the node ships three in fleetmesh_node/planes/services/sidecars/manifests/ (grasp, ncontainer, magpie);
an operator adds a fourth with `sidecars.<name>.manifest`; a plugin may
declare more. This compartment reads the `sidecars` config block, resolves
each entry to a manifest, runs the manifest's checks and declares what
passes. Adding a sidecar is a JSON file - not a change to this one.

A node with no `sidecars` block behaves exactly as before: no capability
tag, no endpoint tag, nothing to test, nothing in `trouble()`.
"""
from __future__ import annotations

import urllib.parse
from typing import Any, Callable, Mapping, MutableMapping, NamedTuple, Optional

from fleetmesh_node.transport import netfetch
from . import manifest as manifest_mod
from fleetmesh_node import compartment as compartment_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, INFO, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Every, Need
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_core.wire import validate
from fleetmesh_node.config import SidecarEntry, sidecars_of, SidecarsConfig
from .manifest import Check, ManifestError, Sidecar  # noqa: F401 - the compartment's own vocabulary

# The origin whose capability identifiers the schema digest governs: only a
# shipped manifest may declare under it. Derived from the schema, never
# written down here (fleetmesh_node/ names no host; see test_node's unplug scan).
RESERVED_ORIGIN = validate.FLEETMESH_ORIGIN.rstrip('/')

# The manifests the node ships, in descriptor order. Operator and plugin
# manifests are appended behind them, in config order.
BUILTINS: tuple[Sidecar, ...] = manifest_mod.builtins()
_BUILTIN_CAPS = {s.name: s.capability for s in BUILTINS}
CAP_GRASP = _BUILTIN_CAPS['grasp']
CAP_NCONTAINER = _BUILTIN_CAPS['ncontainer']
CAP_MAGPIE = _BUILTIN_CAPS['magpie']
SIDECARS = BUILTINS   # the historical name; the live set is Sidecars.registry()
_BUILTIN_BY_NAME: dict[str, Sidecar] = {s.name: s for s in BUILTINS}


from .types import SidecarStatusReport


[docs]def resolve_manifest(name: str, entry: Mapping[str, Any], extra: Mapping[str, Sidecar] = {}) -> Sidecar: """The manifest a config entry stands for: inline or `@file` when given, else a built-in or plugin-declared manifest of that name. A manifest's `name` must match the entry key it is configured under - the key is the systemd unit and the Caddy route, the manifest the protocol.""" raw = entry.get('manifest') if raw is None: s = _BUILTIN_BY_NAME.get(name) or extra.get(name) if s is None: raise ManifestError('sidecars.%s: not a built-in (%s) and no manifest given' % ( name, ', '.join(_BUILTIN_BY_NAME))) return s if isinstance(raw, str): if not raw.startswith('@'): raise ManifestError('sidecars.%s.manifest must be "@/path/to/manifest.json" or an object' % name) s = manifest_mod.load_file(raw[1:], origin=RESERVED_ORIGIN) else: s = manifest_mod.parse(raw, origin=RESERVED_ORIGIN) if s.name != name: raise ManifestError('sidecars.%s: the manifest calls itself %r; the key and the name must agree' % (name, s.name)) return s
# How often a tick re-runs the checks - matching the cadence other # compartments use for anything that costs a network round trip. The first # run is at start(), not a cadence later. _CHECK_EVERY_SECONDS = 300 _PROBE_TIMEOUT = 3.0
[docs]class ProbeResult(NamedTuple): """What one check said. `status` is the HTTP status when there was one.""" name: str ok: bool status: Optional[int] why: str
Fetcher = Callable[[str, str], tuple[int, Mapping[str, str]]] def _http(url: str, method: str, timeout: float = _PROBE_TIMEOUT) -> tuple[int, Mapping[str, str]]: """One request to the sidecar: (status, headers). Only http/https are ever dialled - the URL is operator config, but urlopen would otherwise honour file:// and ftp:// too - and it goes through netfetch so redirects stay inside the same policy. allow_private=True because sidecars are local by design. Raises on anything that is not an HTTP answer.""" parts = urllib.parse.urlsplit(url) if parts.scheme not in ('http', 'https'): raise ValueError('not an http(s) URL') try: _body, headers = netfetch.fetch(url, max_bytes=4096, timeout=timeout, allow_private=True, method=method) return 200, headers except netfetch.FetchHTTPError as e: return e.code, e.headers or {} def _reachable(url: str, timeout: float = _PROBE_TIMEOUT) -> bool: """HEAD the sidecar. Reachable means it answered HTTP at all: a 4xx is a live process that disliked HEAD, not an absent one.""" try: status, _ = _http(url, 'HEAD', timeout) return status < 500 except Exception: return False
[docs]def run_check(fetch: Fetcher, endpoint: str, check: Check) -> ProbeResult: """One check against one endpoint, never raising.""" url = endpoint.rstrip('/') + check.path try: status, headers = fetch(url, check.method) except Exception as e: return ProbeResult(check.name, False, None, '%s: %s' % (type(e).__name__, str(e)[:80] or 'no answer')) if status not in check.expect: return ProbeResult(check.name, False, status, 'HTTP %d, wanted one of %s' % (status, sorted(check.expect))) if status == 200 and check.content_type: ctype = next((v for k, v in headers.items() if k.lower() == 'content-type'), '') if not str(ctype).lower().startswith(check.content_type): return ProbeResult(check.name, False, status, 'content-type %r is not %s' % (ctype, check.content_type)) return ProbeResult(check.name, True, status, '')
[docs]class Sidecars(compartment_mod.Compartment): """Declares one capability per sidecar whose self-tests pass, and keeps testing the sidecars behind them. See module docstring for what it does not do.""" name = 'sidecars' reads = ('sidecars',) + tuple(s.name for s in BUILTINS) # the old top-level keys, one release more live = ('sidecars',) + tuple(s.name for s in BUILTINS) config_cls = SidecarsConfig STORES = ( st.Store(key='probes', description='recent sidecar probe results', kind=st.RING, shape=dict, bound=256), ) CHANNELS = ( Channel('sidecar_verified', 'Sidecar self-tests passed and capability declared', level=INFO, say='sidecar {name} verified and declared', tags=('sidecars', 'probe')), Channel('sidecar_failed', 'Sidecar self-test failed and capability retracted', level=NOTE, say='sidecar {name} self-test failed: {why}', tags=('sidecars', 'probe')), ) METRICS = ( Metric(name='sidecars_verified_total', description='Times sidecars verified.', kind=COUNTER, counts='sidecar_verified', tags=('sidecars', 'probe')), Metric(name='sidecars_failed_total', description='Times sidecar probes failed.', kind=COUNTER, counts='sidecar_failed', tags=('sidecars', 'probe')), ) NEEDS = ( Need('cfg', 'cfg', getter=True), Need('descriptor', 'descriptor'), Need('redescribe', 'redescribe'), ) EVERY = (Every(seconds=_CHECK_EVERY_SECONDS, run='_check_endpoints'),) COMMANDS = ( Command( key='sidecars', name='Configured Sidecars', description='each configured sidecar, its endpoint, whether it is verified and its last self-test', tags=('sidecars', 'status'), permission='sidecars/sidecars', run='list_sidecars', reply='dict', ), Command( key='sidecar_test', name='Sidecar Self-Test', description='run the self-tests of one sidecar (or all) now and redeclare if the result changed', tags=('sidecars', 'test'), parameters=( Parameter(key='name', name='name', description='test: only this sidecar (default all enabled)', type=str, default=None, required=False), ), permission='sidecars/sidecar_test', run='run_sidecar_test', reply='{name: {verified, checks: [{name, ok, status, why}], at}}', ), )
[docs] def __init__(self, cfg: Callable[[], Mapping[str, Any]], clock: Optional[Callable[[], int]] = None, checker: Optional[Callable[[str], bool]] = None, node: Any = None, fetch: Fetcher = _http, descriptor: Optional[Callable[[], Optional[Mapping[str, Any]]]] = None, redescribe: Optional[Callable[[], Any]] = None) -> None: super().__init__(node, clock) self._cfg = cfg # `checker(url) -> bool` is the older, coarser injection: when given it # stands in for every check of a sidecar. `fetch` is the real seam. self._checker = checker self._fetch = fetch self._descriptor = descriptor or (lambda: None) self._redescribe = redescribe self._last_checked: dict[str, int] = {} # not a store: local configured sidecars probe timestamps self._results: dict[str, list[ProbeResult]] = {} # not a store: local configured sidecars probe results self._verified: dict[str, bool] = {} # not a store: local configured sidecars verified flags self._declared: dict[str, Sidecar] = {} # not a store: plugin-declared manifests, by name self._manifest_errors: dict[str, str] = {} # not a store: manifest errors for configured sidecars self.redeclared = 0 self._closed = False
# -- config ----------------------------------------------------------- def _entries(self) -> dict[str, SidecarEntry]: """The `sidecars` block as it is now (old top-level keys folded in); every sidecar reads live.""" out = {} for name, e in sidecars_of(self._cfg()).items(): e = e or {} out[name] = SidecarEntry(enabled=bool(e.get('enabled', False)), endpoint=e.get('endpoint'), manifest=e.get('manifest')) return out
[docs] def block(self, name: str) -> SidecarEntry: return self._entries().get(name, SidecarEntry())
[docs] def enabled(self, name: str) -> bool: return self.block(name).on
[docs] def registry(self) -> list[Sidecar]: """Every sidecar this node knows of, in descriptor order: the built-ins, then plugin-declared ones, then any the operator named only by manifest. A configured entry whose manifest does not resolve is remembered as a manifest problem (trouble()) and skipped - a bad manifest declares nothing, same as a failing check.""" self._manifest_errors = {} # not a store: reset on registry evaluation out: list[Sidecar] = list(BUILTINS) seen = {s.name for s in out} for name, s in self._declared.items(): if name not in seen: out.append(s); seen.add(name) for name, e in self._entries().items(): if e.manifest is None and name in seen: continue try: s = resolve_manifest(name, {'manifest': e.manifest}, self._declared) except (ManifestError, OSError) as err: self._manifest_errors[name] = str(err) continue if name in seen: # operator manifest overrides a plugin's, never a built-in if name in _BUILTIN_BY_NAME: self._manifest_errors[name] = 'sidecars.%s: %s is a built-in; its manifest is not overridable' % (name, name) continue out = [s if x.name == name else x for x in out] else: out.append(s); seen.add(name) return out
[docs] def manifest(self, name: str) -> Sidecar: for s in self.registry(): if s.name == name: return s raise KeyError('no sidecar named %r (have: %s)' % (name, ', '.join(x.name for x in self.registry())))
[docs] def declare(self, sidecar: Sidecar) -> None: """A plugin (or test) adds a manifest the node did not ship. Refused under the reserved origin, like any other third-party manifest.""" manifest_mod.parse(sidecar.to_dict(), origin=RESERVED_ORIGIN) self._declared[sidecar.name] = sidecar
# Readable names for the individual sidecars; enabled(name) is the # generic form and what the table-driven paths use.
[docs] def grasp_enabled(self) -> bool: return self.enabled('grasp')
[docs] def ncontainer_enabled(self) -> bool: return self.enabled('ncontainer')
[docs] def magpie_enabled(self) -> bool: return self.enabled('magpie')
def _endpoints(self) -> dict[str, str]: """Enabled entries whose manifest resolved. One with a manifest problem is not an endpoint: it is neither tested nor declared.""" known = {s.name for s in self.registry()} - set(self._manifest_errors) return {n: str(e.endpoint) for n, e in self._entries().items() if e.on and n in known}
[docs] def verified(self, name: str) -> bool: """Enabled now AND its last self-test passed.""" return self.enabled(name) and self._verified.get(name, False)
# -- what the descriptor says ------------------------------------------
[docs] def capability_protocols(self) -> list[str]: """Extra `c` protocol identifiers this node's descriptor should carry, alongside the required ones `descriptor.build` always adds. Only the sidecars whose self-tests pass: enabling one declares nothing until it answers.""" return [s.capability for s in self.registry() if self.verified(s.name)]
[docs] def descriptor_tags(self) -> list[list[str]]: """<name>-endpoint, only for a verified sidecar - see schema/kinds.json's note on kind 11801.""" eps = self._endpoints() return [[s.tag, eps[s.name]] for s in self.registry() if s.name in eps and self.verified(s.name)]
def _declared_on_descriptor(self) -> set[str]: desc = self._descriptor() if not desc: return set() caps = {t[1] for t in desc.get('tags', []) if len(t) > 1 and t[0] == 'c'} return {s.name for s in self.registry() if s.capability in caps} # -- the self-tests ------------------------------------------------------
[docs] def test(self, name: str, now: Optional[int] = None) -> list[ProbeResult]: """Run one sidecar's checks now. Records the verdict; does not redeclare.""" s = self.manifest(name) at = now if now is not None else self.clock() url = self._endpoints().get(name) if url is None: results = [ProbeResult('enabled', False, None, 'not enabled')] elif self._checker is not None: up = self._checker(url) results = [ProbeResult(c.name, up, 200 if up else None, '' if up else 'no answer') for c in s.checks] else: results = [run_check(self._fetch, url, c) for c in s.checks] self._last_checked[name] = at self._results[name] = results self._verified[name] = url is not None and all(r.ok for r in results) for r in results: if not r.ok: self.refusals.refuse('sidecar', 'self-test %s failed' % r.name, subject=name, detail={'endpoint': url or '', 'status': r.status, 'why': r.why}, at=at) return results
[docs] def test_all(self, now: Optional[int] = None, only: Optional[str] = None) -> dict[str, list[ProbeResult]]: names = [only] if only else list(self._endpoints()) return {n: self.test(n, now) for n in names}
[docs] def redeclare_if_changed(self) -> bool: """Rebuild the descriptor when what should be declared differs from what the current one says. True when it was rebuilt.""" want = {s.name for s in self.registry() if self.verified(s.name)} if want == self._declared_on_descriptor(): return False if self._redescribe is None: return False self._redescribe() self.redeclared += 1 return True
# -- lifecycle -----------------------------------------------------------
[docs] def start(self) -> None: """Test every enabled sidecar before anything is declared, and bring the descriptor into agreement at once rather than a cadence later.""" if self._endpoints(): self.test_all(self.clock()) self._every_last['_check_endpoints'] = self.clock() self.redeclare_if_changed()
def _check_endpoints(self, now: Optional[int] = None) -> list[tuple[Any, ...]]: now = now if now is not None else self.clock() out: list[tuple[Any, ...]] = [] changed = False for name in self._endpoints(): was = self._verified.get(name) self.test(name, now) is_ok = self._verified[name] if was is not None and was != is_ok: out.append(('sidecar', name, 'verified' if is_ok else 'failing')) changed = changed or (was != is_ok) if changed and self.redeclare_if_changed(): out.append(('sidecar', 'descriptor', 'redeclared')) return out _tick_runs_every = True
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]: out: list[tuple[Any, ...]] = [] if any(name not in self._last_checked for name in self._endpoints()): self._every_last['_check_endpoints'] = now out += self._check_endpoints(now) else: out += self.run_every(now) changed = False # A sidecar removed from the config stops being verified without a probe. for name in [n for n in self._verified if n not in self._endpoints()]: del self._verified[name] changed = True if changed and self.redeclare_if_changed(): out.append(('sidecar', 'descriptor', 'redeclared')) return out
def _report(self, name: str) -> SidecarStatusReport: return {'verified': self.verified(name), 'last_checked': self._last_checked.get(name), 'checks': [r._asdict() for r in self._results.get(name, [])]}
[docs] def state(self) -> Mapping[str, Any]: if self._closed: return {'stopped': True} out: MutableMapping[str, Any] = {} entries = self._entries() for s in self.registry(): if s.name in entries: out[s.name] = {'enabled': self.enabled(s.name), 'endpoint': self.block(s.name).endpoint, 'capability': s.capability, 'builtin': s.builtin, 'reachable': self._verified.get(s.name), **self._report(s.name)} else: out[s.name] = {'enabled': False, 'capability': s.capability, 'builtin': s.builtin} for name, why in self._manifest_errors.items(): out[name] = {'enabled': entries[name].enabled, 'manifest_error': why} out['redeclared'] = self.redeclared return out
[docs] def trouble(self) -> list[str]: out = [] eps = self._endpoints() self.registry() for name, why in self._manifest_errors.items(): out.append('%s sidecar has no usable manifest: %s' % (name, why)) for name, url in eps.items(): if self._verified.get(name) is False: failed = [r for r in self._results.get(name, []) if not r.ok] out.append('%s sidecar is configured at %s but not verified: %s' % ( name, url, '; '.join('%s (%s)' % (r.name, r.why) for r in failed) or 'untested')) return out
[docs] def list_sidecars(self) -> Mapping[str, Any]: return {n: {'endpoint': url, **self._report(n)} for n, url in self._endpoints().items()}
[docs] def run_sidecar_test(self, args: Mapping[str, Any]) -> Mapping[str, Any]: only = args.get('name') if only is not None: self.manifest(only) # KeyError names the ones there are if only is not None and not self.enabled(only): raise KeyError('%s is not enabled' % only) now = self.clock() results = self.test_all(now, only) redeclared = self.redeclare_if_changed() return {n: {'verified': all(r.ok for r in rs), 'checks': [r._asdict() for r in rs], 'at': now} for n, rs in results.items()} | {'redeclared': redeclared}
[docs] def stop(self) -> Any: self._closed = True