"""The operator's own decisions about peers, carried out once each.
A tier is only ever raised by a person. The ladder earns standing with observed
time; the specification says an operator MAY seed a tier for a node it knows out
of band, recorded as a signed event and reversible. `meshnode trust` writes that
decision to the config file and this compartment is what acts on it.
It ran inside the node's tick before, which meant a seed that could not be
applied - no peering yet, or a regrant that came back empty - was skipped in
silence. An operator who had written a decision and watched nothing happen had
no way to ask why. Every skip is a refusal here.
"""
from __future__ import annotations
import dataclasses
from typing import Any, Callable, Mapping, Optional
from fleetmesh_node.storage import grants as grants_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, NOTE, TROUBLE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Need
from fleetmesh_node.config.parameter import Parameter, ParamBlock
from .types import (
TrustStandingRow,
TrustStateReport,
)
[docs]def decision(entry: Any) -> tuple[str, int]:
"""A trust entry is {'tier': t, 'at': unix} - or, from before decisions
were dated, the bare tier, which counts as decided at time zero."""
if isinstance(entry, dict):
return entry.get('tier', 'probation'), int(entry.get('at') or 0)
return entry, 0
[docs]def tiers(trust: Optional[Mapping[str, Any]]) -> dict[str, str]:
"""The policy's view: peer -> tier."""
return {pk: decision(e)[0] for pk, e in (trust or {}).items()}
[docs]@dataclasses.dataclass(frozen=True)
class TrustConfig(ParamBlock):
key = 'trust'
name = 'trust'
DERIVED: bool = False
PARAMS = (
Parameter(key='max_decisions', name='Max Decisions', description='Maximum trust decisions remembered', type=int, default=4096, live=True, tags=('trust', 'policy')),
)
max_decisions: int = 4096
[docs]class Trust(Compartment):
name: str = 'trust'
reads: tuple[str, ...] = ('trust',)
live: tuple[str, ...] = ('trust',)
config_cls = TrustConfig
CHANNELS = (
Channel('trust_applied', 'Trust decision applied for peer', level=NOTE, say='trust decision applied for peer {peer}', tags=('trust', 'policy')),
Channel('trust_refused', 'Trust decision refused for peer', level=TROUBLE, say='trust decision refused for peer {peer}: {reason}', tags=('trust', 'policy')),
)
METRICS = (
Metric(name='trust_decisions_applied', description='Times trust decisions were applied.', kind=COUNTER, counts='trust_applied', tags=('trust', 'policy')),
Metric(name='trust_decisions_refused', description='Times trust decisions were refused.', kind=COUNTER, counts='trust_refused', tags=('trust', 'policy')),
)
NEEDS = (
Need('grant_store', 'stores.grants'),
Need('policy', 'policy'),
Need('regrant', 'regrant'),
)
COMMANDS = (
Command(
key='show',
name='Trust',
description='the decisions this operator has made, and which of them are still in '
'force: carried out is not the same as true, and the ladder can move a '
'seeded peer',
tags=('trust', 'status'),
permission='trust/show',
run='state',
reply='{decided, in_force, pending, overridden, waiting}',
),
)
STORES = (
st.Store(key='_said', description='refused decisions said once, per peer',
kind=st.TABLE, shape=tuple, bound=4096, keyed_by='peer', evict=st.LRU),
)
[docs] def __init__(
self,
grant_store: grants_mod.GrantStore,
policy: Any,
regrant: Callable[[str, str, int], Any],
clock: Optional[Callable[[], int]] = None,
node: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
self.grants: grants_mod.GrantStore = grant_store
self.policy: Any = policy
self.regrant: Callable[[str, str, int], Any] = regrant
self._said: st.Table[str, tuple[tuple[str, int], str]] # peer -> the decision already refused, so a wait is said once
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
"""Carry out each dated decision in the config once.
A decision is compared with the record of decisions applied, kept
beside the grants, and not with the grant's tier: comparing tiers made
the seed re-issue `trusted` on the tick after every demotion the ladder
earned, so the ladder never held and every re-issue was one more
amend. A decision written while the node was down is still found on
the first tick back, because the record is on disk too. Applying a
seed also drops any grant the ladder had waiting for the boundary: the
operator's word is the later one.
"""
did: list[tuple[Any, ...]] = []
trust = self.config('trust') or {}
self.policy.tiers = tiers(trust)
for peer_pk, entry in trust.items():
tier, at = decision(entry)
if self.grants.seed_applied(peer_pk) == (tier, at):
continue
row = self.grants.get(peer_pk, grants_mod.ISSUED)
if row is None or row['status'] != 'live':
# No peering yet: the policy seeds the next one. Recorded once
# per decision and not once a tick - a refusal repeated every
# thirty seconds rolls the buffer that holds the real ones, and
# makes trouble() shout about a wait that is not a fault.
self._say(peer_pk, (tier, at), 'no live peering to amend', now)
continue
if row['payload'].get('tier') == tier and self.grants.pending(peer_pk) is None:
self.grants.put_seed(peer_pk, tier, at) # already so; nothing to re-issue
continue
if self.regrant(peer_pk, tier, now) is not None:
self.grants.put_seed(peer_pk, tier, at)
self._said.pop(peer_pk, None)
did.append(('granted', peer_pk, tier))
else:
self._say(peer_pk, (tier, at), 'the grant could not be re-issued', now)
return did
def _say(self, peer_pk: str, decided: tuple[str, int], why: str, now: int) -> None:
if self._said.get(peer_pk) == (decided, why):
return
self._said[peer_pk] = (decided, why)
self.refusals.refuse('seed', why, subject=peer_pk,
detail={'tier': decided[0], 'at': decided[1]}, at=now)
[docs] def standing(self) -> list[TrustStandingRow]:
"""Each decision, and what has become of it.
Three states, not two. `applied` used to mean "this dated decision was
carried out at some point", and an operator read it as "the tier I
chose is the tier in force". Those diverge the moment the ladder
demotes a seeded peer: the seed is recorded as applied, this compartment
correctly declines to re-issue it - re-issuing on every demotion is what
made the ladder never hold, and the dated record is the fix for that -
and the operator's decision quietly stops being true.
Seen on the mesh. A node seeded `trusted` was demoted to probation in a
run of S1 windows, and twelve hours later `trust` still said
`applied: 1, waiting: []` while every window ran under a probation
ceiling the operator had never chosen. Nothing was wrong with the
mechanism. The report was answering a different question from the one
being asked of it.
waiting decided, not carried out yet
pending carried out, and published at the next window boundary
in force carried out, and the grant says so
overridden carried out, and the ladder has since moved it
Only the last needs a person, which is why only the last is trouble.
`pending` exists because a decision no longer lands the moment it is
made: publishing a grant inside a running window breaks the subject's
receipt chain for the rest of it, so every decision waits for the
boundary. Without this state, the hour between deciding and landing
would read as overridden and shout for a person who need do nothing.
"""
out: list[TrustStandingRow] = []
for pk, entry in (self.config('trust') or {}).items():
tier, at = decision(entry)
row = self.grants.get(pk, grants_mod.ISSUED)
held = row['payload'].get('tier') if row else None
pend = self.grants.pending(pk)
if self.grants.seed_applied(pk) != (tier, at):
state = 'waiting'
elif held == tier:
state = 'in force'
elif pend is not None and (pend.get('payload') or {}).get('tier') == tier:
state = 'pending'
else:
state = 'overridden'
out.append({'peer': pk, 'tier': tier, 'at': at, 'held': held, 'state': state})
return out
[docs] def state(self) -> TrustStateReport:
rows = self.standing()
return {
'decided': {
r['peer'][:16]: {
'tier': r['tier'],
'at': r['at'],
'held': r['held'],
'state': r['state'],
}
for r in rows
},
'in_force': sum(1 for r in rows if r['state'] == 'in force'),
'pending': sorted(r['peer'][:16] for r in rows if r['state'] == 'pending'),
'overridden': sorted(r['peer'][:16] for r in rows if r['state'] == 'overridden'),
'waiting': sorted(r['peer'][:16] for r in rows if r['state'] == 'waiting'),
}
[docs] def trouble(self) -> list[str]:
"""A decision waiting on a peering that does not exist yet is not
trouble - it is what a seed written before the peering looks like, and
the policy will apply it to the next handshake. A decision that reached
a live peering and could not be written is. So is one the ladder has
since overridden, because that one looks applied and is not in force,
and only a person can decide which of the two answers should stand.
"""
out: list[str] = []
stuck = sorted({(pk or '')[:16] for pk, (_, why) in self._said.items()
if why == 'the grant could not be re-issued'})
if stuck:
out.append('%d trust decision(s) reached a live peering and could not be written: %s. '
'The tier the operator decided on is not the tier in force'
% (len(stuck), ', '.join(stuck)))
gone = [r for r in self.standing() if r['state'] == 'overridden']
if gone:
out.append('%d trust decision(s) the ladder has overridden: %s. '
'Each was carried out and is no longer in force; re-deciding means '
'writing the tier again with a later date'
% (len(gone), ', '.join('%s decided %s, holds %s'
% (r['peer'][:16], r['tier'], r['held']) for r in gone)))
return out