"""The pulls this node is configured to make, and every reason one did not happen.
This was seventy lines in the middle of the node's tick, and it was the worst
place in the node to be a stranger. Five things can stop a scheduled pull: the
peer has no peering yet and the proposal raised; the proposal came back
declined; this node is over its own ceiling and has stopped pulling as the last
shed rung; the grant this node holds covers no session this window; or the
session itself failed. Every one of them was appended to the tick's return value
as a tuple, printed once, and then gone. `meshnode refusals` could not show any
of them, `health` counted none of them, and an operator watching a peer go quiet
had to read the journal back far enough to find the line - if the journal still
had it.
A refusal that is not recorded is indistinguishable from silence. This
compartment is the five of them given a channel.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional
if TYPE_CHECKING:
from fleetmesh_node.planes.peering.budget.core import Budget
from fleetmesh_node.planes.peering.capacity.compartment import Capacity
from fleetmesh_node.planes.peering.strain.compartment import StrainLedger
import dataclasses
from fleetmesh_node.interfaces import ThrottledRow
from fleetmesh_node.planes.store.sync.shape import SyncSessionResult
from .types import SchedulerStateReport
from fleetmesh_core.accounting import accounting
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.storage import grants as grants_mod
from fleetmesh_node.channels import Channel, NOTE
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.compartment import Command, Compartment, Need
from fleetmesh_node.config.parameter import Parameter, ParamBlock
# A decline names a reason that does not change in ten minutes: a digest, a
# The cadence when a peer has no grant to derive one from: there is no
# granted rate to pace against until one is issued.
DEFAULT_PULL_EVERY = 900
# missing protocol, a closed door. Ask again in an hour, not at the pull cadence.
DECLINE_RETRY: int = 3600
[docs]@dataclasses.dataclass(frozen=True)
class SchedulerConfig(ParamBlock):
key = 'scheduler'
name = 'scheduler'
DERIVED: bool = False
PARAMS = (
Parameter(key='default_pull_every', name='Default Pull Cadence', description='Seconds between pulls when peer grant has no rate', type=int, default=DEFAULT_PULL_EVERY, tags=('scheduler', 'pull')),
Parameter(key='decline_retry', name='Decline Retry Horizon', description='Seconds before re-proposing to declined peer', type=int, default=DECLINE_RETRY, tags=('scheduler', 'pull')),
)
default_pull_every: int = DEFAULT_PULL_EVERY
decline_retry: int = DECLINE_RETRY
[docs]class Scheduler(Compartment):
requires: tuple[str, ...] = ('budget', 'capacity', 'strain')
name: str = 'scheduler'
reads: tuple[str, ...] = ('peers',)
live: tuple[str, ...] = ('peers',)
config_cls = SchedulerConfig
CHANNELS = (
Channel('pull_scheduled', 'Pull scheduled for peer and shard', level=NOTE, say='scheduled pull for peer {peer} on shard {shard}', tags=('scheduler', 'pull')),
Channel('pull_refused', 'Scheduled pull refused', level=NOTE, say='pull refused for peer {peer} on shard {shard}: {reason}', tags=('scheduler', 'pull')),
)
METRICS = (
Metric(name='scheduler_pulls_scheduled', description='Times pulls were scheduled.', kind=COUNTER, counts='pull_scheduled', tags=('scheduler', 'pull')),
Metric(name='scheduler_pulls_refused', description='Times scheduled pulls were refused.', kind=COUNTER, counts='pull_refused', tags=('scheduler', 'pull')),
)
NEEDS = (
Need('grant_store', 'stores.grants'),
Need('budget'),
Need('capacity'),
Need('propose'),
Need('pull'),
Need('strain'),
)
STORES = (
st.Store(
key='_last_pull',
description='when a session last ran for (peer, shard)',
bound=1024,
evict=st.LRU,
kind=st.TABLE,
keyed_by='pull',
),
st.Store(
key='_declined',
description='when a declined proposal may be tried again for (peer, shard)',
bound=1024,
evict=st.LRU,
kind=st.TABLE,
keyed_by='pull',
),
st.Store(
key='_throttled',
description='(epoch, why) last recorded for (peer, shard)',
bound=1024,
evict=st.LRU,
kind=st.TABLE,
keyed_by='pull',
),
st.Store(
key='_shed',
description='whether the pull is held under the ceiling for (peer, shard)',
bound=1024,
evict=st.LRU,
kind=st.TABLE,
keyed_by='pull',
),
)
COMMANDS = (
Command(
key='show',
name='Scheduler',
description='every scheduled pull, when it last ran, and what is holding it back now',
tags=('scheduler', 'status'),
permission='scheduler/show',
run='state',
reply='{pulls, throttled, held, declined_until}',
),
)
[docs] def __init__(
self,
grant_store: grants_mod.GrantStore,
budget: Budget,
capacity: Optional[Capacity],
propose: Callable[..., tuple[str, Any]],
pull: Callable[..., SyncSessionResult],
strain: Optional[StrainLedger] = None,
clock: Optional[Callable[[], int]] = None,
node: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
self.grants: grants_mod.GrantStore = grant_store
self.budget: Budget = budget
self.capacity: Optional[Capacity] = capacity
self.propose: Callable[..., tuple[str, Any]] = propose
self.pull: Callable[..., SyncSessionResult] = pull
self.strain: Optional[StrainLedger] = strain
self._last_pull: st.Table[tuple[str, str], int] # (peer, shard) -> when a session last ran
self._declined: st.Table[tuple[str, str], int] # (peer, shard) -> when a declined proposal may be tried again
self._throttled: st.Table[tuple[str, str], tuple[int, str]] # (peer, shard) -> (epoch, why) last recorded
self._shed: st.Table[tuple[str, str], bool] # (peer, shard) -> whether the pull is held under the ceiling
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
did: list[tuple[Any, ...]] = []
for p in self.config('peers') or []:
key = (p['pubkey'], p['shard'])
held = self.grants.get(p['pubkey'], grants_mod.RECEIVED)
last = self._last_pull.get(key)
if held is not None and held['status'] == 'live':
# The budget is a ceiling, not a timetable. `conn_rate` is a
# rate limit - so many sessions per window - and a node is
# entitled to spend it as fast as it likes up to the limit.
# An earlier draft of this spaced the remaining sessions
# evenly across the remaining window, which is a politeness
# policy nothing asked for and which slowed exactly the node a
# tier promotion was meant to speed up.
#
# So: a spent budget is a hard stop, and the cadence is a
# preference whose DEFAULT is derived from the grant -
# window / conn_rate. An unconfigured node therefore paces at
# the rate it was granted, and a promotion from four sessions
# an hour to twelve moves it from 900s to 300s by itself,
# where before the two agreed only by the coincidence that
# 3600/4 is the configured 900.
#
# The hard stop is also what closes a defect. `_last_pull` is
# in memory, so after a restart `last is None` and no cadence
# gate can fire - the node pulled immediately however recently
# it had pulled, spending budget the window had already
# allocated. Three restarts inside window 497614 is how that
# window came to carry more sessions than the grant allowed.
# `sessions_left` reads the meter, which is persisted and is
# the record the peer prices, so a restart inherits what was
# spent instead of forgetting it.
left, closes = self.budget.sessions_left(p['pubkey'], now)
if left <= 0:
# Spent: pace, do not attempt. Before this the node tried
# anyway and `throttled()` below refused it, recording
# "the grant covers no session this window" for what is
# ordinary pacing. That path stays as the backstop for a
# real overrun on another counter.
continue
every = self._cadence(p)
if last is not None and now - last < int(every):
continue
if held is None or held['status'] != 'live':
# No grant, or one a terminate zeroed. `terminate` leaves the
# revoked row so a restart knows the peering ended, and `peer`
# afterwards is the operator saying it should begin again; a
# row that is not live is as much a reason to propose as none,
# and pull_every is a cadence for pulling, not for asking. A
# decline still holds its own retry below.
if now < self._declined.get(key, 0):
continue
try:
kw = {'timeout': p['timeout']} if 'timeout' in p else {}
try:
status, detail = self.propose(p['pubkey'], p['shard'], p['filter'], **kw)
except TypeError:
status, detail = self.propose(p['pubkey'], p['shard'], p['filter'])
except Exception as e:
# A peer with no descriptor yet raised out of the whole tick,
# so every other peer's receipts waited on it.
self.refusals.refuse('propose', 'the handshake raised', subject=p['pubkey'],
detail={'shard': p['shard'],
'error': '%s: %s' % (type(e).__name__, e)}, at=now)
did.append(('propose failed', p['pubkey'], '%s: %s' % (type(e).__name__, e)))
self._last_pull[key] = now
continue
did.append(('proposed', p['pubkey'], status))
if status != 'peered':
self._last_pull[key] = now
retry = int(self.strain.reserve(DECLINE_RETRY)) if self.strain is not None else DECLINE_RETRY
self._declined[key] = now + retry
self.refusals.refuse('propose', 'the peer declined', subject=p['pubkey'],
detail={'shard': p['shard'], 'status': status,
'retry_at': self._declined[key]}, at=now)
continue
self._declined.pop(key, None)
if self.capacity is not None and self.capacity.leafing():
# The last rung. A node over its own ceiling stops pulling, which
# is the only load it chooses. It keeps answering receipts and
# complaints, because accounting is what makes a shed legible
# rather than a partition.
if self._shed.get(key) is not True:
self._shed[key] = True
self.refusals.refuse('pull', 'this node is over its own ceiling',
subject=p['pubkey'], detail={'shard': p['shard']}, at=now)
did.append(('pull held', p['pubkey'], 'this node is over its own ceiling'))
self._last_pull[key] = now
continue
self._shed.pop(key, None)
# The grant this node holds is read before it acts, not only after.
# pull_every is the operator's ceiling; the grant's is the lower one.
why = self.budget.throttled(p['pubkey'], now)
if why is not None:
epoch = self.budget.epoch(p['pubkey'], grants_mod.RECEIVED, now)
if self._throttled.get(key) != (epoch, why):
self._throttled[key] = (epoch, why)
# The reason is the category and the numbers are the detail:
# "sync_ranges 48 over the grant's 40" as a reason would put
# a new key in the counter for every window that overran.
self.refusals.refuse('pull', 'the grant covers no session this window',
subject=p['pubkey'],
detail={'shard': p['shard'], 'epoch': epoch, 'over': why}, at=now)
did.append(('throttled', p['pubkey'], why))
self._last_pull[key] = now
continue
try:
res = self.pull(p['pubkey'], p['shard'], p['filter'])
did.append(('pulled', p['pubkey'], res['status'], res['counters'].get('events_in', 0)))
except Exception as e:
self.refusals.refuse('pull', 'the session raised', subject=p['pubkey'],
detail={'shard': p['shard'],
'error': '%s: %s' % (type(e).__name__, e)}, at=now)
did.append(('pull failed', p['pubkey'], str(e)))
self._last_pull[key] = now
return did
# ---- being watched --------------------------------------------------------
[docs] def throttled(self) -> list[ThrottledRow]:
"""The pulls this node is holding back this window, and until when.
A throttled peer goes quiet in both directions: no sessions, so no
usage, so no receipts due either way. The relay line then says "no
event yet" and an operator reads a dead node. Four nodes on three
machines produced exactly that reading for twenty minutes. Nothing was
wrong; the grant's ceiling had been reached. Say so, and say when it
turns.
"""
out: list[ThrottledRow] = []
for (pk, shard), (epoch, why) in sorted(self._throttled.items()):
try:
w = self.budget._window_of(pk, grants_mod.RECEIVED)
if self.budget.epoch(pk, grants_mod.RECEIVED) != epoch:
continue
until = accounting.window_bounds(epoch, w)[1]
except Exception:
continue
out.append({'peer': pk, 'shard': shard, 'why': why, 'until': until})
return out
def _cadence(self, p: Mapping[str, Any]) -> int:
"""Seconds between pulls of this peer: the operator's number, or the
grant's own rate when there is none.
Here rather than inline because `state()` reports it and the loop
obeys it, and the two read a hardcoded 900 and a derived value
respectively - so an unconfigured node pulled every window/conn_rate
while telling its operator 900. Two places agreeing by hand is how
that happened; one method is the fix.
"""
every = p.get('pull_every')
if every is not None:
return int(every)
allowed, w = self.budget.granted_rate(p['pubkey'])
return (w // allowed) if allowed > 0 else DEFAULT_PULL_EVERY
[docs] def state(self) -> SchedulerStateReport:
return {
'pulls': [
{
'peer': p['pubkey'][:16],
'shard': p['shard'],
'every': self._cadence(p),
'last': self._last_pull.get((p['pubkey'], p['shard'])),
}
for p in self.config('peers') or []
],
'throttled': self.throttled(),
'held': sorted('%s %s' % (pk[:16], shard) for (pk, shard), v in self._shed.items() if v),
'declined_until': {
'%s %s' % (pk[:16], shard): at
for (pk, shard), at in self._declined.items()
},
}
def _live_with(self, peer: str) -> bool:
held = self.grants.get(peer, grants_mod.RECEIVED)
return held is not None and held['status'] == 'live'
[docs] def trouble(self) -> list[str]:
out: list[str] = []
now = self.clock()
failed = self.refusals.within(self.TROUBLE_WINDOW, now, kind='pull', why='the session raised')
if failed:
out.append('%d scheduled pull(s) failed: %s'
% (len(failed), ', '.join(sorted({(r.subject or '')[:16] for r in failed}))))
raised = self.refusals.within(self.TROUBLE_WINDOW, now, kind='propose', why='the handshake raised')
# A proposal that got no answer because the peer was not up yet - the
# first minute of a mesh deploy, every time - is answered by the
# retry. Once the grant is live the failure is history, not trouble.
raised = [r for r in raised if not self._live_with(r.subject or '')]
if raised:
out.append('%d peering(s) could not be proposed: %s'
% (len(raised), ', '.join(sorted({(r.subject or '')[:16] for r in raised}))))
held = sorted(k for k, v in self._shed.items() if v)
if held:
out.append('%d scheduled pull(s) held because this node is over its own ceiling: %s'
% (len(held), ', '.join('%s %s' % (pk[:16], shard) for pk, shard in held)))
return out