"""The relay links this node holds open, as a compartment.
The transport keeps the sockets and counts what it drops. Nothing watched it:
the tick read `transport.health` directly to decide what to say to the journal,
and `health()` read six more of its counters by name. A subscription that
dropped was therefore visible in two hand-written places and in no refusal
channel at all, so `meshnode refusals` - the one view that answers "what has
this node been saying no to" - could not show a relay that had been refusing
this node for an hour.
This compartment owns what the journal has already been told, so it says a
transition once and not once a tick, and it records the drop as a refusal like
every other no.
It also records the transport's verdicts on stored messages. A kind 1059 the
relay accepted and never served used to be indistinguishable from one that
arrived; now every stored message asks for an acknowledgment, and a peer that
has spoken since and not acknowledged is a `stored: unacknowledged` refusal
here, with the message type and the relays it went through, so `meshnode
refusals` can show the day the control plane went quiet.
"""
from __future__ import annotations
from typing import Any, Callable, Optional
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.channels import Channel, NOTE, TROUBLE
from fleetmesh_node.compartment import Command, Compartment, Need
from fleetmesh_node.metrics import COUNTER, Metric
from fleetmesh_node.interfaces import AwaitingAckRow
from fleetmesh_node.transport.ladder import Fault
from fleetmesh_node.transport.relay_rung.cooldown import Cooling
from fleetmesh_node.config import TransportConfig
from .types import LinkState
[docs]class Link(Compartment):
name: str = 'link'
reads: tuple[str, ...] = ('transport',)
config_cls = TransportConfig
# strain is constructed before link so a reconnect can be recorded into it,
# but ticks after link (the journal first); it is bound, not required.
requires: tuple[str, ...] = ()
NEEDS = (
Need('transport'),
Need('strain', late=True),
)
COMMANDS = (
Command(
key='show',
name='Link',
description='every relay subscription, what it has dropped, what is waiting to be '
'read, and the stored messages still waiting to be acknowledged',
tags=('link', 'status'),
permission='link/show',
run='state',
reply='{relays, dropped, recent_faults, publish_refused_in_window, cooling, inbox, '
'wraps_in, wraps_seen_again, unfinished_envelopes, awaiting_ack, acked, '
'acks_in, acks_out, unacknowledged}',
),
)
STORES = (
st.Store(key='_up', description='last reported connection state per relay url',
kind=st.TABLE, shape=bool, bound=1024, keyed_by='url', evict=st.LRU),
st.Store(key='_said', description='last reported unreachable reason per relay url',
kind=st.TABLE, bound=1024, keyed_by='url', evict=st.LRU),
st.Store(key='_ack', description='last reported subscription-acknowledged state per relay url',
kind=st.TABLE, shape=bool, bound=1024, keyed_by='url', evict=st.LRU),
)
CHANNELS = (
Channel('relay_up', 'Relay subscription connected', say='relay {url} subscription up'),
Channel('relay_down', 'Relay subscription disconnected', level=TROUBLE,
say='relay {url} subscription down ({reason}), reconnecting'),
Channel('relay_unreachable', 'Relay not reachable', level=TROUBLE,
say='relay {url} not reachable ({reason}), retrying'),
Channel('unacknowledged', 'Stored message was never acknowledged', level=NOTE,
say='stored {peer} {type} never acknowledged; the peer has spoken since via {relays}'),
Channel('relay_deaf', 'Relay holds the socket but never acknowledged the subscription',
level=TROUBLE,
say='relay {url} took the subscription and never acknowledged it ({reason})'),
Channel('relay_acked', 'Relay acknowledged the subscription',
say='relay {url} acknowledged the subscription'),
)
METRICS = (
Metric(name='link_relays_connected', description='Times a relay connection came up.',
kind=COUNTER, counts='relay_up', tags=('link', 'relay')),
Metric(name='link_relays_dropped', description='Times a relay connection dropped.',
kind=COUNTER, counts='relay_down', tags=('link', 'relay')),
Metric(name='link_relays_unreachable', description='Times a relay was unreachable.',
kind=COUNTER, counts='relay_unreachable', tags=('link', 'relay')),
Metric(name='link_unacknowledged', description='Times a stored message was unacknowledged.',
kind=COUNTER, counts='unacknowledged', tags=('link', 'transport')),
)
[docs] def bind(self, strain: Any) -> None:
"""The ledger a reconnect is recorded into. Bound after both exist
because strain ticks after link and the assembler builds in tick order."""
self.strain = strain
[docs] def __init__(
self,
transport: Any,
strain: Optional[Any] = None,
clock: Optional[Callable[[], int]] = None,
node: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
self.transport: Any = transport
self.strain = strain
self._up: st.Table[str, bool] # url -> the connected state the journal was last told
self._said: st.Table[str, Optional[str]] # url -> the last unreachable reason the journal was told
self._ack: st.Table[str, bool] # url -> the acknowledged state the journal was last told
[docs] def tick(self, now: int) -> list[tuple[Any, ...]]:
"""Say what changed, once each way it changed.
A relay that is down stays down, and saying so every thirty seconds
buries everything else in the journal. A transition is news; a state
is not.
"""
for url, h in self.transport.health.items():
prev = self._up.get(url)
if h['connected'] and prev is not True:
self.say('relay_up', url=url)
elif not h['connected'] and prev is True:
self.say('relay_down', url=url, reason=h['last_error'] or 'no reason')
self.refusals.refuse('subscription', 'dropped', subject=url,
detail={'error': h['last_error']}, at=now)
self._said[url] = h['last_error'] # the next tick must not say the same reason again
if self.strain is not None:
self.strain.record('reconnect', transport=url)
elif not h['connected'] and h['last_error'] and self._said.get(url) != h['last_error']:
self.say('relay_unreachable', url=url, reason=h['last_error'])
self._said[url] = h['last_error']
self.refusals.refuse('subscription', 'not reachable', subject=url,
detail={'error': h['last_error']}, at=now)
# `connected` is the socket; `confirmed` is the relay's own EOSE.
# A relay that holds the socket and never acknowledges carries no
# events to us, so the inbox is deaf - and every branch above keys
# off `connected`, which is True, so the journal said 'subscription
# up' and nothing further (audit I13). Said on transition, like
# `connected`; trouble() is what covers a run that is never
# acknowledged at all, because that run has no transition to say.
if h['connected']:
was = self._ack.get(url)
acked = bool(h.get('confirmed'))
if acked and was is not True:
self.say('relay_acked', url=url)
elif not acked and was is True:
self.say('relay_deaf', url=url, reason=h['last_error'] or 'no EOSE')
self._ack[url] = acked
self._up[url] = h['connected']
out: list[tuple[Any, ...]] = []
self.transport.sweep(now)
for v in self.transport.take_unacked():
if v['verdict'] != 'unacknowledged':
continue # expired: nothing could arrive any more, and nobody was heard
self.say('unacknowledged', peer=v['peer'][:16], type=v['type'], relays=', '.join(v['relays']) or 'no relay')
self.refusals.refuse('stored', 'unacknowledged', subject=v['peer'],
detail={'type': v['type'], 'sent_at': v['at'], 'heard_at': v['heard_at'],
'relays': v['relays'], 'mid': v['mid'][:16]}, at=now)
out.append(('unacknowledged', v['peer'], v['type']))
return out
FAULTS_SHOWN: int = 20
def _faults(self, since: Optional[int] = None) -> list[Fault]:
recent = getattr(self.transport, 'recent_faults', None)
if recent is None: # a transport without a fault log: nothing to read
return []
return list(recent(since))
def _cooling(self) -> dict[str, Cooling]:
cd = self.transport.cooldown
return cd.state() if cd is not None else {}
[docs] def publish_refused(self, now: Optional[int] = None) -> list[Fault]:
"""Wraps a relay refused within TROUBLE_WINDOW: the fault the OK
reports and nothing else shows, because a refused wrap is counted
as sent by the caller that published it."""
at = self.clock() if now is None else now
return [f for f in self._faults(at - self.TROUBLE_WINDOW)
if f['reason'].split(':', 1)[0] == 'publish refused']
[docs] def state(self) -> LinkState:
t = self.transport
return LinkState(
relays={url: dict(h) for url, h in t.health.items()}, # type: ignore[misc]
rungs=dict(t.rung_health()) if hasattr(t, 'rung_health') else {},
dropped=dict(t.drop_counts),
recent_faults=list(reversed(self._faults()))[:self.FAULTS_SHOWN],
publish_refused_in_window=len(self.publish_refused()),
cooling=self._cooling(),
inbox=t.inbox.qsize(),
wraps_in={str(k): v for k, v in t.wraps_in.items()},
wraps_seen_again={str(k): v for k, v in t.wraps_seen_again.items()},
unfinished_envelopes=t.unfinished(),
awaiting_ack=[AwaitingAckRow(peer=e['peer'][:16], type=e['type'], at=e['at'], relays=e['relays'])
for e in list(t.awaiting.values())],
acked=dict(t.acked),
acks_in=t.acks_in,
acks_out=t.acks_out,
unacknowledged=[AwaitingAckRow(peer=v['peer'][:16], type=v['type'], at=v['at'],
heard_at=v['heard_at'], relays=v['relays'])
for v in list(t.unacked) if v['verdict'] == 'unacknowledged'],
)
[docs] def notes(self) -> list[str]:
cooling = self._cooling()
if not cooling:
return []
# Not wrong in the node, but a relay it is not publishing to; an
# operator who sees a quiet relay should know it was asked to stop.
return ['%d relay(s) cooling after a rate-limit: %s'
% (len(cooling), '; '.join('%s until %d (%s)' % (u, c['until'], c['why'])
for u, c in sorted(cooling.items())))]
[docs] def trouble(self) -> list[str]:
out: list[str] = []
down = sorted(u for u, h in self.transport.health.items() if not h['connected'])
if down:
out.append('relay subscription down: %s' % ', '.join(down))
# The deaf inbox. `wait_listening` counts `confirmed`, and app start
# warns once on a timeout justifying itself with "`link` will keep
# saying so" - which it did not, because every row here read
# `connected` instead. A relay that holds the socket and never sends
# EOSE is connected, unconfirmed, quiet, and green in health and
# `meshctl watch`, while live wraps sent to it are lost: kind 21059 is
# ephemeral, so no relay keeps a copy (audit I13).
deaf = sorted((u, h['last_error']) for u, h in self.transport.health.items()
if h['connected'] and not h.get('confirmed'))
if deaf:
out.append('%d relay subscription(s) connected but never acknowledged: %s'
% (len(deaf), '; '.join('%s (%s)' % (u, e or 'no EOSE') for u, e in deaf)))
refused = self.publish_refused()
if refused:
# A relay that takes the subscription and refuses every wrap
# looks healthy in every other row: connected, confirmed, quiet.
by_relay: dict[str, str] = {}
for f in refused:
rest = f['reason'].split(':', 1)[1].strip() # 'wss://relay reason text'
url, _, why = rest.partition(' ')
by_relay.setdefault(url, why)
out.append('%d wrap(s) refused by relays in the last %ds: %s'
% (len(refused), self.TROUBLE_WINDOW,
'; '.join('%s (%s)' % (u, w) for u, w in sorted(by_relay.items()))))
recent = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='stored')
if recent:
# The one fault the publish result cannot show: what was said to
# the peer never reached it, and the peer was there to hear it.
by = sorted({'%s %s' % ((r.subject or '')[:16], r.detail.get('type', '')) for r in recent})
out.append('%d stored message(s) never acknowledged by a peer that has spoken since: %s'
% (len(recent), '; '.join(by)))
return out