"""Sync scheduling, shard coverage claims, capacity metering, and strain telemetry."""
from __future__ import annotations
import argparse
import json
import os
import sys
import time
from fleetmesh_node import app as app_mod
from fleetmesh_node.app import socket as control
from .base import (
_ask,
_live,
_peer,
_status_file,
_table,
add_command,
emit,
fail,
load_json_arg,
status_health,
)
def _strain(st: dict) -> None:
if not st or not st.get('kinds'):
print('nothing survived yet. This node has not been stressed, which is')
print('not the same as being strong: a tree raised without wind grows')
print('tall and slender and cannot stand.')
return
print('survived:')
for k, n in sorted(st['kinds'].items(), key=lambda kv: -kv[1]):
print(' %-12s %d' % (k, n))
print('\n%d directions, %.2f weighted, reserves at %.2fx'
% (st.get('directions', 0), st.get('weighted_directions', 0),
st.get('reserve_multiple', 1)))
print('one peer moves this node by at most one direction, however hard it pushes.')
if st.get('flag_tree'):
print('\n!! flag tree: %.0f%% of it is one direction, %s'
% (100 * st.get('dominant_share', 0),
' / '.join(x for x in st['flag_tree']['direction'] if x)[:40]))
print(' %s' % st['flag_tree']['note'])
print('\nnone of this is a finding, and none of it crosses the wire.')
[docs]def cmd_strain(a: argparse.Namespace) -> int:
"""What this node has survived, and what it holds because of it."""
st = _live(a, 'strain')
if st is None:
st = status_health(a.dir).get('strain') or {}
return emit(a, st, _strain)
def _subsystems(subs: dict) -> None:
if not subs:
print('the node has not run yet')
return
for name in sorted(subs):
st = dict(subs[name])
ref = st.pop('refusals', {}) or {}
print('%s' % name)
for k in sorted(st):
v = st[k]
if isinstance(v, (dict, list)) and not v:
continue
print(' %-22s %s' % (k, json.dumps(v) if isinstance(v, (dict, list)) else v))
if ref.get('total'):
print(' %-22s %d' % ('refused', ref['total']))
for why, n in sorted((ref.get('by_reason') or {}).items(), key=lambda kv: -kv[1]):
print(' %6d %s' % (n, why))
print()
[docs]def cmd_subsystems(a: argparse.Namespace) -> int:
"""Every compartment of this node, side by side."""
subs = _live(a, 'subsystems') or status_health(a.dir).get('subsystems') or {}
return emit(a, subs, _subsystems)
def _render_event(e: dict) -> str:
data = e.get('data') or {}
short = ' '.join('%s=%s' % (k, (v[:12] + '…') if isinstance(v, str) and len(v) > 24 else v) for k, v in data.items())
return '%s %-20s %s' % (time.strftime('%H:%M:%S', time.gmtime(e.get('ts') or 0)), e.get('topic'), short)
def _events(out: dict) -> None:
print('seq %s, ring holds %s (oldest %s), %s subscriber(s); topics: %s'
% (out['seq'], out['ring'], out['oldest'], out['subscribers'], ' '.join(out['topics'])))
for e in out['events']:
print(' %6s %s' % (e['seq'], _render_event(e)))
[docs]def cmd_events(a: argparse.Namespace) -> int:
"""What the node's bus carried lately, from memory; needs the daemon."""
out = _live(a, 'events', {'since': a.since, 'topics': a.topics or None, 'last': a.last})
if out is None:
fail('events needs a running node; is one up in %s?' % a.dir)
return emit(a, out, _events)
[docs]def cmd_subscribe(a: argparse.Namespace) -> int:
"""Follow the node's bus live: one line per event until interrupted."""
args = {}
if a.topics:
args['topics'] = a.topics
if a.since is not None:
args['since'] = a.since
try:
it = control.stream(a.dir, 'subscribe', timeout=None, **args)
ack = next(it)
if not a.json:
print('subscribed at seq %s%s' % (ack['seq'], (', replaying %d' % ack['replaying']) if ack.get('replaying') else ''),
file=sys.stderr)
for e in it:
print(json.dumps(e) if a.json else _render_event(e), flush=True)
except control.ControlError as e:
fail(str(e))
except KeyboardInterrupt:
return 0
return 0
def _refusals(rows: list) -> None:
if not rows:
print('this node has refused nothing it recorded.')
return
print('%-10s %-8s %-12s %-16s %s' % ('when', 'from', 'kind', 'subject', 'why'))
for r in rows:
print('%-10s %-8s %-12s %-16s %s' % (
time.strftime('%H:%M:%S', time.gmtime(r['at'])), r['subsystem'][:8],
r['kind'][:12], (r.get('subject') or '-')[:16], r['why']))
if r.get('detail'):
print('%-10s %s' % ('', json.dumps(r['detail'])[:100]))
[docs]def cmd_refusals(a: argparse.Namespace) -> int:
"""Every no this node said, in one place."""
rows = _live(a, 'refusals', {'last': a.last})
if rows is None:
subs = status_health(a.dir).get('subsystems') or {}
rows = []
for name, sub in subs.items():
ref = (sub or {}).get('refusals') or {}
for r in ref.get('recent') or []:
row = dict(r)
row['subsystem'] = name
rows.append(row)
rows.sort(key=lambda r: r.get('at', 0))
if a.last:
rows = rows[-int(a.last):]
return emit(a, rows, _refusals)
def _capacity(c: dict) -> None:
if not c:
print('the node has not run yet, so it has carried nothing')
return
if not c.get('declared'):
print('no aggregate ceiling is declared.')
print('Every grant bounds one peer and nothing bounds this node, so it will')
print('accept load until it fails. What it has carried in one window:')
for k, v in sorted((c.get('peak') or c.get('measured') or {}).items()):
print(' %-12s %s' % (k, v))
print('\nPut a total in config.json under "capacity" to turn the shed ladder on.')
print('The number is yours: a class declares connectivity and not capability,')
print('so nothing in the protocol can guess what this machine will carry.')
return
print('pressure %.0f%%%s' % (100 * (c.get('pressure') or 0),
(' on %s' % c['binding']) if c.get('binding') else ''))
print('%-12s %10s %10s %10s' % ('counter', 'declared', 'this window', 'peak'))
for k in sorted(set(list(c.get('total') or {}) + list(c.get('peak') or {}))):
print('%-12s %10s %10s %10s' % (k, (c.get('total') or {}).get(k, '-'),
(c.get('measured') or {}).get(k, 0),
(c.get('peak') or {}).get(k, 0)))
if c.get('held'):
print('\nshedding, in ladder order:')
for action in c['held']:
since = (c.get('since') or {}).get(action)
print(' %-16s%s' % (action, (' since %s UTC' % time.strftime('%H:%M', time.gmtime(since)))
if since else ''))
print('none of this is a finding about any peer.')
else:
print('\nnothing is being shed.')
[docs]def cmd_capacity(a: argparse.Namespace) -> int:
"""The node's own ceiling: what it promised its peers against what the
machine was told it can do."""
c = _live(a, 'capacity')
if c is None:
c = status_health(a.dir).get('capacity') or {}
return emit(a, c, _capacity)
def _scheduled(s: dict) -> None:
pulls = s.get('pulls') or []
if not pulls:
print('no pulls scheduled')
else:
print('%-16s %-16s %-12s %s' % ('peer', 'shard', 'every', 'last run'))
for p in pulls:
last = ('%ds ago' % (int(time.time()) - p['last'])) if p.get('last') else 'never'
print('%-16s %-16s %-12s %s' % (p['peer'], p['shard'], '%ds' % p.get('every', 900), last))
if s.get('throttled'):
print('\nthrottled (rate-limited by peer grant):')
for t in s['throttled']:
until = time.strftime('%H:%M:%S UTC', time.gmtime(t['until'])) if t.get('until') else 'unknown'
print(' %-16s %-16s %s (until %s)' % (t.get('peer', '')[:16], t.get('shard', ''), t.get('why', ''), until))
if s.get('held'):
print('\nheld (under local capacity ceiling):')
for h in s['held']:
print(' %s' % h)
if s.get('declined_until'):
print('\ndeclined (waiting out interval):')
for ps, until in s['declined_until'].items():
t_str = time.strftime('%H:%M:%S UTC', time.gmtime(until))
print(' %-32s until %s' % (ps, t_str))
[docs]def cmd_scheduled(a: argparse.Namespace) -> int:
"""Every scheduled pull, when it last ran, and what is holding it back."""
s = _live(a, 'scheduled')
if s is None:
cfg = app_mod.load_config(a.dir)
st = _status_file(a.dir)
health = st.get('health') or {}
subs = health.get('subsystems') or {}
sched_sub = subs.get('scheduler') or {}
s = {
'pulls': [
{
'peer': (p.get('pubkey') or p.get('peer') or '')[:16],
'shard': p.get('shard', ''),
'every': p.get('pull_every', 900),
'last': None,
}
for p in cfg.get('peers') or []
],
'throttled': sched_sub.get('throttled') or health.get('throttled') or [],
'held': sched_sub.get('held') or [],
'declined_until': sched_sub.get('declined_until') or {},
}
return emit(a, s, _scheduled)
def _coverage(cov: list) -> None:
if not cov:
print('no shards claimed')
return
_table(('shard', 'completeness', 'filter'),
[(c['shard'], c.get('completeness', 'best-effort'), json.dumps(c['filter'])) for c in cov])
print('\n`asserted` is a request: it is honoured only for a shard every event of which arrived at the relay face')
from ..config.parameter import Parameter
from .adapter import CommandDispatcher
PARAM_SHARD = Parameter(key='shard', name='shard', description='shard name', type=str, default='', positional=True, required=False, nargs='?')
PARAM_FILTER = Parameter(key='filter', name='filter', description='a nostr filter, JSON or @file (add)', type=str, default='')
PARAM_ASSERTED = Parameter(key='asserted', name='asserted', description='claim completeness (honoured only if every event arrived directly)', type=bool, default=False)
def _coverage_list(a: argparse.Namespace) -> int:
cfg = app_mod.load_config(a.dir)
cov = list(cfg.get('coverage') or [])
return emit(a, cov, _coverage)
def _coverage_remove(a: argparse.Namespace) -> int:
cfg = dict(app_mod.load_config(a.dir))
cov = list(cfg.get('coverage') or [])
if not a.shard:
fail('remove needs a SHARD')
kept = [c for c in cov if c['shard'] != a.shard]
if len(kept) == len(cov):
fail('%s is not claimed' % a.shard)
cfg['coverage'] = kept
app_mod.save_config(a.dir, cfg)
print('no longer claiming %s; the claim on the relays expires on its own' % a.shard)
return 0
def _coverage_add(a: argparse.Namespace) -> int:
cfg = dict(app_mod.load_config(a.dir))
cov = list(cfg.get('coverage') or [])
if not a.shard:
fail('add needs a SHARD')
if not a.filter:
fail('add needs --filter')
filt = load_json_arg(a.filter, '--filter')
entry = {'shard': a.shard, 'filter': filt, 'completeness': 'asserted' if a.asserted else 'best-effort'}
cfg['coverage'] = [c for c in cov if c['shard'] != a.shard] + [entry]
problems = app_mod.validate_config(cfg)
if problems:
fail('\n'.join(problems))
app_mod.save_config(a.dir, cfg)
print('claiming %s as %s on the next %s' % (a.shard, entry['completeness'], 'tick' if control.running(a.dir) else 'run'))
return 0
COVERAGE_DISPATCHER = CommandDispatcher('coverage', default_action='list')
COVERAGE_DISPATCHER.add_custom('list', _coverage_list)
COVERAGE_DISPATCHER.add_custom('add', _coverage_add, parameters=(PARAM_SHARD, PARAM_FILTER, PARAM_ASSERTED))
COVERAGE_DISPATCHER.add_custom('remove', _coverage_remove, parameters=(PARAM_SHARD,))
[docs]def cmd_coverage(a: argparse.Namespace) -> int:
return COVERAGE_DISPATCHER.dispatch(a)
[docs]def cmd_descriptor(a: argparse.Namespace) -> int:
dpath = os.path.join(a.dir, app_mod.NodeApp.DESCRIPTOR_FILE)
if not os.path.exists(dpath):
fail('no descriptor.json yet; it is mined on the first run')
with open(dpath, encoding='utf-8') as _fh:
d = json.load(_fh)
if a.republish:
r = _ask(a.dir, 'republish', timeout=60)
def human(r: dict) -> None:
for url, (okay, note) in (r.get('relays') or {}).items():
print(' %-8s %s %s' % ('sent' if okay else 'FAILED', url, note))
return emit(a, r, human)
# The descriptor is JSON whether or not --json was asked for.
print(json.dumps(d, indent=2))
return 0
[docs]def cmd_pull(a: argparse.Namespace) -> int:
pk = _peer(a.dir, a.peer)
filt = None
if a.filter:
filt = load_json_arg(a.filter, '--filter')
else:
for p in app_mod.load_config(a.dir).get('peers', []):
if p['pubkey'] == pk and p['shard'] == a.shard:
filt = p['filter']
if filt is None:
fail('no filter for shard %s with this peer; give --filter' % a.shard)
# The daemon's own pull deadline is sent as a request argument and the
# client waits a little longer than it, so the node's answer (done, or
# timed out by name) always arrives before this end gives up.
r = _ask(a.dir, 'pull', peer=pk, shard=a.shard, filter=filt, force=a.force,
timeout=a.timeout + 10, request={'timeout': a.timeout})
def human(r: dict) -> None:
c = r.get('counters') or {}
print('%s: %s events in, %s ranges, %s bytes in, dup ratio %s'
% (r.get('status'), c.get('events_in', 0), c.get('sync_ranges', 0), c.get('bytes_in', 0), r.get('dup_ratio')))
for sev, why in r.get('findings') or []:
print(' finding %s: %s' % (sev, why))
return emit(a, r, human)
[docs]class SyncCommandProvider:
"""Registers sync, shard coverage, capacity, strain, and pull commands."""
[docs] @classmethod
def register(cls, sub: argparse._SubParsersAction) -> None:
add_command(sub, 'strain', cmd_strain, "what this node has survived, by direction, and the reserves it sets", js=True)
add_command(sub, 'subsystems', cmd_subsystems, 'every compartment of this node, and what each has refused', js=True)
rf = add_command(sub, 'refusals', cmd_refusals, 'every no this node said, across every compartment', js=True)
rf.add_argument('--last', type=int, default=5, help='how many per compartment')
ev = add_command(sub, 'events', cmd_events, "the node's recent bus events, from memory", js=True)
ev.add_argument('--since', type=int, default=0, help='only events after this seq')
ev.add_argument('--topics', nargs='*', help='keep only these topics')
ev.add_argument('--last', type=int, default=50, help='at most this many, newest')
sb = add_command(sub, 'subscribe', cmd_subscribe, "follow the node's bus live, one line per event", js=True)
sb.add_argument('--since', type=int, help='replay what the node still holds after this seq first')
sb.add_argument('--topics', nargs='*', help='keep only these topics')
add_command(sub, 'capacity', cmd_capacity, "this node's own ceiling, the pressure on it, and what it is shedding", js=True)
add_command(sub, 'scheduled', cmd_scheduled, 'every scheduled pull, throttles and holds', js=True)
c = add_command(sub, 'coverage', cmd_coverage, 'the shards this node claims to hold, as kind 30803', js=True)
COVERAGE_DISPATCHER.register_arguments(c, action_choices=('list', 'add', 'remove'))
d = add_command(sub, 'descriptor', cmd_descriptor, 'the kind 11801 this node published', js=True)
d.add_argument('--republish', action='store_true', help='send it to the relays again (needs the node running)')
pl = add_command(sub, 'pull', cmd_pull, 'pull a shard from a peer now (needs the node running)', peer=True, js=True)
pl.add_argument('shard')
pl.add_argument('--filter', help='a nostr filter, JSON or @file; default: the one configured for this peer and shard')
pl.add_argument('--force', action='store_true', help='pull even when the grant does not cover a session now')
pl.add_argument('--timeout', type=float, default=60.0, help='seconds the node gives the session (default 60, max 600)')