Source code for fleetmesh_node.commands.sync

"""Sync scheduling, shard coverage claims, capacity metering, and strain telemetry."""
from __future__ import annotations

import argparse
import json
import os
import sys
import time

from fleetmesh_node import app as app_mod
from fleetmesh_node.app import socket as control
from .base import (
    _ask,
    _live,
    _peer,
    _status_file,
    _table,
    add_command,
    emit,
    fail,
    load_json_arg,
    status_health,
)


def _strain(st: dict) -> None:
    if not st or not st.get('kinds'):
        print('nothing survived yet. This node has not been stressed, which is')
        print('not the same as being strong: a tree raised without wind grows')
        print('tall and slender and cannot stand.')
        return
    print('survived:')
    for k, n in sorted(st['kinds'].items(), key=lambda kv: -kv[1]):
        print('  %-12s %d' % (k, n))
    print('\n%d directions, %.2f weighted, reserves at %.2fx'
          % (st.get('directions', 0), st.get('weighted_directions', 0),
             st.get('reserve_multiple', 1)))
    print('one peer moves this node by at most one direction, however hard it pushes.')
    if st.get('flag_tree'):
        print('\n!! flag tree: %.0f%% of it is one direction, %s'
              % (100 * st.get('dominant_share', 0),
                 ' / '.join(x for x in st['flag_tree']['direction'] if x)[:40]))
        print('   %s' % st['flag_tree']['note'])
    print('\nnone of this is a finding, and none of it crosses the wire.')


[docs]def cmd_strain(a: argparse.Namespace) -> int: """What this node has survived, and what it holds because of it.""" st = _live(a, 'strain') if st is None: st = status_health(a.dir).get('strain') or {} return emit(a, st, _strain)
def _subsystems(subs: dict) -> None: if not subs: print('the node has not run yet') return for name in sorted(subs): st = dict(subs[name]) ref = st.pop('refusals', {}) or {} print('%s' % name) for k in sorted(st): v = st[k] if isinstance(v, (dict, list)) and not v: continue print(' %-22s %s' % (k, json.dumps(v) if isinstance(v, (dict, list)) else v)) if ref.get('total'): print(' %-22s %d' % ('refused', ref['total'])) for why, n in sorted((ref.get('by_reason') or {}).items(), key=lambda kv: -kv[1]): print(' %6d %s' % (n, why)) print()
[docs]def cmd_subsystems(a: argparse.Namespace) -> int: """Every compartment of this node, side by side.""" subs = _live(a, 'subsystems') or status_health(a.dir).get('subsystems') or {} return emit(a, subs, _subsystems)
def _render_event(e: dict) -> str: data = e.get('data') or {} short = ' '.join('%s=%s' % (k, (v[:12] + '…') if isinstance(v, str) and len(v) > 24 else v) for k, v in data.items()) return '%s %-20s %s' % (time.strftime('%H:%M:%S', time.gmtime(e.get('ts') or 0)), e.get('topic'), short) def _events(out: dict) -> None: print('seq %s, ring holds %s (oldest %s), %s subscriber(s); topics: %s' % (out['seq'], out['ring'], out['oldest'], out['subscribers'], ' '.join(out['topics']))) for e in out['events']: print(' %6s %s' % (e['seq'], _render_event(e)))
[docs]def cmd_events(a: argparse.Namespace) -> int: """What the node's bus carried lately, from memory; needs the daemon.""" out = _live(a, 'events', {'since': a.since, 'topics': a.topics or None, 'last': a.last}) if out is None: fail('events needs a running node; is one up in %s?' % a.dir) return emit(a, out, _events)
[docs]def cmd_subscribe(a: argparse.Namespace) -> int: """Follow the node's bus live: one line per event until interrupted.""" args = {} if a.topics: args['topics'] = a.topics if a.since is not None: args['since'] = a.since try: it = control.stream(a.dir, 'subscribe', timeout=None, **args) ack = next(it) if not a.json: print('subscribed at seq %s%s' % (ack['seq'], (', replaying %d' % ack['replaying']) if ack.get('replaying') else ''), file=sys.stderr) for e in it: print(json.dumps(e) if a.json else _render_event(e), flush=True) except control.ControlError as e: fail(str(e)) except KeyboardInterrupt: return 0 return 0
def _refusals(rows: list) -> None: if not rows: print('this node has refused nothing it recorded.') return print('%-10s %-8s %-12s %-16s %s' % ('when', 'from', 'kind', 'subject', 'why')) for r in rows: print('%-10s %-8s %-12s %-16s %s' % ( time.strftime('%H:%M:%S', time.gmtime(r['at'])), r['subsystem'][:8], r['kind'][:12], (r.get('subject') or '-')[:16], r['why'])) if r.get('detail'): print('%-10s %s' % ('', json.dumps(r['detail'])[:100]))
[docs]def cmd_refusals(a: argparse.Namespace) -> int: """Every no this node said, in one place.""" rows = _live(a, 'refusals', {'last': a.last}) if rows is None: subs = status_health(a.dir).get('subsystems') or {} rows = [] for name, sub in subs.items(): ref = (sub or {}).get('refusals') or {} for r in ref.get('recent') or []: row = dict(r) row['subsystem'] = name rows.append(row) rows.sort(key=lambda r: r.get('at', 0)) if a.last: rows = rows[-int(a.last):] return emit(a, rows, _refusals)
def _capacity(c: dict) -> None: if not c: print('the node has not run yet, so it has carried nothing') return if not c.get('declared'): print('no aggregate ceiling is declared.') print('Every grant bounds one peer and nothing bounds this node, so it will') print('accept load until it fails. What it has carried in one window:') for k, v in sorted((c.get('peak') or c.get('measured') or {}).items()): print(' %-12s %s' % (k, v)) print('\nPut a total in config.json under "capacity" to turn the shed ladder on.') print('The number is yours: a class declares connectivity and not capability,') print('so nothing in the protocol can guess what this machine will carry.') return print('pressure %.0f%%%s' % (100 * (c.get('pressure') or 0), (' on %s' % c['binding']) if c.get('binding') else '')) print('%-12s %10s %10s %10s' % ('counter', 'declared', 'this window', 'peak')) for k in sorted(set(list(c.get('total') or {}) + list(c.get('peak') or {}))): print('%-12s %10s %10s %10s' % (k, (c.get('total') or {}).get(k, '-'), (c.get('measured') or {}).get(k, 0), (c.get('peak') or {}).get(k, 0))) if c.get('held'): print('\nshedding, in ladder order:') for action in c['held']: since = (c.get('since') or {}).get(action) print(' %-16s%s' % (action, (' since %s UTC' % time.strftime('%H:%M', time.gmtime(since))) if since else '')) print('none of this is a finding about any peer.') else: print('\nnothing is being shed.')
[docs]def cmd_capacity(a: argparse.Namespace) -> int: """The node's own ceiling: what it promised its peers against what the machine was told it can do.""" c = _live(a, 'capacity') if c is None: c = status_health(a.dir).get('capacity') or {} return emit(a, c, _capacity)
def _scheduled(s: dict) -> None: pulls = s.get('pulls') or [] if not pulls: print('no pulls scheduled') else: print('%-16s %-16s %-12s %s' % ('peer', 'shard', 'every', 'last run')) for p in pulls: last = ('%ds ago' % (int(time.time()) - p['last'])) if p.get('last') else 'never' print('%-16s %-16s %-12s %s' % (p['peer'], p['shard'], '%ds' % p.get('every', 900), last)) if s.get('throttled'): print('\nthrottled (rate-limited by peer grant):') for t in s['throttled']: until = time.strftime('%H:%M:%S UTC', time.gmtime(t['until'])) if t.get('until') else 'unknown' print(' %-16s %-16s %s (until %s)' % (t.get('peer', '')[:16], t.get('shard', ''), t.get('why', ''), until)) if s.get('held'): print('\nheld (under local capacity ceiling):') for h in s['held']: print(' %s' % h) if s.get('declined_until'): print('\ndeclined (waiting out interval):') for ps, until in s['declined_until'].items(): t_str = time.strftime('%H:%M:%S UTC', time.gmtime(until)) print(' %-32s until %s' % (ps, t_str))
[docs]def cmd_scheduled(a: argparse.Namespace) -> int: """Every scheduled pull, when it last ran, and what is holding it back.""" s = _live(a, 'scheduled') if s is None: cfg = app_mod.load_config(a.dir) st = _status_file(a.dir) health = st.get('health') or {} subs = health.get('subsystems') or {} sched_sub = subs.get('scheduler') or {} s = { 'pulls': [ { 'peer': (p.get('pubkey') or p.get('peer') or '')[:16], 'shard': p.get('shard', ''), 'every': p.get('pull_every', 900), 'last': None, } for p in cfg.get('peers') or [] ], 'throttled': sched_sub.get('throttled') or health.get('throttled') or [], 'held': sched_sub.get('held') or [], 'declined_until': sched_sub.get('declined_until') or {}, } return emit(a, s, _scheduled)
def _coverage(cov: list) -> None: if not cov: print('no shards claimed') return _table(('shard', 'completeness', 'filter'), [(c['shard'], c.get('completeness', 'best-effort'), json.dumps(c['filter'])) for c in cov]) print('\n`asserted` is a request: it is honoured only for a shard every event of which arrived at the relay face') from ..config.parameter import Parameter from .adapter import CommandDispatcher PARAM_SHARD = Parameter(key='shard', name='shard', description='shard name', type=str, default='', positional=True, required=False, nargs='?') PARAM_FILTER = Parameter(key='filter', name='filter', description='a nostr filter, JSON or @file (add)', type=str, default='') PARAM_ASSERTED = Parameter(key='asserted', name='asserted', description='claim completeness (honoured only if every event arrived directly)', type=bool, default=False) def _coverage_list(a: argparse.Namespace) -> int: cfg = app_mod.load_config(a.dir) cov = list(cfg.get('coverage') or []) return emit(a, cov, _coverage) def _coverage_remove(a: argparse.Namespace) -> int: cfg = dict(app_mod.load_config(a.dir)) cov = list(cfg.get('coverage') or []) if not a.shard: fail('remove needs a SHARD') kept = [c for c in cov if c['shard'] != a.shard] if len(kept) == len(cov): fail('%s is not claimed' % a.shard) cfg['coverage'] = kept app_mod.save_config(a.dir, cfg) print('no longer claiming %s; the claim on the relays expires on its own' % a.shard) return 0 def _coverage_add(a: argparse.Namespace) -> int: cfg = dict(app_mod.load_config(a.dir)) cov = list(cfg.get('coverage') or []) if not a.shard: fail('add needs a SHARD') if not a.filter: fail('add needs --filter') filt = load_json_arg(a.filter, '--filter') entry = {'shard': a.shard, 'filter': filt, 'completeness': 'asserted' if a.asserted else 'best-effort'} cfg['coverage'] = [c for c in cov if c['shard'] != a.shard] + [entry] problems = app_mod.validate_config(cfg) if problems: fail('\n'.join(problems)) app_mod.save_config(a.dir, cfg) print('claiming %s as %s on the next %s' % (a.shard, entry['completeness'], 'tick' if control.running(a.dir) else 'run')) return 0 COVERAGE_DISPATCHER = CommandDispatcher('coverage', default_action='list') COVERAGE_DISPATCHER.add_custom('list', _coverage_list) COVERAGE_DISPATCHER.add_custom('add', _coverage_add, parameters=(PARAM_SHARD, PARAM_FILTER, PARAM_ASSERTED)) COVERAGE_DISPATCHER.add_custom('remove', _coverage_remove, parameters=(PARAM_SHARD,))
[docs]def cmd_coverage(a: argparse.Namespace) -> int: return COVERAGE_DISPATCHER.dispatch(a)
[docs]def cmd_descriptor(a: argparse.Namespace) -> int: dpath = os.path.join(a.dir, app_mod.NodeApp.DESCRIPTOR_FILE) if not os.path.exists(dpath): fail('no descriptor.json yet; it is mined on the first run') with open(dpath, encoding='utf-8') as _fh: d = json.load(_fh) if a.republish: r = _ask(a.dir, 'republish', timeout=60) def human(r: dict) -> None: for url, (okay, note) in (r.get('relays') or {}).items(): print(' %-8s %s %s' % ('sent' if okay else 'FAILED', url, note)) return emit(a, r, human) # The descriptor is JSON whether or not --json was asked for. print(json.dumps(d, indent=2)) return 0
[docs]def cmd_pull(a: argparse.Namespace) -> int: pk = _peer(a.dir, a.peer) filt = None if a.filter: filt = load_json_arg(a.filter, '--filter') else: for p in app_mod.load_config(a.dir).get('peers', []): if p['pubkey'] == pk and p['shard'] == a.shard: filt = p['filter'] if filt is None: fail('no filter for shard %s with this peer; give --filter' % a.shard) # The daemon's own pull deadline is sent as a request argument and the # client waits a little longer than it, so the node's answer (done, or # timed out by name) always arrives before this end gives up. r = _ask(a.dir, 'pull', peer=pk, shard=a.shard, filter=filt, force=a.force, timeout=a.timeout + 10, request={'timeout': a.timeout}) def human(r: dict) -> None: c = r.get('counters') or {} print('%s: %s events in, %s ranges, %s bytes in, dup ratio %s' % (r.get('status'), c.get('events_in', 0), c.get('sync_ranges', 0), c.get('bytes_in', 0), r.get('dup_ratio'))) for sev, why in r.get('findings') or []: print(' finding %s: %s' % (sev, why)) return emit(a, r, human)
[docs]class SyncCommandProvider: """Registers sync, shard coverage, capacity, strain, and pull commands."""
[docs] @classmethod def register(cls, sub: argparse._SubParsersAction) -> None: add_command(sub, 'strain', cmd_strain, "what this node has survived, by direction, and the reserves it sets", js=True) add_command(sub, 'subsystems', cmd_subsystems, 'every compartment of this node, and what each has refused', js=True) rf = add_command(sub, 'refusals', cmd_refusals, 'every no this node said, across every compartment', js=True) rf.add_argument('--last', type=int, default=5, help='how many per compartment') ev = add_command(sub, 'events', cmd_events, "the node's recent bus events, from memory", js=True) ev.add_argument('--since', type=int, default=0, help='only events after this seq') ev.add_argument('--topics', nargs='*', help='keep only these topics') ev.add_argument('--last', type=int, default=50, help='at most this many, newest') sb = add_command(sub, 'subscribe', cmd_subscribe, "follow the node's bus live, one line per event", js=True) sb.add_argument('--since', type=int, help='replay what the node still holds after this seq first') sb.add_argument('--topics', nargs='*', help='keep only these topics') add_command(sub, 'capacity', cmd_capacity, "this node's own ceiling, the pressure on it, and what it is shedding", js=True) add_command(sub, 'scheduled', cmd_scheduled, 'every scheduled pull, throttles and holds', js=True) c = add_command(sub, 'coverage', cmd_coverage, 'the shards this node claims to hold, as kind 30803', js=True) COVERAGE_DISPATCHER.register_arguments(c, action_choices=('list', 'add', 'remove')) d = add_command(sub, 'descriptor', cmd_descriptor, 'the kind 11801 this node published', js=True) d.add_argument('--republish', action='store_true', help='send it to the relays again (needs the node running)') pl = add_command(sub, 'pull', cmd_pull, 'pull a shard from a peer now (needs the node running)', peer=True, js=True) pl.add_argument('shard') pl.add_argument('--filter', help='a nostr filter, JSON or @file; default: the one configured for this peer and shard') pl.add_argument('--force', action='store_true', help='pull even when the grant does not cover a session now') pl.add_argument('--timeout', type=float, default=60.0, help='seconds the node gives the session (default 60, max 600)')