Source code for fleetmesh_node.config.parameter

"""One declaration per setting, and one rule for which layer wins.

A `Block` states a compartment's config as a frozen dataclass. That works, but
each setting's knowledge ends up in four places: the docstring says what it
means, the field says its type and default, `check` says what is wrong with a
bad value, and `from_dict` repeats the default while coercing. `UpdatesConfig`
wrote `DEFAULT_CHECK_EVERY_SECONDS` twice for exactly that reason. Four places
agreeing by hand is how a control comes to report something the node does not
do, which is a defect this tree has now met several times.

A `Parameter` is that knowledge said once. `ParamBlock` derives the other
three from it, so a compartment declares and does not re-implement.

It also buys what no amount of dataclass fields could: the node can be *asked*
what it is configurable by. `every_parameter()` enumerates the lot with types,
defaults, tags and whether a reload applies them, which is what makes uniform
defaults something you can check rather than hope for.

## Escalation

Highest wins, and the order is stated here rather than per call site - before
this, exactly one setting in the node had a documented precedence
(`app/config.py:_blob_storage_cap`, env then file then manifest), so an
operator had nothing general to rely on. That chain is preserved: FILE beats
MANIFEST because a manifest is compiled *into* the file by
`config_from_manifests`, so the file is the later word.

    OVERRIDE   set live over the control socket; only for `live` parameters
    ARG        a command line argument
    ENV        an environment variable, when the parameter names one
    FILE       config.json, which is the operator's own file
    MANIFEST   compiled from a meshctl manifest
    DEFAULT    the parameter's own default

## Field names

The design was given in a schema; these are its Python spellings.

    key             key
    name            name
    description     description
    tags            tags
    type            type (plus `item` for what a list or mapping holds)
    valueRequired   required
    defaultValue    default
    optionalValues  choices      - the permitted set, named for what it is
    verifyRegex     verify_regex
    liveToggle      live

Two things were added. `minimum` and `maximum`, because a bound is part of
what most of this tree's integers are - `non_negative_int` and `positive_int`
are among base.py's most used predicates, and a `must` clause promising
"non-negative" with nothing enforcing it is a sentence an operator never gets
to read. And `must` itself, which is not a duplicate of `description`:
the description is what the setting means, `must` is what a wrong value is
told. "seconds between checks; 0 = only on command" against "be a
non-negative integer of seconds". Deriving the second from the type would
produce worse English than the sentences operators already read.
"""
from __future__ import annotations

import dataclasses
import re
from typing import Any, Callable, Generic, Mapping, Optional, Sequence, Type, TypedDict, TypeVar, Union

from .base import Block, Problems

T = TypeVar('T')

# Precedence, highest first. The index is the precedence, so a layer added
# later must be inserted where it belongs rather than appended.
OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT = 'override', 'arg', 'env', 'file', 'manifest', 'default'
SOURCES: tuple[str, ...] = (OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT)

# Which half of a parameter's access statement the node acts on today.
#
# `sources` is enforced: `resolve` skips a layer the parameter does not
# permit and `check` refuses such a value by name.
#
# `requires` is DECLARED ONLY. Nothing authenticates a principal yet - the
# control socket is chmod 0o600 and all or nothing, with no peer credential
# read and no token - so a parameter naming an ability is stating an intent,
# not defending itself. Shipping it as though it were a defence would be a
# control reporting a protection that does not exist, which is worse than no
# field at all because it reads as security. Same footing, and for the same
# reason, as `BaseStore.DURABILITY`. `every_parameter` reports which is which
# so the two cannot be confused, and enforcement arrives with an
# authenticated principal on the socket.
ACCESS_ENFORCED: tuple[str, ...] = ('sources',)
ACCESS_DECLARED_ONLY: tuple[str, ...] = ('requires',)


class _Stated(TypedDict):
    """The part of a parameter that is always reported."""
    key: str
    name: str
    description: str
    type: str
    default: Any
    required: bool
    live: bool
    tags: list[str]


[docs]class ParameterRecord(_Stated, total=False): """A parameter as `meshnode config` and the docs read it. Named records rather than loose mappings, for the reason check-dicts.py enforces it: a row an operator reads by field name is the case that rule exists for. The optional half is present only when the parameter states it, so a report does not show an empty pattern or a bound that is not there. """ choices: list[Any] verify_regex: str minimum: Union[int, float] maximum: Union[int, float] env: str secret: bool requires: str requires_enforced: bool sources: list[str] resource: str short: str cli_name: str positional: bool nargs: str
[docs]class ParameterRow(ParameterRecord, total=False): """A `ParameterRecord` with the block it belongs to, for the flat list.""" block: str
[docs]class BlockRecord(TypedDict): """One block, and every parameter it declares.""" key: str label: str parameters: list[ParameterRecord]
[docs]@dataclasses.dataclass(frozen=True) class Parameter(Generic[T]): """Everything true about one setting, in one place. `validate` is the escape hatch, and it is deliberate. A regex and a set of choices describe a scalar or a list of scalars completely; they cannot describe `updates.policy`, which is a mapping whose *values* are an enum and which has a special sentence for a policy that does not exist yet. A parameter that cannot state its own rule says so by carrying the callable that can, and stays the single declaration either way - which is the point, because a setting this file does not know about is a setting `every_parameter()` cannot show an operator. """ key: str name: str description: str # `typing.Type` rather than the builtin: the field below is named `type`, # which shadows the builtin everywhere else in this class body, so an # annotation written `type[...]` would resolve to the field. This keeps # the schema's own name for it. type: Type[Any] # A value, or a factory for one. A mutable default needs the factory, and # `release_key` derives its default from the running release. `value()` # resolves either, so nothing else has to know which it is. default: Union[T, Callable[[], T]] must: str = '' tags: tuple[str, ...] = () required: bool = False choices: tuple[Any, ...] = () verify_regex: str = '' live: bool = False item: Optional[Type[Any]] = None # Access. `requires` is the UCAN ability a delegation must carry to set # this - UCAN's `can`, a verb and not a role, so a role registry no mesh # can agree on is never needed. Empty means operator only and is the # default on purpose: remote settability is opted into per parameter and # is never inherited, so a parameter added without thinking about access # is closed rather than open. # The dataclass field this lands on, when it is not the file's own key. # The MeshNode manifest writes `maxBytes` and `allowPrivateMirror`; the # block holds `max_bytes` and `allow_private_mirror`. The file's spelling # is the key, because that is what an operator types. field: str = '' # A credential, so a report must not print it. `settlement.nwc_uri` is a # wallet connection string. The lease refusal in fleetmesh_node/storage/base.py put # a database password in an operator's log earlier today for want of # exactly this distinction, so it is stated rather than remembered. secret: bool = False requires: str = '' # Which escalation layers may set it. Empty means all of them. This is # the half an ability string cannot state: the six layers are six # different principals, so "host only, never over the wire" is said here # and nowhere else. sources: tuple[str, ...] = () minimum: Optional[Union[int, float]] = None maximum: Optional[Union[int, float]] = None env: str = '' validate: Optional[Callable[[str, Any, Problems], None]] = None coerce: Optional[Callable[[Any], Any]] = None short: str = '' cli_name: str = '' positional: bool = False nargs: str = '' cli: bool = True @property def attr(self) -> str: """The dataclass field name: the key unless an alias was given.""" return self.field or self.key
[docs] def value(self) -> Any: """The default, with a factory called. One place knows which it is.""" d: Any = self.default return d() if callable(d) else d
[docs] def resource(self, label: str, node_did: str = '') -> str: """The UCAN resource this parameter is, as `with` would name it. Derived rather than declared: 151 hand-written resource strings would be 151 chances to name the wrong one, and the name is already implied by the block and the key. """ path = 'config/%s.%s' % (label, self.key) return '%s#%s' % (node_did, path) if node_did else path
[docs] def settable_by(self, source: str) -> bool: """Whether `source` is a layer allowed to set this parameter. A data rule, decided by what the parameter declares about layers - not by who is asking, which is a question this class must never answer. An empty `sources` permits every layer, and DEFAULT is always permitted because it is not a layer anyone sets. """ if source == DEFAULT or not self.sources: return True return source in self.sources
[docs] def sentence(self, label: str) -> str: """What an operator is told when the value is wrong.""" return '%s.%s must %s' % (label, self.key, self.must)
[docs] def ok(self, value: Any) -> bool: """Whether `value` satisfies what this parameter states about itself. Only the stated things: type, the element type, the permitted set and the regex. A parameter carrying `validate` is not judged here. """ if self.type is bool: # bool before int, because bool is a subclass of int and `true` # is not an acceptable integer anywhere in this file. if not isinstance(value, bool): return False elif self.type is int: if not isinstance(value, int) or isinstance(value, bool): return False # The bound is part of the type for most of this tree's integers: # `non_negative_int` and `positive_int` are two of base.py's most # used predicates, and a `must` promising "non-negative" that # nothing enforces is a sentence an operator never sees. if self.minimum is not None and value < self.minimum: return False if self.maximum is not None and value > self.maximum: return False elif self.type in (tuple, list): if not isinstance(value, (list, tuple)): return False if self.item is not None and not all(_is(v, self.item) for v in value): return False if self.choices and not all(v in self.choices for v in value): return False if self.verify_regex and not all( isinstance(v, str) and re.fullmatch(self.verify_regex, v) for v in value): return False return True elif self.type is dict: if not isinstance(value, Mapping): return False elif not isinstance(value, self.type): return False if self.choices and value not in self.choices: return False if self.verify_regex: if not isinstance(value, str) or not re.fullmatch(self.verify_regex, value): return False return True
[docs] def read(self, raw: Mapping[str, Any]) -> Any: """This parameter's value out of a raw config object. An absent key and an explicit `null` both mean "not said", so both take the default. `coerce` is what turns what the file is allowed to say into what the compartment reads - a pubkey lowercased, a list frozen to a tuple. """ v = raw.get(self.key) if isinstance(raw, Mapping) else None if v is None: v = self.value() elif self.coerce is not None: v = self.coerce(v) elif self.type is tuple and isinstance(v, (list, tuple)): v = tuple(v) return v
[docs] def describe(self) -> ParameterRecord: """The parameter as data, for `meshnode config` and for the docs. `default` goes through the factory when it is one, so what is shown is a value an operator could paste rather than the word 'function'. """ # A secret's default is not shown. The point of a report is that an # operator can read it, and a credential printed into a terminal, a # log or a paste is a credential disclosed. shown: Any = '(secret)' if self.secret and self.value() else _jsonable(self.value()) out: ParameterRecord = { 'key': self.key, 'name': self.name, 'description': self.description, 'type': _type_name(self.type, self.item), 'default': shown, 'required': self.required, 'live': self.live, 'tags': list(self.tags), } if self.secret: out['secret'] = True if self.choices: out['choices'] = [_jsonable(c) for c in self.choices] if self.verify_regex: out['verify_regex'] = self.verify_regex if self.minimum is not None: out['minimum'] = self.minimum if self.maximum is not None: out['maximum'] = self.maximum if self.env: out['env'] = self.env if self.requires: out['requires'] = self.requires # Stated on the row, not only in a footer, because a row is what # an operator reads and a declared ability must not look enforced. out['requires_enforced'] = False if self.sources: out['sources'] = list(self.sources) if self.short: out['short'] = self.short if self.cli_name: out['cli_name'] = self.cli_name if self.positional: out['positional'] = True if self.nargs: out['nargs'] = self.nargs return out
def _is(v: Any, t: Type[Any]) -> bool: if t is int: return isinstance(v, int) and not isinstance(v, bool) return isinstance(v, t) def _type_name(t: Type[Any], item: Optional[Type[Any]]) -> str: base = {int: 'int', str: 'str', bool: 'bool', dict: 'object', tuple: 'list', list: 'list'}.get(t, getattr(t, '__name__', str(t))) if item is not None and base == 'list': return 'list[%s]' % _type_name(item, None) return base def _jsonable(v: Any) -> Any: if isinstance(v, tuple): return [_jsonable(x) for x in v] if isinstance(v, Mapping): return {k: _jsonable(x) for k, x in v.items()} return v
[docs]@dataclasses.dataclass(frozen=True) class Arg(Generic[T]): """A resolved value, and where it came from. `source` is not in the original schema and is the one thing added to it. The purpose of this system is that an operator can see how the node is configured, and "900" does not answer "why 900" - "900, from config.json" does, and distinguishes a deliberate setting from a default nobody chose. """ key: str type: Type[Any] value: T source: str = DEFAULT @property def is_default(self) -> bool: return self.source == DEFAULT
[docs]def resolve(param: Parameter[T], layers: Mapping[str, Any]) -> Arg[T]: """The winning value for `param`, by the order at the top of this file. A layer that does not mention the parameter is skipped, and so is an explicit `null` - a file saying `null` is declining to set it, not setting it to nothing. OVERRIDE is ignored for a parameter that is not `live`, because applying one would be a change the node cannot honour until it restarts, reported as though it had taken effect. A layer the parameter does not permit is skipped too. That is enforcement, and it is enforceable here because it reads only what the parameter declares: no identity is involved, so no security decision is being taken inside a class whose job is to describe data. `ParamBlock.check` refuses such a value by name as well, because a setting silently doing nothing is the failure this whole mechanism exists to remove. """ for source in SOURCES: if source == DEFAULT: break if source == OVERRIDE and not param.live: continue if not param.settable_by(source): continue layer = layers.get(source) if not isinstance(layer, Mapping) or layer.get(param.key) is None: continue v = layer[param.key] return Arg(param.key, param.type, param.coerce(v) if param.coerce else v, source) return Arg(param.key, param.type, param.value(), DEFAULT)
[docs]class ParamBlock(Block): """A `Block` whose `check` and `from_dict` come from its `PARAMS`. A subclass declares `PARAMS` and the dataclass fields that hold them, and writes neither method. Sentences come out in `PARAMS` order, which is why the order is the order an operator reads in the file. `SWITCHABLE` keeps the shorthand an optional compartment already has: the block may be plain `true`, meaning on with every default. """ PARAMS: tuple[Parameter[Any], ...] = () SWITCHABLE: bool = False # The parameter that is true because the block is present at all. # # `p2p` and `lan` both declare `enabled: bool = False` and mean opposite # things by it: p2p is on as soon as its key appears, lan is not. That # distinction lived only in the body of `from_dict` - two blocks that # look identical behaving differently - and a conversion that trusted the # dataclass default would have silently turned the libp2p transport off # on every node configuring it as an object. Said here instead. PRESENCE_ENABLES: str = '' # False when PARAMS are a declaration for the census and the methods stay # hand-written, because the raw form is not an object of named settings: # `capacity` IS the mapping, and carries no field names in the file at # all. Stated rather than inferred, so a block that merely has not been # converted cannot pass for a deliberate exception. DERIVED: bool = True
[docs] @classmethod def check(cls, raw: Any, p: Problems) -> None: label = cls.label or cls.key if cls.SWITCHABLE: if isinstance(raw, bool): return if not isinstance(raw, Mapping): p.add('%s must be true, false or an object' % label) return elif not isinstance(raw, Mapping): p.add('%s must be an object' % label) return for param in cls.PARAMS: # config.json is the FILE layer. A parameter that does not permit # it is refused by name rather than ignored: an operator who # writes a setting that silently does nothing is exactly the # failure this mechanism removes. if (param.key in raw and raw[param.key] is not None and not param.settable_by(FILE)): p.add('%s.%s cannot be set in config.json; it is settable by %s' % (label, param.key, ', '.join(param.sources))) continue if param.validate is not None: param.validate(label, raw, p) continue if param.key not in raw or raw[param.key] is None: if param.required: p.add('%s.%s is required' % (label, param.key)) continue if not param.ok(raw[param.key]): p.add(param.sentence(label))
[docs] @classmethod def from_dict(cls, raw: Any) -> Any: if cls.SWITCHABLE and isinstance(raw, bool): # `true` is on with every default. `false` is the block's own # default, which for a presence-enabled block is off. if raw and cls.PRESENCE_ENABLES: return cls(**{cls.PRESENCE_ENABLES: True}) return cls() src: Mapping[str, Any] = raw if isinstance(raw, Mapping) else {} vals = {p.attr: p.read(src) for p in cls.PARAMS} if cls.PRESENCE_ENABLES and src.get(cls.PRESENCE_ENABLES) is None: # Reached only because the key is in the file, so the block is on # unless the operator wrote it off explicitly. vals[cls.PRESENCE_ENABLES] = True return cls(**vals)
[docs] @classmethod def parameters(cls) -> tuple[Parameter[Any], ...]: return cls.PARAMS
[docs] @classmethod def describe(cls) -> BlockRecord: """This block as data: its key and every parameter it declares.""" return {'key': cls.key, 'label': cls.label or cls.key, 'parameters': [p.describe() for p in cls.PARAMS]}
[docs]def declared(block_cls: type) -> tuple[Parameter[Any], ...]: """The parameters a block declares, or () for one not yet converted. Returns () rather than raising so the two kinds of block can coexist while the tree is converted one compartment at a time, and so a report can say honestly which blocks it cannot describe yet. """ return tuple(getattr(block_cls, 'PARAMS', ()) or ())
[docs]def every_parameter(blocks: Sequence[type]) -> list[ParameterRow]: """Every declared parameter across `blocks`, as flat rows. This is the visibility the system exists for: one list, with the block it belongs to, its type, its default and whether a reload applies it. """ out: list[ParameterRow] = [] for b in blocks: for param in declared(b): row: ParameterRow = {**param.describe(), 'block': getattr(b, 'key', '') or getattr(b, '__name__', '')} out.append(row) return out
[docs]def converted(block_cls: type) -> bool: """Whether this block states its settings as parameters. Asked of the class, not of its PARAMS: `mediation` and `transport` are reserved and have no settings at all, so an empty PARAMS is their correct and final answer. Counting them as outstanding would never reach zero. """ return isinstance(block_cls, type) and issubclass(block_cls, ParamBlock)
[docs]def undeclared(blocks: Sequence[type]) -> list[str]: """Blocks not yet stating their settings as parameters.""" return [getattr(b, 'key', '') or b.__name__ for b in blocks if not converted(b)]
[docs]def hand_written(blocks: Sequence[type]) -> list[str]: """Blocks that declare parameters but keep their own check and from_dict. Separate from `undeclared` because they mean different things: this list is complete for the census and deliberately not derived, that one is work outstanding. """ return [getattr(b, 'key', '') or b.__name__ for b in blocks if declared(b) and not getattr(b, 'DERIVED', True)]
__all__ = ['Parameter', 'ParameterRecord', 'ParameterRow', 'BlockRecord', 'Arg', 'ParamBlock', 'resolve', 'declared', 'every_parameter', 'undeclared', 'SOURCES', 'OVERRIDE', 'ARG', 'ENV', 'FILE', 'MANIFEST', 'DEFAULT', 'ACCESS_ENFORCED', 'ACCESS_DECLARED_ONLY', 'hand_written', 'converted']