"""One declaration per setting, and one rule for which layer wins.
A `Block` states a compartment's config as a frozen dataclass. That works, but
each setting's knowledge ends up in four places: the docstring says what it
means, the field says its type and default, `check` says what is wrong with a
bad value, and `from_dict` repeats the default while coercing. `UpdatesConfig`
wrote `DEFAULT_CHECK_EVERY_SECONDS` twice for exactly that reason. Four places
agreeing by hand is how a control comes to report something the node does not
do, which is a defect this tree has now met several times.
A `Parameter` is that knowledge said once. `ParamBlock` derives the other
three from it, so a compartment declares and does not re-implement.
It also buys what no amount of dataclass fields could: the node can be *asked*
what it is configurable by. `every_parameter()` enumerates the lot with types,
defaults, tags and whether a reload applies them, which is what makes uniform
defaults something you can check rather than hope for.
## Escalation
Highest wins, and the order is stated here rather than per call site - before
this, exactly one setting in the node had a documented precedence
(`app/config.py:_blob_storage_cap`, env then file then manifest), so an
operator had nothing general to rely on. That chain is preserved: FILE beats
MANIFEST because a manifest is compiled *into* the file by
`config_from_manifests`, so the file is the later word.
OVERRIDE set live over the control socket; only for `live` parameters
ARG a command line argument
ENV an environment variable, when the parameter names one
FILE config.json, which is the operator's own file
MANIFEST compiled from a meshctl manifest
DEFAULT the parameter's own default
## Field names
The design was given in a schema; these are its Python spellings.
key key
name name
description description
tags tags
type type (plus `item` for what a list or mapping holds)
valueRequired required
defaultValue default
optionalValues choices - the permitted set, named for what it is
verifyRegex verify_regex
liveToggle live
Two things were added. `minimum` and `maximum`, because a bound is part of
what most of this tree's integers are - `non_negative_int` and `positive_int`
are among base.py's most used predicates, and a `must` clause promising
"non-negative" with nothing enforcing it is a sentence an operator never gets
to read. And `must` itself, which is not a duplicate of `description`:
the description is what the setting means, `must` is what a wrong value is
told. "seconds between checks; 0 = only on command" against "be a
non-negative integer of seconds". Deriving the second from the type would
produce worse English than the sentences operators already read.
"""
from __future__ import annotations
import dataclasses
import re
from typing import Any, Callable, Generic, Mapping, Optional, Sequence, Type, TypedDict, TypeVar, Union
from .base import Block, Problems
T = TypeVar('T')
# Precedence, highest first. The index is the precedence, so a layer added
# later must be inserted where it belongs rather than appended.
OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT = 'override', 'arg', 'env', 'file', 'manifest', 'default'
SOURCES: tuple[str, ...] = (OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT)
# Which half of a parameter's access statement the node acts on today.
#
# `sources` is enforced: `resolve` skips a layer the parameter does not
# permit and `check` refuses such a value by name.
#
# `requires` is DECLARED ONLY. Nothing authenticates a principal yet - the
# control socket is chmod 0o600 and all or nothing, with no peer credential
# read and no token - so a parameter naming an ability is stating an intent,
# not defending itself. Shipping it as though it were a defence would be a
# control reporting a protection that does not exist, which is worse than no
# field at all because it reads as security. Same footing, and for the same
# reason, as `BaseStore.DURABILITY`. `every_parameter` reports which is which
# so the two cannot be confused, and enforcement arrives with an
# authenticated principal on the socket.
ACCESS_ENFORCED: tuple[str, ...] = ('sources',)
ACCESS_DECLARED_ONLY: tuple[str, ...] = ('requires',)
class _Stated(TypedDict):
"""The part of a parameter that is always reported."""
key: str
name: str
description: str
type: str
default: Any
required: bool
live: bool
tags: list[str]
[docs]class ParameterRecord(_Stated, total=False):
"""A parameter as `meshnode config` and the docs read it.
Named records rather than loose mappings, for the reason check-dicts.py
enforces it: a row an operator reads by field name is the case that rule
exists for. The optional half is present only when the parameter states
it, so a report does not show an empty pattern or a bound that is not
there.
"""
choices: list[Any]
verify_regex: str
minimum: Union[int, float]
maximum: Union[int, float]
env: str
secret: bool
requires: str
requires_enforced: bool
sources: list[str]
resource: str
short: str
cli_name: str
positional: bool
nargs: str
[docs]class ParameterRow(ParameterRecord, total=False):
"""A `ParameterRecord` with the block it belongs to, for the flat list."""
block: str
[docs]class BlockRecord(TypedDict):
"""One block, and every parameter it declares."""
key: str
label: str
parameters: list[ParameterRecord]
[docs]@dataclasses.dataclass(frozen=True)
class Parameter(Generic[T]):
"""Everything true about one setting, in one place.
`validate` is the escape hatch, and it is deliberate. A regex and a set of
choices describe a scalar or a list of scalars completely; they cannot
describe `updates.policy`, which is a mapping whose *values* are an enum
and which has a special sentence for a policy that does not exist yet. A
parameter that cannot state its own rule says so by carrying the callable
that can, and stays the single declaration either way - which is the point,
because a setting this file does not know about is a setting
`every_parameter()` cannot show an operator.
"""
key: str
name: str
description: str
# `typing.Type` rather than the builtin: the field below is named `type`,
# which shadows the builtin everywhere else in this class body, so an
# annotation written `type[...]` would resolve to the field. This keeps
# the schema's own name for it.
type: Type[Any]
# A value, or a factory for one. A mutable default needs the factory, and
# `release_key` derives its default from the running release. `value()`
# resolves either, so nothing else has to know which it is.
default: Union[T, Callable[[], T]]
must: str = ''
tags: tuple[str, ...] = ()
required: bool = False
choices: tuple[Any, ...] = ()
verify_regex: str = ''
live: bool = False
item: Optional[Type[Any]] = None
# Access. `requires` is the UCAN ability a delegation must carry to set
# this - UCAN's `can`, a verb and not a role, so a role registry no mesh
# can agree on is never needed. Empty means operator only and is the
# default on purpose: remote settability is opted into per parameter and
# is never inherited, so a parameter added without thinking about access
# is closed rather than open.
# The dataclass field this lands on, when it is not the file's own key.
# The MeshNode manifest writes `maxBytes` and `allowPrivateMirror`; the
# block holds `max_bytes` and `allow_private_mirror`. The file's spelling
# is the key, because that is what an operator types.
field: str = ''
# A credential, so a report must not print it. `settlement.nwc_uri` is a
# wallet connection string. The lease refusal in fleetmesh_node/storage/base.py put
# a database password in an operator's log earlier today for want of
# exactly this distinction, so it is stated rather than remembered.
secret: bool = False
requires: str = ''
# Which escalation layers may set it. Empty means all of them. This is
# the half an ability string cannot state: the six layers are six
# different principals, so "host only, never over the wire" is said here
# and nowhere else.
sources: tuple[str, ...] = ()
minimum: Optional[Union[int, float]] = None
maximum: Optional[Union[int, float]] = None
env: str = ''
validate: Optional[Callable[[str, Any, Problems], None]] = None
coerce: Optional[Callable[[Any], Any]] = None
short: str = ''
cli_name: str = ''
positional: bool = False
nargs: str = ''
cli: bool = True
@property
def attr(self) -> str:
"""The dataclass field name: the key unless an alias was given."""
return self.field or self.key
[docs] def value(self) -> Any:
"""The default, with a factory called. One place knows which it is."""
d: Any = self.default
return d() if callable(d) else d
[docs] def resource(self, label: str, node_did: str = '') -> str:
"""The UCAN resource this parameter is, as `with` would name it.
Derived rather than declared: 151 hand-written resource strings would
be 151 chances to name the wrong one, and the name is already implied
by the block and the key.
"""
path = 'config/%s.%s' % (label, self.key)
return '%s#%s' % (node_did, path) if node_did else path
[docs] def settable_by(self, source: str) -> bool:
"""Whether `source` is a layer allowed to set this parameter.
A data rule, decided by what the parameter declares about layers -
not by who is asking, which is a question this class must never
answer. An empty `sources` permits every layer, and DEFAULT is always
permitted because it is not a layer anyone sets.
"""
if source == DEFAULT or not self.sources:
return True
return source in self.sources
[docs] def sentence(self, label: str) -> str:
"""What an operator is told when the value is wrong."""
return '%s.%s must %s' % (label, self.key, self.must)
[docs] def ok(self, value: Any) -> bool:
"""Whether `value` satisfies what this parameter states about itself.
Only the stated things: type, the element type, the permitted set and
the regex. A parameter carrying `validate` is not judged here.
"""
if self.type is bool:
# bool before int, because bool is a subclass of int and `true`
# is not an acceptable integer anywhere in this file.
if not isinstance(value, bool):
return False
elif self.type is int:
if not isinstance(value, int) or isinstance(value, bool):
return False
# The bound is part of the type for most of this tree's integers:
# `non_negative_int` and `positive_int` are two of base.py's most
# used predicates, and a `must` promising "non-negative" that
# nothing enforces is a sentence an operator never sees.
if self.minimum is not None and value < self.minimum:
return False
if self.maximum is not None and value > self.maximum:
return False
elif self.type in (tuple, list):
if not isinstance(value, (list, tuple)):
return False
if self.item is not None and not all(_is(v, self.item) for v in value):
return False
if self.choices and not all(v in self.choices for v in value):
return False
if self.verify_regex and not all(
isinstance(v, str) and re.fullmatch(self.verify_regex, v) for v in value):
return False
return True
elif self.type is dict:
if not isinstance(value, Mapping):
return False
elif not isinstance(value, self.type):
return False
if self.choices and value not in self.choices:
return False
if self.verify_regex:
if not isinstance(value, str) or not re.fullmatch(self.verify_regex, value):
return False
return True
[docs] def read(self, raw: Mapping[str, Any]) -> Any:
"""This parameter's value out of a raw config object.
An absent key and an explicit `null` both mean "not said", so both
take the default. `coerce` is what turns what the file is allowed to
say into what the compartment reads - a pubkey lowercased, a list
frozen to a tuple.
"""
v = raw.get(self.key) if isinstance(raw, Mapping) else None
if v is None:
v = self.value()
elif self.coerce is not None:
v = self.coerce(v)
elif self.type is tuple and isinstance(v, (list, tuple)):
v = tuple(v)
return v
[docs] def describe(self) -> ParameterRecord:
"""The parameter as data, for `meshnode config` and for the docs.
`default` goes through the factory when it is one, so what is shown is
a value an operator could paste rather than the word 'function'.
"""
# A secret's default is not shown. The point of a report is that an
# operator can read it, and a credential printed into a terminal, a
# log or a paste is a credential disclosed.
shown: Any = '(secret)' if self.secret and self.value() else _jsonable(self.value())
out: ParameterRecord = {
'key': self.key, 'name': self.name, 'description': self.description,
'type': _type_name(self.type, self.item), 'default': shown,
'required': self.required, 'live': self.live, 'tags': list(self.tags),
}
if self.secret:
out['secret'] = True
if self.choices:
out['choices'] = [_jsonable(c) for c in self.choices]
if self.verify_regex:
out['verify_regex'] = self.verify_regex
if self.minimum is not None:
out['minimum'] = self.minimum
if self.maximum is not None:
out['maximum'] = self.maximum
if self.env:
out['env'] = self.env
if self.requires:
out['requires'] = self.requires
# Stated on the row, not only in a footer, because a row is what
# an operator reads and a declared ability must not look enforced.
out['requires_enforced'] = False
if self.sources:
out['sources'] = list(self.sources)
if self.short:
out['short'] = self.short
if self.cli_name:
out['cli_name'] = self.cli_name
if self.positional:
out['positional'] = True
if self.nargs:
out['nargs'] = self.nargs
return out
def _is(v: Any, t: Type[Any]) -> bool:
if t is int:
return isinstance(v, int) and not isinstance(v, bool)
return isinstance(v, t)
def _type_name(t: Type[Any], item: Optional[Type[Any]]) -> str:
base = {int: 'int', str: 'str', bool: 'bool', dict: 'object',
tuple: 'list', list: 'list'}.get(t, getattr(t, '__name__', str(t)))
if item is not None and base == 'list':
return 'list[%s]' % _type_name(item, None)
return base
def _jsonable(v: Any) -> Any:
if isinstance(v, tuple):
return [_jsonable(x) for x in v]
if isinstance(v, Mapping):
return {k: _jsonable(x) for k, x in v.items()}
return v
[docs]@dataclasses.dataclass(frozen=True)
class Arg(Generic[T]):
"""A resolved value, and where it came from.
`source` is not in the original schema and is the one thing added to it.
The purpose of this system is that an operator can see how the node is
configured, and "900" does not answer "why 900" - "900, from config.json"
does, and distinguishes a deliberate setting from a default nobody chose.
"""
key: str
type: Type[Any]
value: T
source: str = DEFAULT
@property
def is_default(self) -> bool:
return self.source == DEFAULT
[docs]def resolve(param: Parameter[T], layers: Mapping[str, Any]) -> Arg[T]:
"""The winning value for `param`, by the order at the top of this file.
A layer that does not mention the parameter is skipped, and so is an
explicit `null` - a file saying `null` is declining to set it, not setting
it to nothing. OVERRIDE is ignored for a parameter that is not `live`,
because applying one would be a change the node cannot honour until it
restarts, reported as though it had taken effect.
A layer the parameter does not permit is skipped too. That is enforcement,
and it is enforceable here because it reads only what the parameter
declares: no identity is involved, so no security decision is being taken
inside a class whose job is to describe data. `ParamBlock.check` refuses
such a value by name as well, because a setting silently doing nothing is
the failure this whole mechanism exists to remove.
"""
for source in SOURCES:
if source == DEFAULT:
break
if source == OVERRIDE and not param.live:
continue
if not param.settable_by(source):
continue
layer = layers.get(source)
if not isinstance(layer, Mapping) or layer.get(param.key) is None:
continue
v = layer[param.key]
return Arg(param.key, param.type, param.coerce(v) if param.coerce else v, source)
return Arg(param.key, param.type, param.value(), DEFAULT)
[docs]class ParamBlock(Block):
"""A `Block` whose `check` and `from_dict` come from its `PARAMS`.
A subclass declares `PARAMS` and the dataclass fields that hold them, and
writes neither method. Sentences come out in `PARAMS` order, which is why
the order is the order an operator reads in the file.
`SWITCHABLE` keeps the shorthand an optional compartment already has: the
block may be plain `true`, meaning on with every default.
"""
PARAMS: tuple[Parameter[Any], ...] = ()
SWITCHABLE: bool = False
# The parameter that is true because the block is present at all.
#
# `p2p` and `lan` both declare `enabled: bool = False` and mean opposite
# things by it: p2p is on as soon as its key appears, lan is not. That
# distinction lived only in the body of `from_dict` - two blocks that
# look identical behaving differently - and a conversion that trusted the
# dataclass default would have silently turned the libp2p transport off
# on every node configuring it as an object. Said here instead.
PRESENCE_ENABLES: str = ''
# False when PARAMS are a declaration for the census and the methods stay
# hand-written, because the raw form is not an object of named settings:
# `capacity` IS the mapping, and carries no field names in the file at
# all. Stated rather than inferred, so a block that merely has not been
# converted cannot pass for a deliberate exception.
DERIVED: bool = True
[docs] @classmethod
def check(cls, raw: Any, p: Problems) -> None:
label = cls.label or cls.key
if cls.SWITCHABLE:
if isinstance(raw, bool):
return
if not isinstance(raw, Mapping):
p.add('%s must be true, false or an object' % label)
return
elif not isinstance(raw, Mapping):
p.add('%s must be an object' % label)
return
for param in cls.PARAMS:
# config.json is the FILE layer. A parameter that does not permit
# it is refused by name rather than ignored: an operator who
# writes a setting that silently does nothing is exactly the
# failure this mechanism removes.
if (param.key in raw and raw[param.key] is not None
and not param.settable_by(FILE)):
p.add('%s.%s cannot be set in config.json; it is settable by %s'
% (label, param.key, ', '.join(param.sources)))
continue
if param.validate is not None:
param.validate(label, raw, p)
continue
if param.key not in raw or raw[param.key] is None:
if param.required:
p.add('%s.%s is required' % (label, param.key))
continue
if not param.ok(raw[param.key]):
p.add(param.sentence(label))
[docs] @classmethod
def from_dict(cls, raw: Any) -> Any:
if cls.SWITCHABLE and isinstance(raw, bool):
# `true` is on with every default. `false` is the block's own
# default, which for a presence-enabled block is off.
if raw and cls.PRESENCE_ENABLES:
return cls(**{cls.PRESENCE_ENABLES: True})
return cls()
src: Mapping[str, Any] = raw if isinstance(raw, Mapping) else {}
vals = {p.attr: p.read(src) for p in cls.PARAMS}
if cls.PRESENCE_ENABLES and src.get(cls.PRESENCE_ENABLES) is None:
# Reached only because the key is in the file, so the block is on
# unless the operator wrote it off explicitly.
vals[cls.PRESENCE_ENABLES] = True
return cls(**vals)
[docs] @classmethod
def parameters(cls) -> tuple[Parameter[Any], ...]:
return cls.PARAMS
[docs] @classmethod
def describe(cls) -> BlockRecord:
"""This block as data: its key and every parameter it declares."""
return {'key': cls.key, 'label': cls.label or cls.key,
'parameters': [p.describe() for p in cls.PARAMS]}
[docs]def declared(block_cls: type) -> tuple[Parameter[Any], ...]:
"""The parameters a block declares, or () for one not yet converted.
Returns () rather than raising so the two kinds of block can coexist while
the tree is converted one compartment at a time, and so a report can say
honestly which blocks it cannot describe yet.
"""
return tuple(getattr(block_cls, 'PARAMS', ()) or ())
[docs]def every_parameter(blocks: Sequence[type]) -> list[ParameterRow]:
"""Every declared parameter across `blocks`, as flat rows.
This is the visibility the system exists for: one list, with the block it
belongs to, its type, its default and whether a reload applies it.
"""
out: list[ParameterRow] = []
for b in blocks:
for param in declared(b):
row: ParameterRow = {**param.describe(),
'block': getattr(b, 'key', '') or getattr(b, '__name__', '')}
out.append(row)
return out
[docs]def converted(block_cls: type) -> bool:
"""Whether this block states its settings as parameters.
Asked of the class, not of its PARAMS: `mediation` and `transport` are
reserved and have no settings at all, so an empty PARAMS is their correct
and final answer. Counting them as outstanding would never reach zero.
"""
return isinstance(block_cls, type) and issubclass(block_cls, ParamBlock)
[docs]def undeclared(blocks: Sequence[type]) -> list[str]:
"""Blocks not yet stating their settings as parameters."""
return [getattr(b, 'key', '') or b.__name__ for b in blocks if not converted(b)]
[docs]def hand_written(blocks: Sequence[type]) -> list[str]:
"""Blocks that declare parameters but keep their own check and from_dict.
Separate from `undeclared` because they mean different things: this list
is complete for the census and deliberately not derived, that one is work
outstanding.
"""
return [getattr(b, 'key', '') or b.__name__ for b in blocks
if declared(b) and not getattr(b, 'DERIVED', True)]
__all__ = ['Parameter', 'ParameterRecord', 'ParameterRow', 'BlockRecord', 'Arg', 'ParamBlock',
'resolve', 'declared', 'every_parameter', 'undeclared', 'SOURCES', 'OVERRIDE', 'ARG',
'ENV', 'FILE', 'MANIFEST', 'DEFAULT', 'ACCESS_ENFORCED', 'ACCESS_DECLARED_ONLY',
'hand_written', 'converted']