fleetmesh-node 0.2.0__py3-none-any.whl

node/config/parameter.py

@@ -0,0 +1,563 @@
+"""One declaration per setting, and one rule for which layer wins.
+
+A `Block` states a compartment's config as a frozen dataclass. That works, but
+each setting's knowledge ends up in four places: the docstring says what it
+means, the field says its type and default, `check` says what is wrong with a
+bad value, and `from_dict` repeats the default while coercing. `UpdatesConfig`
+wrote `DEFAULT_CHECK_EVERY_SECONDS` twice for exactly that reason. Four places
+agreeing by hand is how a control comes to report something the node does not
+do, which is a defect this tree has now met several times.
+
+A `Parameter` is that knowledge said once. `ParamBlock` derives the other
+three from it, so a compartment declares and does not re-implement.
+
+It also buys what no amount of dataclass fields could: the node can be *asked*
+what it is configurable by. `every_parameter()` enumerates the lot with types,
+defaults, tags and whether a reload applies them, which is what makes uniform
+defaults something you can check rather than hope for.
+
+## Escalation
+
+Highest wins, and the order is stated here rather than per call site - before
+this, exactly one setting in the node had a documented precedence
+(`app/config.py:_blob_storage_cap`, env then file then manifest), so an
+operator had nothing general to rely on. That chain is preserved: FILE beats
+MANIFEST because a manifest is compiled *into* the file by
+`config_from_manifests`, so the file is the later word.
+
+    OVERRIDE   set live over the control socket; only for `live` parameters
+    ARG        a command line argument
+    ENV        an environment variable, when the parameter names one
+    FILE       config.json, which is the operator's own file
+    MANIFEST   compiled from a meshctl manifest
+    DEFAULT    the parameter's own default
+
+## Field names
+
+The design was given in a schema; these are its Python spellings.
+
+    key             key
+    name            name
+    description     description
+    tags            tags
+    type            type (plus `item` for what a list or mapping holds)
+    valueRequired   required
+    defaultValue    default
+    optionalValues  choices      - the permitted set, named for what it is
+    verifyRegex     verify_regex
+    liveToggle      live
+
+Two things were added. `minimum` and `maximum`, because a bound is part of
+what most of this tree's integers are - `non_negative_int` and `positive_int`
+are among base.py's most used predicates, and a `must` clause promising
+"non-negative" with nothing enforcing it is a sentence an operator never gets
+to read. And `must` itself, which is not a duplicate of `description`:
+the description is what the setting means, `must` is what a wrong value is
+told. "seconds between checks; 0 = only on command" against "be a
+non-negative integer of seconds". Deriving the second from the type would
+produce worse English than the sentences operators already read.
+"""
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import Any, Callable, Generic, Mapping, Optional, Sequence, Type, TypedDict, TypeVar, Union
+
+from .base import Block, Problems
+
+T = TypeVar('T')
+
+# Precedence, highest first. The index is the precedence, so a layer added
+# later must be inserted where it belongs rather than appended.
+OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT = 'override', 'arg', 'env', 'file', 'manifest', 'default'
+SOURCES: tuple[str, ...] = (OVERRIDE, ARG, ENV, FILE, MANIFEST, DEFAULT)
+
+# Which half of a parameter's access statement the node acts on today.
+#
+# `sources` is enforced: `resolve` skips a layer the parameter does not
+# permit and `check` refuses such a value by name.
+#
+# `requires` is DECLARED ONLY. Nothing authenticates a principal yet - the
+# control socket is chmod 0o600 and all or nothing, with no peer credential
+# read and no token - so a parameter naming an ability is stating an intent,
+# not defending itself. Shipping it as though it were a defence would be a
+# control reporting a protection that does not exist, which is worse than no
+# field at all because it reads as security. Same footing, and for the same
+# reason, as `BaseStore.DURABILITY`. `every_parameter` reports which is which
+# so the two cannot be confused, and enforcement arrives with an
+# authenticated principal on the socket.
+ACCESS_ENFORCED: tuple[str, ...] = ('sources',)
+ACCESS_DECLARED_ONLY: tuple[str, ...] = ('requires',)
+
+
+class _Stated(TypedDict):
+    """The part of a parameter that is always reported."""
+    key: str
+    name: str
+    description: str
+    type: str
+    default: Any
+    required: bool
+    live: bool
+    tags: list[str]
+
+
+class ParameterRecord(_Stated, total=False):
+    """A parameter as `meshnode config` and the docs read it.
+
+    Named records rather than loose mappings, for the reason check-dicts.py
+    enforces it: a row an operator reads by field name is the case that rule
+    exists for. The optional half is present only when the parameter states
+    it, so a report does not show an empty pattern or a bound that is not
+    there.
+    """
+    choices: list[Any]
+    verify_regex: str
+    minimum: Union[int, float]
+    maximum: Union[int, float]
+    env: str
+    secret: bool
+    requires: str
+    requires_enforced: bool
+    sources: list[str]
+    resource: str
+    short: str
+    cli_name: str
+    positional: bool
+    nargs: str
+
+
+class ParameterRow(ParameterRecord, total=False):
+    """A `ParameterRecord` with the block it belongs to, for the flat list."""
+    block: str
+
+
+class BlockRecord(TypedDict):
+    """One block, and every parameter it declares."""
+    key: str
+    label: str
+    parameters: list[ParameterRecord]
+
+
+@dataclasses.dataclass(frozen=True)
+class Parameter(Generic[T]):
+    """Everything true about one setting, in one place.
+
+    `validate` is the escape hatch, and it is deliberate. A regex and a set of
+    choices describe a scalar or a list of scalars completely; they cannot
+    describe `updates.policy`, which is a mapping whose *values* are an enum
+    and which has a special sentence for a policy that does not exist yet. A
+    parameter that cannot state its own rule says so by carrying the callable
+    that can, and stays the single declaration either way - which is the point,
+    because a setting this file does not know about is a setting
+    `every_parameter()` cannot show an operator.
+    """
+    key: str
+    name: str
+    description: str
+    # `typing.Type` rather than the builtin: the field below is named `type`,
+    # which shadows the builtin everywhere else in this class body, so an
+    # annotation written `type[...]` would resolve to the field. This keeps
+    # the schema's own name for it.
+    type: Type[Any]
+    # A value, or a factory for one. A mutable default needs the factory, and
+    # `release_key` derives its default from the running release. `value()`
+    # resolves either, so nothing else has to know which it is.
+    default: Union[T, Callable[[], T]]
+    must: str = ''
+    tags: tuple[str, ...] = ()
+    required: bool = False
+    choices: tuple[Any, ...] = ()
+    verify_regex: str = ''
+    live: bool = False
+    item: Optional[Type[Any]] = None
+    # Access. `requires` is the UCAN ability a delegation must carry to set
+    # this - UCAN's `can`, a verb and not a role, so a role registry no mesh
+    # can agree on is never needed. Empty means operator only and is the
+    # default on purpose: remote settability is opted into per parameter and
+    # is never inherited, so a parameter added without thinking about access
+    # is closed rather than open.
+    # The dataclass field this lands on, when it is not the file's own key.
+    # The MeshNode manifest writes `maxBytes` and `allowPrivateMirror`; the
+    # block holds `max_bytes` and `allow_private_mirror`. The file's spelling
+    # is the key, because that is what an operator types.
+    field: str = ''
+    # A credential, so a report must not print it. `settlement.nwc_uri` is a
+    # wallet connection string. The lease refusal in node/storage/base.py put
+    # a database password in an operator's log earlier today for want of
+    # exactly this distinction, so it is stated rather than remembered.
+    secret: bool = False
+    requires: str = ''
+    # Which escalation layers may set it. Empty means all of them. This is
+    # the half an ability string cannot state: the six layers are six
+    # different principals, so "host only, never over the wire" is said here
+    # and nowhere else.
+    sources: tuple[str, ...] = ()
+    minimum: Optional[Union[int, float]] = None
+    maximum: Optional[Union[int, float]] = None
+    env: str = ''
+    validate: Optional[Callable[[str, Any, Problems], None]] = None
+    coerce: Optional[Callable[[Any], Any]] = None
+    short: str = ''
+    cli_name: str = ''
+    positional: bool = False
+    nargs: str = ''
+    cli: bool = True
+
+    @property
+    def attr(self) -> str:
+        """The dataclass field name: the key unless an alias was given."""
+        return self.field or self.key
+
+    def value(self) -> Any:
+        """The default, with a factory called. One place knows which it is."""
+        d: Any = self.default
+        return d() if callable(d) else d
+
+    def resource(self, label: str, node_did: str = '') -> str:
+        """The UCAN resource this parameter is, as `with` would name it.
+
+        Derived rather than declared: 151 hand-written resource strings would
+        be 151 chances to name the wrong one, and the name is already implied
+        by the block and the key.
+        """
+        path = 'config/%s.%s' % (label, self.key)
+        return '%s#%s' % (node_did, path) if node_did else path
+
+    def settable_by(self, source: str) -> bool:
+        """Whether `source` is a layer allowed to set this parameter.
+
+        A data rule, decided by what the parameter declares about layers -
+        not by who is asking, which is a question this class must never
+        answer. An empty `sources` permits every layer, and DEFAULT is always
+        permitted because it is not a layer anyone sets.
+        """
+        if source == DEFAULT or not self.sources:
+            return True
+        return source in self.sources
+
+    def sentence(self, label: str) -> str:
+        """What an operator is told when the value is wrong."""
+        return '%s.%s must %s' % (label, self.key, self.must)
+
+    def ok(self, value: Any) -> bool:
+        """Whether `value` satisfies what this parameter states about itself.
+
+        Only the stated things: type, the element type, the permitted set and
+        the regex. A parameter carrying `validate` is not judged here.
+        """
+        if self.type is bool:
+            # bool before int, because bool is a subclass of int and `true`
+            # is not an acceptable integer anywhere in this file.
+            if not isinstance(value, bool):
+                return False
+        elif self.type is int:
+            if not isinstance(value, int) or isinstance(value, bool):
+                return False
+            # The bound is part of the type for most of this tree's integers:
+            # `non_negative_int` and `positive_int` are two of base.py's most
+            # used predicates, and a `must` promising "non-negative" that
+            # nothing enforces is a sentence an operator never sees.
+            if self.minimum is not None and value < self.minimum:
+                return False
+            if self.maximum is not None and value > self.maximum:
+                return False
+        elif self.type in (tuple, list):
+            if not isinstance(value, (list, tuple)):
+                return False
+            if self.item is not None and not all(_is(v, self.item) for v in value):
+                return False
+            if self.choices and not all(v in self.choices for v in value):
+                return False
+            if self.verify_regex and not all(
+                    isinstance(v, str) and re.fullmatch(self.verify_regex, v) for v in value):
+                return False
+            return True
+        elif self.type is dict:
+            if not isinstance(value, Mapping):
+                return False
+        elif not isinstance(value, self.type):
+            return False
+        if self.choices and value not in self.choices:
+            return False
+        if self.verify_regex:
+            if not isinstance(value, str) or not re.fullmatch(self.verify_regex, value):
+                return False
+        return True
+
+    def read(self, raw: Mapping[str, Any]) -> Any:
+        """This parameter's value out of a raw config object.
+
+        An absent key and an explicit `null` both mean "not said", so both
+        take the default. `coerce` is what turns what the file is allowed to
+        say into what the compartment reads - a pubkey lowercased, a list
+        frozen to a tuple.
+        """
+        v = raw.get(self.key) if isinstance(raw, Mapping) else None
+        if v is None:
+            v = self.value()
+        elif self.coerce is not None:
+            v = self.coerce(v)
+        elif self.type is tuple and isinstance(v, (list, tuple)):
+            v = tuple(v)
+        return v
+
+    def describe(self) -> ParameterRecord:
+        """The parameter as data, for `meshnode config` and for the docs.
+
+        `default` goes through the factory when it is one, so what is shown is
+        a value an operator could paste rather than the word 'function'.
+        """
+        # A secret's default is not shown. The point of a report is that an
+        # operator can read it, and a credential printed into a terminal, a
+        # log or a paste is a credential disclosed.
+        shown: Any = '(secret)' if self.secret and self.value() else _jsonable(self.value())
+        out: ParameterRecord = {
+            'key': self.key, 'name': self.name, 'description': self.description,
+            'type': _type_name(self.type, self.item), 'default': shown,
+            'required': self.required, 'live': self.live, 'tags': list(self.tags),
+        }
+        if self.secret:
+            out['secret'] = True
+        if self.choices:
+            out['choices'] = [_jsonable(c) for c in self.choices]
+        if self.verify_regex:
+            out['verify_regex'] = self.verify_regex
+        if self.minimum is not None:
+            out['minimum'] = self.minimum
+        if self.maximum is not None:
+            out['maximum'] = self.maximum
+        if self.env:
+            out['env'] = self.env
+        if self.requires:
+            out['requires'] = self.requires
+            # Stated on the row, not only in a footer, because a row is what
+            # an operator reads and a declared ability must not look enforced.
+            out['requires_enforced'] = False
+        if self.sources:
+            out['sources'] = list(self.sources)
+        if self.short:
+            out['short'] = self.short
+        if self.cli_name:
+            out['cli_name'] = self.cli_name
+        if self.positional:
+            out['positional'] = True
+        if self.nargs:
+            out['nargs'] = self.nargs
+        return out
+
+
+def _is(v: Any, t: Type[Any]) -> bool:
+    if t is int:
+        return isinstance(v, int) and not isinstance(v, bool)
+    return isinstance(v, t)
+
+
+def _type_name(t: Type[Any], item: Optional[Type[Any]]) -> str:
+    base = {int: 'int', str: 'str', bool: 'bool', dict: 'object',
+            tuple: 'list', list: 'list'}.get(t, getattr(t, '__name__', str(t)))
+    if item is not None and base == 'list':
+        return 'list[%s]' % _type_name(item, None)
+    return base
+
+
+def _jsonable(v: Any) -> Any:
+    if isinstance(v, tuple):
+        return [_jsonable(x) for x in v]
+    if isinstance(v, Mapping):
+        return {k: _jsonable(x) for k, x in v.items()}
+    return v
+
+
+@dataclasses.dataclass(frozen=True)
+class Arg(Generic[T]):
+    """A resolved value, and where it came from.
+
+    `source` is not in the original schema and is the one thing added to it.
+    The purpose of this system is that an operator can see how the node is
+    configured, and "900" does not answer "why 900" - "900, from config.json"
+    does, and distinguishes a deliberate setting from a default nobody chose.
+    """
+    key: str
+    type: Type[Any]
+    value: T
+    source: str = DEFAULT
+
+    @property
+    def is_default(self) -> bool:
+        return self.source == DEFAULT
+
+
+def resolve(param: Parameter[T], layers: Mapping[str, Any]) -> Arg[T]:
+    """The winning value for `param`, by the order at the top of this file.
+
+    A layer that does not mention the parameter is skipped, and so is an
+    explicit `null` - a file saying `null` is declining to set it, not setting
+    it to nothing. OVERRIDE is ignored for a parameter that is not `live`,
+    because applying one would be a change the node cannot honour until it
+    restarts, reported as though it had taken effect.
+
+    A layer the parameter does not permit is skipped too. That is enforcement,
+    and it is enforceable here because it reads only what the parameter
+    declares: no identity is involved, so no security decision is being taken
+    inside a class whose job is to describe data. `ParamBlock.check` refuses
+    such a value by name as well, because a setting silently doing nothing is
+    the failure this whole mechanism exists to remove.
+    """
+    for source in SOURCES:
+        if source == DEFAULT:
+            break
+        if source == OVERRIDE and not param.live:
+            continue
+        if not param.settable_by(source):
+            continue
+        layer = layers.get(source)
+        if not isinstance(layer, Mapping) or layer.get(param.key) is None:
+            continue
+        v = layer[param.key]
+        return Arg(param.key, param.type, param.coerce(v) if param.coerce else v, source)
+    return Arg(param.key, param.type, param.value(), DEFAULT)
+
+
+class ParamBlock(Block):
+    """A `Block` whose `check` and `from_dict` come from its `PARAMS`.
+
+    A subclass declares `PARAMS` and the dataclass fields that hold them, and
+    writes neither method. Sentences come out in `PARAMS` order, which is why
+    the order is the order an operator reads in the file.
+
+    `SWITCHABLE` keeps the shorthand an optional compartment already has: the
+    block may be plain `true`, meaning on with every default.
+    """
+    PARAMS: tuple[Parameter[Any], ...] = ()
+    SWITCHABLE: bool = False
+    # The parameter that is true because the block is present at all.
+    #
+    # `p2p` and `lan` both declare `enabled: bool = False` and mean opposite
+    # things by it: p2p is on as soon as its key appears, lan is not. That
+    # distinction lived only in the body of `from_dict` - two blocks that
+    # look identical behaving differently - and a conversion that trusted the
+    # dataclass default would have silently turned the libp2p transport off
+    # on every node configuring it as an object. Said here instead.
+    PRESENCE_ENABLES: str = ''
+    # False when PARAMS are a declaration for the census and the methods stay
+    # hand-written, because the raw form is not an object of named settings:
+    # `capacity` IS the mapping, and carries no field names in the file at
+    # all. Stated rather than inferred, so a block that merely has not been
+    # converted cannot pass for a deliberate exception.
+    DERIVED: bool = True
+
+    @classmethod
+    def check(cls, raw: Any, p: Problems) -> None:
+        label = cls.label or cls.key
+        if cls.SWITCHABLE:
+            if isinstance(raw, bool):
+                return
+            if not isinstance(raw, Mapping):
+                p.add('%s must be true, false or an object' % label)
+                return
+        elif not isinstance(raw, Mapping):
+            p.add('%s must be an object' % label)
+            return
+        for param in cls.PARAMS:
+            # config.json is the FILE layer. A parameter that does not permit
+            # it is refused by name rather than ignored: an operator who
+            # writes a setting that silently does nothing is exactly the
+            # failure this mechanism removes.
+            if (param.key in raw and raw[param.key] is not None
+                    and not param.settable_by(FILE)):
+                p.add('%s.%s cannot be set in config.json; it is settable by %s'
+                      % (label, param.key, ', '.join(param.sources)))
+                continue
+            if param.validate is not None:
+                param.validate(label, raw, p)
+                continue
+            if param.key not in raw or raw[param.key] is None:
+                if param.required:
+                    p.add('%s.%s is required' % (label, param.key))
+                continue
+            if not param.ok(raw[param.key]):
+                p.add(param.sentence(label))
+
+    @classmethod
+    def from_dict(cls, raw: Any) -> Any:
+        if cls.SWITCHABLE and isinstance(raw, bool):
+            # `true` is on with every default. `false` is the block's own
+            # default, which for a presence-enabled block is off.
+            if raw and cls.PRESENCE_ENABLES:
+                return cls(**{cls.PRESENCE_ENABLES: True})
+            return cls()
+        src: Mapping[str, Any] = raw if isinstance(raw, Mapping) else {}
+        vals = {p.attr: p.read(src) for p in cls.PARAMS}
+        if cls.PRESENCE_ENABLES and src.get(cls.PRESENCE_ENABLES) is None:
+            # Reached only because the key is in the file, so the block is on
+            # unless the operator wrote it off explicitly.
+            vals[cls.PRESENCE_ENABLES] = True
+        return cls(**vals)
+
+    @classmethod
+    def parameters(cls) -> tuple[Parameter[Any], ...]:
+        return cls.PARAMS
+
+    @classmethod
+    def describe(cls) -> BlockRecord:
+        """This block as data: its key and every parameter it declares."""
+        return {'key': cls.key, 'label': cls.label or cls.key,
+                'parameters': [p.describe() for p in cls.PARAMS]}
+
+
+def declared(block_cls: type) -> tuple[Parameter[Any], ...]:
+    """The parameters a block declares, or () for one not yet converted.
+
+    Returns () rather than raising so the two kinds of block can coexist while
+    the tree is converted one compartment at a time, and so a report can say
+    honestly which blocks it cannot describe yet.
+    """
+    return tuple(getattr(block_cls, 'PARAMS', ()) or ())
+
+
+def every_parameter(blocks: Sequence[type]) -> list[ParameterRow]:
+    """Every declared parameter across `blocks`, as flat rows.
+
+    This is the visibility the system exists for: one list, with the block it
+    belongs to, its type, its default and whether a reload applies it.
+    """
+    out: list[ParameterRow] = []
+    for b in blocks:
+        for param in declared(b):
+            row: ParameterRow = {**param.describe(),
+                                 'block': getattr(b, 'key', '') or getattr(b, '__name__', '')}
+            out.append(row)
+    return out
+
+
+def converted(block_cls: type) -> bool:
+    """Whether this block states its settings as parameters.
+
+    Asked of the class, not of its PARAMS: `mediation` and `transport` are
+    reserved and have no settings at all, so an empty PARAMS is their correct
+    and final answer. Counting them as outstanding would never reach zero.
+    """
+    return isinstance(block_cls, type) and issubclass(block_cls, ParamBlock)
+
+
+def undeclared(blocks: Sequence[type]) -> list[str]:
+    """Blocks not yet stating their settings as parameters."""
+    return [getattr(b, 'key', '') or b.__name__ for b in blocks if not converted(b)]
+
+
+def hand_written(blocks: Sequence[type]) -> list[str]:
+    """Blocks that declare parameters but keep their own check and from_dict.
+
+    Separate from `undeclared` because they mean different things: this list
+    is complete for the census and deliberately not derived, that one is work
+    outstanding.
+    """
+    return [getattr(b, 'key', '') or b.__name__ for b in blocks
+            if declared(b) and not getattr(b, 'DERIVED', True)]
+
+
+__all__ = ['Parameter', 'ParameterRecord', 'ParameterRow', 'BlockRecord', 'Arg', 'ParamBlock',
+           'resolve', 'declared', 'every_parameter', 'undeclared', 'SOURCES', 'OVERRIDE', 'ARG',
+           'ENV', 'FILE', 'MANIFEST', 'DEFAULT', 'ACCESS_ENFORCED', 'ACCESS_DECLARED_ONLY',
+           'hand_written', 'converted']