fleetmesh-node 0.2.0__py3-none-any.whl

node/compartment/lifecycle.py

@@ -0,0 +1,460 @@
+"""How a compartment lives: built, wired, running, degraded, quarantined, retired.
+
+The fifth thing a compartment declares, beside PARAMS (what it can be told),
+CHANNELS (what it says), METRICS (what it measures) and COMMANDS (what it can
+be asked to do): LIFECYCLE, how it behaves when it fails.
+
+Before this a compartment had no state. It was built or it was not, and once
+built it was ticked forever. A tick that raised was recorded as a refusal and
+then raised again on the next tick, and the one after, for as long as the node
+ran - a broken compartment cost a traceback per cycle, filled its refusal
+ring with one reason, and could not tell an operator anything they did not
+already know after the first one. Nothing distinguished "failed once and
+recovered" from "has been failing for a day".
+
+So the Registry now carries one `Tracker` per compartment and moves it through
+
+    CONFIGURED -> WIRED -> RUNNING <-> DEGRADED -> QUARANTINED -> RUNNING
+                             |  ^                (breaker)      (trial / resume)
+                             |  |
+                             v  |  restart (only when declared)
+                           DRAINING
+                             |
+                             v
+                           RETIRED          start() raised:  -> FAILED, or
+                                                              -> QUARANTINED when restartable
+
+and each move is a declared channel, so the journal, the control socket and
+the metrics all see the same transition.
+
+  CONFIGURED   built from its config block; not yet reaching its neighbours
+  WIRED        back-edges reached (`wire()` ran)
+  RUNNING      started and ticking cleanly
+  DEGRADED     the last tick raised; still ticked, and back to RUNNING after
+               one clean tick. A note, not trouble: one failure is weather.
+  QUARANTINED  `Breaker.failures` ticks raised within `Breaker.window`
+               seconds. Not ticked until the backoff passes, then given one
+               trial tick: clean returns it to RUNNING, a raise sends it back
+               with the backoff doubled (to `Breaker.backoff[1]` at most).
+               Trouble, because something is wrong now. `resume` lets an
+               operator end it early.
+  FAILED       an optional compartment's start() raised and it does not
+               declare `restart`, so nothing here can safely try again. Not
+               ticked, not stopped. Trouble. (A required compartment whose
+               start() raises still stops the node - see Registry.start.)
+  DRAINING     being stopped - for good, or for a restart
+  RETIRED      stopped; never ticked again
+
+`Lifecycle(restart=True)` is a promise the compartment makes: its stop() and
+then start() on the SAME instance leave it as good as new. Only a compartment
+that makes it is restarted - by the `restart` control command, by a config
+reload that changes its own block, by a quarantine's trial (which restarts it
+before the trial tick), and after a start() that raised (which quarantines it
+rather than failing it). The default is False because most compartments were
+written to start once: a restart nobody designed for would read as resilience
+the node does not have.
+
+`Lifecycle(snapshot=N)` is the other promise, and it is about the PROCESS
+restarting rather than one compartment: a core upgrade, a digest change, a
+`systemctl restart`. The compartment implements `snapshot()` - a JSON-able
+mapping of the state worth carrying - and `restore(state)`, and N is the
+version of that mapping's shape. On a clean stop the node writes every
+started compartment's snapshot to SNAPSHOT_FILE; the next start hands each one
+back BEFORE start() runs, but only when the version still matches, the file
+is recent (SNAPSHOT_MAX_AGE), and restore() returns. Anything else is a
+refusal recorded in the compartment's lifecycle (`restored`), and it starts
+fresh - which is what it did before snapshots existed, so a refused restore
+costs continuity and nothing else. The file is removed once a start
+succeeds: a snapshot is handed over once.
+
+restore() must be all-or-nothing: validate first, then assign. A raise is
+caught, but a half-applied restore is not something the node can see.
+
+A package rather than a root module for the reason node/channels is one:
+node/*.py is held to a monotone baseline (fleetmesh.org/scripts/check-root.py).
+Standard library only, beside the two surfaces it emits through.
+"""
+from __future__ import annotations
+
+import collections
+import dataclasses
+import json
+import os
+from typing import Any, Callable, Deque, Mapping, NamedTuple, Optional, TypedDict
+
+from ..channels import NOTE, TROUBLE, Channel
+from ..metrics import COUNTER, Metric
+
+# ---- the states ---------------------------------------------------------------
+CONFIGURED = 'configured'
+WIRED = 'wired'
+RUNNING = 'running'
+DEGRADED = 'degraded'
+QUARANTINED = 'quarantined'
+FAILED = 'failed'
+DRAINING = 'draining'
+RETIRED = 'retired'
+STATES: tuple[str, ...] = (CONFIGURED, WIRED, RUNNING, DEGRADED, QUARANTINED, FAILED, DRAINING, RETIRED)
+
+# The states in which a compartment is ticked on the ordinary cadence. A
+# quarantined one is ticked only for its trial; a retired one never.
+TICKED: frozenset[str] = frozenset({CONFIGURED, WIRED, RUNNING, DEGRADED})
+
+
+@dataclasses.dataclass(frozen=True)
+class Breaker:
+    """When repeated failure stops being weather and becomes a fault.
+
+    `failures` ticks that raise within `window` seconds trip it. `backoff` is
+    (first, ceiling) in seconds: the first quarantine lasts `first`, each
+    failed trial doubles it, and it never exceeds `ceiling`. `failures=0`
+    turns the breaker off for a compartment that must be ticked whatever
+    happens - the old behaviour, chosen rather than defaulted into.
+    """
+    failures: int = 5
+    window: int = 300
+    backoff: tuple[int, int] = (30, 600)
+
+    def delay(self, trips: int) -> int:
+        """Seconds a compartment stays quarantined on its `trips`-th trip."""
+        first, ceiling = self.backoff
+        return int(min(first * (2 ** max(trips - 1, 0)), ceiling))
+
+    def problems(self) -> list[str]:
+        out: list[str] = []
+        if self.failures < 0:
+            out.append('breaker failures must be 0 (off) or more')
+        if self.window <= 0:
+            out.append('breaker window must be positive')
+        first, ceiling = self.backoff
+        if first <= 0 or ceiling < first:
+            out.append('breaker backoff is (first, ceiling) with 0 < first <= ceiling')
+        return out
+
+
+@dataclasses.dataclass(frozen=True)
+class Lifecycle:
+    """How a compartment behaves when it fails. The default suits nearly all.
+
+    `restart`: stop() then start() on the same instance is safe and leaves it
+    as good as new. `snapshot`: the version of what snapshot() returns and
+    restore() reads; 0 carries nothing across a process restart. See the
+    module docstring for everything that acts on either.
+    """
+    breaker: Breaker = Breaker()
+    restart: bool = False
+    snapshot: int = 0
+
+    def problems(self) -> list[str]:
+        out = self.breaker.problems()
+        if self.snapshot < 0:
+            out.append('snapshot is a version: 0 (none) or more')
+        return out
+
+    def describe(self) -> 'LifecycleDecl':
+        return {'breaker': {'failures': self.breaker.failures, 'window': self.breaker.window,
+                            'backoff': list(self.breaker.backoff)},
+                'restart': self.restart, 'snapshot': self.snapshot}
+
+
+DEFAULT: Lifecycle = Lifecycle()
+
+
+class BreakerDecl(TypedDict):
+    failures: int
+    window: int
+    backoff: list[int]
+
+
+class LifecycleDecl(TypedDict):
+    breaker: BreakerDecl
+    restart: bool
+    snapshot: int
+
+
+class LifecycleRecord(TypedDict):
+    """One compartment's lifecycle as `meshnode lifecycle` and health read it."""
+    state: str
+    since: int
+    failures: int            # ticks that raised inside the breaker's window
+    trips: int               # times the breaker has tripped since the last clean run
+    until: int               # quarantined until (0 when not quarantined)
+    last_error: str
+    started: Optional[bool]  # None: start() not run yet; False: it raised; True: it returned
+    restarts: int
+    restored: str            # '' nothing offered; 'restored (vN, from <at>)'; or why it was refused
+    transitions: dict[str, int]
+    declared: LifecycleDecl
+
+
+class Transition(NamedTuple):
+    compartment: str
+    was: str
+    now: str
+    at: int
+    why: str
+
+
+class Tracker:
+    """One compartment's place in the lifecycle. Owned by the Registry.
+
+    Not thread-safe on its own: the Registry ticks from one thread and the
+    control socket's `resume` goes through the same lock the tick holds.
+    """
+
+    def __init__(self, name: str, lifecycle: Lifecycle, at: int = 0) -> None:
+        self.name: str = name
+        self.lifecycle: Lifecycle = lifecycle
+        self.state: str = CONFIGURED
+        self.since: int = at
+        self.fails: Deque[int] = collections.deque()
+        self.trips: int = 0
+        self.until: int = 0
+        self.last_error: str = ''
+        # Whether start() has returned. Registry.stop() skips a compartment
+        # whose start() raised: a stop() for something that never began is how
+        # a node that never served sent its peers a farewell (audit L1).
+        self.started: Optional[bool] = None
+        self.restarts: int = 0
+        # What happened to the snapshot the last process left for it.
+        self.restored: str = ''
+        self.transitions: collections.Counter[str] = collections.Counter()
+
+    def move(self, to: str, at: int, why: str = '') -> Optional[Transition]:
+        """Go to `to`. Returns the transition, or None when already there."""
+        if to == self.state:
+            return None
+        t = Transition(self.name, self.state, to, at, why)
+        self.state, self.since = to, at
+        self.transitions[to] += 1
+        return t
+
+    def due(self, now: int) -> bool:
+        """Whether the Registry should tick this compartment now."""
+        if self.state in TICKED:
+            return True
+        return self.state == QUARANTINED and now >= self.until
+
+    def succeeded(self, now: int) -> Optional[Transition]:
+        """A clean tick. Clears the failure record and recovers if it was down."""
+        self.fails.clear()
+        if self.state in (DEGRADED, QUARANTINED):
+            self.trips, self.until = 0, 0
+            return self.move(RUNNING, now, 'a clean tick')
+        return None
+
+    def failed(self, now: int, error: str) -> Optional[Transition]:
+        """A tick that raised. Degrades, or trips the breaker."""
+        b = self.lifecycle.breaker
+        self.last_error = error[:200]
+        trial = self.state == QUARANTINED
+        self.fails.append(now)
+        while self.fails and self.fails[0] <= now - b.window:
+            self.fails.popleft()
+        if b.failures and (trial or len(self.fails) >= b.failures):
+            self.trips += 1
+            self.until = now + b.delay(self.trips)
+            why = ('the trial tick raised: %s' % self.last_error if trial else
+                   '%d ticks raised within %ds: %s' % (len(self.fails), b.window, self.last_error))
+            if trial:
+                # Still quarantined; a new transition record keeps the
+                # operator's count honest about how many trials failed.
+                self.transitions[QUARANTINED] += 1
+                return Transition(self.name, QUARANTINED, QUARANTINED, now, why)
+            return self.move(QUARANTINED, now, why)
+        return self.move(DEGRADED, now, self.last_error)
+
+    def resume(self, now: int, why: str = 'resumed by the operator') -> Optional[Transition]:
+        """End a quarantine now. The next tick is an ordinary one."""
+        if self.state != QUARANTINED:
+            return None
+        self.fails.clear()
+        self.trips, self.until = 0, 0
+        return self.move(RUNNING, now, why)
+
+    def start_failed(self, now: int, error: str) -> Optional[Transition]:
+        """start() raised. Quarantined when a restart can try again, else FAILED.
+
+        From QUARANTINED (a trial restart that raised) this is a failed trial:
+        the breaker's `failed` already doubles the backoff and says so.
+        """
+        self.started = False
+        self.last_error = error[:200]
+        if self.state == QUARANTINED:
+            return self.failed(now, error)
+        if not self.lifecycle.restart:
+            return self.move(FAILED, now, 'start raised: %s' % self.last_error)
+        self.trips += 1
+        self.until = now + self.lifecycle.breaker.delay(self.trips)
+        return self.move(QUARANTINED, now, 'start raised: %s' % self.last_error)
+
+    def restarted(self, now: int, why: str) -> Optional[Transition]:
+        """stop() and start() both returned: as good as new, so a clean record."""
+        self.started = True
+        self.restarts += 1
+        self.fails.clear()
+        self.trips, self.until, self.last_error = 0, 0, ''
+        return self.move(RUNNING, now, 'restarted: %s' % why)
+
+    def describe(self) -> LifecycleRecord:
+        return {'state': self.state, 'since': self.since, 'failures': len(self.fails),
+                'trips': self.trips, 'until': self.until, 'last_error': self.last_error,
+                'started': self.started, 'restarts': self.restarts, 'restored': self.restored,
+                'transitions': dict(self.transitions),
+                'declared': self.lifecycle.describe()}
+
+
+# ---- carrying state across a process restart ------------------------------------
+SNAPSHOT_FILE = 'snapshots.json'
+SNAPSHOT_FORMAT = 1
+# A snapshot carries a compartment across a restart - an upgrade, a reload -
+# which takes seconds. One an hour old is from a node that was down, and the
+# state in it describes a world that has moved on (peers learned, links held)
+# while nobody was counting. Starting fresh is the honest answer then.
+SNAPSHOT_MAX_AGE = 3600
+
+
+class SnapshotEntry(TypedDict):
+    version: int
+    state: Mapping[str, Any]
+
+
+class SnapshotFile(TypedDict):
+    format: int
+    at: int
+    compartments: dict[str, SnapshotEntry]
+
+
+def write_snapshots(directory: str, at: int, compartments: Mapping[str, SnapshotEntry]) -> str:
+    """Write the snapshot file atomically. Returns its path."""
+    path = os.path.join(directory, SNAPSHOT_FILE)
+    body: SnapshotFile = {'format': SNAPSHOT_FORMAT, 'at': at, 'compartments': dict(compartments)}
+    tmp = path + '.tmp'
+    with open(tmp, 'w', encoding='utf-8') as f:
+        json.dump(body, f, indent=1, sort_keys=True)
+    os.replace(tmp, path)
+    return path
+
+
+def read_snapshots(directory: str, now: int) -> tuple[dict[str, SnapshotEntry], int, str]:
+    """(compartments, written at, why none) from the snapshot file.
+
+    An empty mapping with a reason when there is a file and it cannot be
+    used - unreadable, another format, or older than SNAPSHOT_MAX_AGE - and
+    with '' when there is simply no file. Never raises: a node must start
+    whatever the last one left behind.
+    """
+    path = os.path.join(directory, SNAPSHOT_FILE)
+    if not os.path.exists(path):
+        return {}, 0, ''
+    try:
+        with open(path, encoding='utf-8') as f:
+            body = json.load(f)
+    except (OSError, ValueError) as e:
+        return {}, 0, 'the snapshot file is unreadable (%s)' % type(e).__name__
+    if not isinstance(body, dict) or body.get('format') != SNAPSHOT_FORMAT:
+        return {}, 0, 'the snapshot file is not format %d' % SNAPSHOT_FORMAT
+    at = body.get('at')
+    comps = body.get('compartments')
+    if not isinstance(at, int) or not isinstance(comps, dict):
+        return {}, 0, 'the snapshot file is malformed'
+    if now - at > SNAPSHOT_MAX_AGE:
+        return {}, at, 'the snapshot is %ds old (more than %ds): starting fresh' % (now - at, SNAPSHOT_MAX_AGE)
+    out: dict[str, SnapshotEntry] = {}
+    for name, entry in comps.items():
+        if isinstance(entry, dict) and isinstance(entry.get('version'), int) and isinstance(entry.get('state'), dict):
+            out[str(name)] = {'version': entry['version'], 'state': entry['state']}
+    return out, at, ''
+
+
+def discard_snapshots(directory: str) -> None:
+    """A snapshot is handed over once: gone after the start that used it."""
+    try:
+        os.unlink(os.path.join(directory, SNAPSHOT_FILE))
+    except FileNotFoundError:
+        pass
+
+
+# ---- what the lifecycle says, and counts ---------------------------------------
+# Declared here, beside the states, rather than on each compartment: every
+# compartment moves through the same machine, so its transitions are the
+# Registry's to say. The payload names the compartment, so one channel per
+# kind of move is enough and a reader filters by field.
+def _payload(t: Transition) -> Mapping[str, Any]:
+    return {'compartment': t.compartment, 'was': t.was, 'now': t.now, 'why': t.why}
+
+
+CHANGED = 'lifecycle.changed'
+DEGRADED_CH = 'lifecycle.degraded'
+QUARANTINED_CH = 'lifecycle.quarantined'
+RECOVERED_CH = 'lifecycle.recovered'
+RESUMED_CH = 'lifecycle.resumed'
+RESTARTED_CH = 'lifecycle.restarted'
+FAILED_CH = 'lifecycle.failed'
+
+CHANNELS: tuple[Channel, ...] = (
+    Channel(name=CHANGED, description='A compartment moved through its lifecycle '
+                                      '(configured, wired, running, draining, retired).',
+            wire=_payload, say='{compartment} {was} -> {now}', tags=('lifecycle',)),
+    Channel(name=DEGRADED_CH, description='A compartment tick raised; it is still ticked and '
+                                          'recovers after one clean tick.',
+            level=NOTE, wire=_payload, say='{compartment} degraded: {why}', tags=('lifecycle',)),
+    Channel(name=QUARANTINED_CH, description='A compartment tripped its breaker and is not ticked '
+                                             'until its backoff passes.',
+            level=TROUBLE, wire=_payload, say='{compartment} quarantined: {why}', tags=('lifecycle',)),
+    Channel(name=RECOVERED_CH, description='A degraded or quarantined compartment ticked cleanly again.',
+            wire=_payload, say='{compartment} recovered ({was} -> {now})', tags=('lifecycle',)),
+    Channel(name=RESUMED_CH, description='An operator ended a quarantine early.',
+            wire=_payload, say='{compartment} resumed: {why}', tags=('lifecycle',)),
+    Channel(name=RESTARTED_CH, description='A compartment that declares Lifecycle(restart=True) was '
+                                           'stopped and started again in place.',
+            wire=_payload, say='{compartment} {why}', tags=('lifecycle',)),
+    Channel(name=FAILED_CH, description='An optional compartment\'s start() raised and it cannot be '
+                                        'restarted in place; it is not ticked until the node restarts.',
+            level=TROUBLE, wire=_payload, say='{compartment} failed: {why}', tags=('lifecycle',)),
+)
+
+METRICS: tuple[Metric, ...] = (
+    Metric(name='lifecycle_degraded', description='Times a compartment went from running to degraded. '
+                                                  'Every raise is in its refusal ring as tick: raised.',
+           kind=COUNTER, counts=DEGRADED_CH, tags=('lifecycle',)),
+    Metric(name='lifecycle_quarantined', description='Times a compartment breaker tripped, '
+                                                     'including failed trial ticks.',
+           kind=COUNTER, counts=QUARANTINED_CH, tags=('lifecycle',)),
+    Metric(name='lifecycle_recovered', description='Times a compartment recovered to running.',
+           kind=COUNTER, counts=RECOVERED_CH, tags=('lifecycle',)),
+    Metric(name='lifecycle_restarted', description='Times a compartment was restarted in place.',
+           kind=COUNTER, counts=RESTARTED_CH, tags=('lifecycle',)),
+    Metric(name='lifecycle_failed', description='Times an optional compartment failed to start.',
+           kind=COUNTER, counts=FAILED_CH, tags=('lifecycle',)),
+)
+
+
+def channel_for(t: Transition) -> str:
+    """Which declared channel a transition is said on."""
+    if t.now == QUARANTINED:
+        return QUARANTINED_CH
+    if t.now == DEGRADED:
+        return DEGRADED_CH
+    if t.now == FAILED:
+        return FAILED_CH
+    if t.now == RUNNING and t.why.startswith('restarted'):
+        return RESTARTED_CH
+    if t.now == RUNNING and t.was in (DEGRADED, QUARANTINED):
+        return RESUMED_CH if t.why.startswith('resumed') else RECOVERED_CH
+    return CHANGED
+
+
+def gauge(count: Callable[[], int], name: str, description: str) -> Metric:
+    """A sampled gauge over the Registry's trackers, built where they live."""
+    return Metric(name=name, description=description, sample=count, tags=('lifecycle',))
+
+
+__all__ = [
+    'BreakerDecl', 'Breaker', 'CHANGED', 'CHANNELS', 'CONFIGURED', 'DEFAULT', 'DEGRADED', 'DEGRADED_CH',
+    'DRAINING', 'FAILED', 'FAILED_CH', 'Lifecycle', 'LifecycleDecl', 'LifecycleRecord', 'METRICS',
+    'QUARANTINED', 'QUARANTINED_CH', 'RECOVERED_CH', 'RESTARTED_CH', 'RESUMED_CH', 'RETIRED', 'RUNNING',
+    'SNAPSHOT_FILE', 'SNAPSHOT_FORMAT', 'SNAPSHOT_MAX_AGE', 'STATES', 'SnapshotEntry', 'SnapshotFile',
+    'TICKED', 'Tracker', 'Transition', 'WIRED', 'channel_for', 'discard_snapshots', 'gauge',
+    'read_snapshots', 'write_snapshots',
+]