moat-link-error 0.0.1__py3-none-any.whl

moat/link/error/__init__.py

@@ -0,0 +1,19 @@
+"""
+Error trace monitoring for :mod:`moat.link`.
+
+The server strips the backtrace from an error record before storing it.
+The full record is only published, non-retained, to ``run.error.*``. The
+:class:`ErrorMon` daemon in this package subscribes to that topic and
+remembers the first and the last trace per error path, so that
+``moat link error PATH get`` can show them after the fact.
+"""
+
+from __future__ import annotations
+
+from moat.lib.config import register as _register
+
+from ._impl import ErrorMon
+
+_register(__name__)
+
+__all__ = ["ErrorMon"]

moat/link/error/_cfg.yaml

@@ -0,0 +1,6 @@
+# The trace monitor ("moat link error monitor")
+monitor:
+  # forget traces that haven't been refreshed for this long (seconds)
+  max_age: 86400
+  # keep a process's announcement (run.id.*) this long after it ended (seconds)
+  id_keep: 60

moat/link/error/_impl.py

@@ -0,0 +1,171 @@
+"""
+The MoaT-Link error trace monitor.
+
+This daemon watches ``run.error.*`` — the non-retained MQTT topic that
+carries the *full* error record, including transient data like
+backtraces — and remembers the first and the last trace per error path.
+``moat link error PATH get`` retrieves the stored traces via the
+service announced by this daemon.
+"""
+
+from __future__ import annotations
+
+import anyio
+import logging
+
+from moat.util import NotGiven, attrdict
+from moat.lib.path import P, Path
+from moat.lib.priomap import TimerMap
+from moat.lib.rpc import MsgHandler
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from moat.link.backend import Message
+    from moat.link.client import Link
+
+    from collections.abc import AsyncIterator, Sequence
+    from typing import Any
+
+__all__ = ["ErrorMon"]
+
+logger = logging.getLogger(__name__)
+
+_TRACE_FIELDS = ("bt", "exc", "log")
+"Fields of an error record that describe one occurrence."
+
+_ERR_PATH = P(":R.run.error")
+"The topic to watch, relative to the link root."
+
+_ID_PATH = P(":R.run.id")
+"The topic where processes announce themselves."
+
+
+class ErrorMon(MsgHandler):
+    """
+    Record transient error data.
+
+    The server persists error records to ``error.<path>`` but strips the
+    backtrace from them. The full record is published, non-retained, to
+    ``run.error.<path>``. This daemon subscribes to that topic and
+    stores, per error path, the first and the last record that carries a
+    backtrace.
+
+    Stored traces are dropped when no new one has arrived for ``max_age``
+    seconds (default: a day). Storage is in memory only; restarting the
+    daemon loses the traces of errors that don't recur, and an error
+    that is active when the daemon starts has no ``first`` trace until a
+    new one arrives.
+
+    Error messages don't say which process sent them, or where it runs.
+    The daemon therefore also keeps the ``run.id.<ident>`` announcements
+    (host, PID, command line). A stored trace includes the announcement of
+    the process that caused it (the message's origin) as ``origin``. The
+    announcement disappears when the process ends, but a late error
+    message may still arrive, so the daemon keeps it for ``id_keep``
+    seconds (default: a minute) afterwards.
+
+    The daemon announces itself as the ``error.monitor`` service; its
+    ``get`` command returns the stored data for an error path.
+    """
+
+    def __init__(self, cfg: attrdict) -> None:
+        """
+        Args:
+            cfg: The ``link.error.monitor`` sub-configuration.
+        """
+        super().__init__()
+        self.max_age: float = cfg.get("max_age", 86400)
+        self.id_keep: float = cfg.get("id_keep", 60)
+        self.traces: dict[Path, dict[str, Any]] = {}
+        "The stored trace data, by error path."
+        self.ids: dict[str, dict[str, Any]] = {}
+        "The announcements of known processes, by ident."
+        self._timers: TimerMap[Path | str] = TimerMap()
+        "Expiry of traces (keyed by error path) and of ended processes (by ident)."
+
+    async def run(self, link: Link, evt: anyio.Event | None = None) -> None:
+        """
+        Watch ``run.error.*`` and record transient data.
+
+        Args:
+            link: A connected link client.
+            evt: Set once the subscription is up.
+        """
+        async with (
+            anyio.create_task_group() as tg,
+            link.monitor(_ID_PATH, subtree=True) as ids,
+            link.monitor(_ERR_PATH, subtree=True, retained=False) as mon,
+        ):
+            tg.start_soon(self._expire)
+            tg.start_soon(self._watch_ids, ids)
+            if evt is not None:
+                evt.set()
+            async for msg in mon:
+                self.record(
+                    msg.topic[len(_ERR_PATH) :], msg.data, msg.meta.timestamp, msg.meta.origin
+                )
+
+    async def _watch_ids(self, mon: AsyncIterator[Message]) -> None:
+        "Track the processes' announcements."
+        async for msg in mon:
+            ident = msg.topic[-1]
+            if not isinstance(ident, str):
+                continue
+            if msg.data is NotGiven:
+                if ident in self.ids:
+                    self._timers[ident] = self.id_keep
+            elif isinstance(msg.data, Mapping):
+                self.ids[ident] = dict(msg.data)
+                self._timers.pop(ident, None)
+
+    async def _expire(self) -> None:
+        """
+        Drop traces that haven't been refreshed for ``max_age``, and the
+        announcements of processes that ended more than ``id_keep`` ago.
+        """
+        async for key in self._timers:
+            if isinstance(key, Path):
+                del self.traces[key]
+            else:
+                del self.ids[key]
+
+    def record(self, path: Path, data: Any, ts: float, origin: str | None = None) -> None:
+        """
+        Store the transient part of one ``run.error`` message.
+
+        Messages without a backtrace (counter updates, clearing an error)
+        are ignored: the stored record that the server includes with them
+        also has ``exc`` and ``log``, which would overwrite the last trace.
+
+        Args:
+            path: The error path, without the ``run.error`` prefix.
+            data: The message's payload.
+            ts: The message's timestamp.
+            origin: The ident of the process that sent the message.
+        """
+        if len(path) == 0 or not isinstance(data, Mapping) or "bt" not in data:
+            return
+
+        trace = {k: data[k] for k in _TRACE_FIELDS if k in data}
+        trace["t"] = ts
+        if origin:
+            trace["origin"] = {**self.ids.get(origin, {}), "id": origin}
+        if (entry := self.traces.get(path)) is None:
+            self.traces[path] = {"trace": {"first": trace, "last": trace}, "seen": ts}
+        else:
+            entry["trace"]["last"] = trace
+            entry["seen"] = ts
+        self._timers[path] = self.max_age
+
+    async def cmd_get(self, path: Sequence[str | int]) -> dict[str, Any]:
+        """
+        Return the stored trace data for this error path, or an empty dict.
+
+        Args:
+            path: The error path elements, without the ``error`` prefix.
+        """
+        return self.traces.get(Path.build(path), {})
+
+    doc_get = "Return the stored first/last trace for an error path, or nothing."

moat/link/error/_main.py

@@ -0,0 +1,258 @@
+# command line interface
+from __future__ import annotations
+
+import anyio
+from contextlib import nullcontext
+
+import asyncclick as click
+
+from moat.util import NotGiven, yprint
+from moat.util import time as mtime
+from moat.lib.path import P, Path
+from moat.lib.run import AliasedGroup, attr_args
+from moat.link._data import data_get, node_attr
+from moat.link.announce import announcing
+from moat.link.client import Link
+from moat.link.meta import MsgMeta
+from moat.util.times import humandelta
+
+from typing import Any
+
+
+@click.group(cls=AliasedGroup, short_help="Manage error data", invoke_without_command=True)  # pylint: disable=undefined-variable
+@click.option("-m", "--meta", is_flag=True, help="include metadata")
+@click.argument("path", type=P, nargs=1)
+@click.pass_context
+async def cli(ctx, path, meta) -> None:
+    """
+    This subcommand reads and manipulates errors stored in the MoaT-Link service.
+
+    The bare command "moat link error monitor" runs the trace monitor
+    daemon, which remembers the backtraces that the server doesn't store.
+    To address an error named "monitor", add a subcommand, e.g.
+    "moat link error monitor get".
+    """
+    obj = ctx.obj
+    cfg = obj.cfg["link"]
+    obj.conn = await ctx.with_async_resource(Link(cfg))
+    obj.meta = meta
+    if ctx.invoked_subcommand is None and path == P("monitor"):
+        await _run_monitor(obj)
+        return
+    path = P("error") + path
+    if ctx.invoked_subcommand is None:
+        await data_get(obj.conn, path, meta=obj.meta, out=obj.stdout, recursive=False)
+    else:
+        obj.path = path
+
+
+async def _run_monitor(obj) -> None:
+    "Run the trace monitor and announce its service."
+    from ._impl import ErrorMon  # noqa: PLC0415
+
+    mon = ErrorMon(obj.cfg.link.error.monitor)
+    async with announcing(
+        obj.conn, host=False, name=P("error.monitor"), wait=True, service=mon
+    ) as ann:
+        await mon.run(obj.conn, evt=ann)
+
+
+async def _traces(link: Link, path: Path) -> dict[str, Any] | None:
+    """
+    Ask the trace monitor for the stored traces of this error.
+
+    Returns `None` if the monitor isn't running or knows nothing.
+    """
+    try:
+        with anyio.fail_after(2):
+            srv = await link.get_service(P("error.monitor"))
+            res = await srv.get(path[1:])
+    except (KeyError, TimeoutError):
+        return None
+    return dict(res.kw) or None
+
+
+@cli.command
+@click.pass_obj
+async def get(obj, **k) -> None:
+    """
+    Retrieve a MoaT-Link error entry.
+
+    If you read a sub-tree recursively, be aware that the whole subtree
+    will be read before anything is printed. Use the "watch --state" subcommand
+    for incremental output.
+
+    If the trace monitor ("moat link error monitor") is running, the
+    first and last backtrace it has seen are added as "trace".
+    """
+    res = await data_get(obj.conn, obj.path, meta=obj.meta, out=False, **k)
+    if (trace := await _traces(obj.conn, obj.path)) is not None:
+        entry = res.get("_") if isinstance(res, dict) else None
+        if obj.meta and isinstance(entry, dict):
+            entry = entry.get("data")
+        if isinstance(entry, dict):
+            entry["trace"] = trace["trace"]
+    yprint(res, stream=obj.stdout)
+
+
+@cli.command("set", short_help="Update an entry")
+@click.option("-k/-K", "--ok/--bad", is_flag=True, help="Mark as ok / not ok")
+@attr_args
+@click.pass_obj
+async def set_(obj, ok, **kw) -> None:
+    """
+    Update some MoaT-Link error entry.
+    """
+    if ok is not None:
+        kw["eval_"] = (*kw["eval_"], ("ok", repr(ok)))
+    res = await node_attr(obj, obj.path, **kw)
+
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+
+
+@cli.command(short_help="Delete an entry / subtree")
+@click.option(
+    "-b",
+    "--before",
+    type=float,
+    help="Don't delete entries created after this timestamp",
+)
+@click.option("-r", "--recursive", is_flag=True, help="Delete a complete subtree")
+@click.pass_obj
+async def delete(obj, before, recursive) -> None:
+    """
+    Delete an entry, or a subtree.
+
+    You might want to use "--before" flag to ensure that no other change
+    arrived after you viewed the data in question.
+
+    Non-recursively deleting an entry with children works and does *not*
+    affect the child entries.
+
+    The root entry cannot be deleted.
+    """
+    args = {}
+    if recursive:
+        args["rec"] = recursive
+    if before:
+        args["ts"] = before
+    res = await obj.conn.d.delete(obj.path, **args)
+    if obj.meta:
+        res = dict(data=res[0], meta=MsgMeta.restore(res[1:]).repr())
+    else:
+        res = res[0]
+    yprint(res, stream=obj.stdout)
+
+
+@cli.command("list")
+@click.pass_obj
+@click.option("-k/-A", "--ok/--all", is_flag=True, help="only list if ok / list all")
+@click.option("-r", "--raw", is_flag=True, help="show age in seconds")
+async def list_(obj, ok, raw) -> None:
+    """
+    List all errors with their paths and age.
+
+    Shows the name and age of each error.
+
+    The default is to list only open errors.
+    """
+    seen = False
+    now = mtime.time()
+    if ok is None:
+        ok = False
+    elif ok is False:
+        ok = None
+
+    async with obj.conn.d_watch(
+        obj.path,
+        state=True,
+        meta=True,
+        subtree=True,
+    ) as mon:
+        async for pdm in mon:
+            if pdm is None:
+                break
+            p, d, m = pdm
+            if ok is not None and d.get("ok", False) != ok:
+                continue
+
+            print(
+                f"{int(now - m.timestamp) if raw else humandelta(now - m.timestamp):10} {p}",
+                file=obj.stdout,
+            )
+            seen = True
+
+    if not seen:
+        print("No errors found.", file=obj.stdout)
+
+
+@cli.command()
+@click.option("-m", "--mode", type=str, help="Retrieval mode", default="s")
+@click.option("-M", "--mark", is_flag=True, help="Retrieval mode")
+@click.option("-s", "--subtree", is_flag=True, help="Read the whole tree.")
+@click.option("-D", "--add-date", is_flag=True, help="Add *_date entries")
+@click.option("-i", "--ignore", multiple=True, type=P, help="Skip this (sub)tree")
+@click.option("-n", "--min-length", type=int, help="Minimum path length")
+@click.option("-N", "--max-length", type=int, help="Maximum path length")
+@click.option("-a", "--max-age", type=int, help="Skip entries older than N seconds")
+@click.option("-t", "--timeout", type=int, help="Stop reading after N seconds")
+@click.pass_obj
+async def monitor(
+    obj,
+    mode,
+    add_date,  # noqa: ARG001
+    ignore,
+    mark,
+    subtree,
+    min_length,
+    max_length,
+    max_age,
+    timeout,
+) -> None:
+    """Monitor a MoaT-Link subtree.
+
+    The mode can be:
+    * c/current   read current data from the server
+    * u/update    read updates from MQTT
+    * s/stream    current plus updates
+    * m/mqtt      subscribe to MQTT stream, including retained data
+    """
+
+    match mode:
+        case "c" | "current":
+            state = True
+        case "u" | "update":
+            state = False
+        case "s" | "stream":
+            state = None
+        case "m" | "mqtt":
+            state = NotGiven
+        case _:
+            raise click.UsageError("Mode must be current|update|stream|mqtt")
+    if mark and state is not None:
+        raise click.UsageError("You can only add a mark in Stream mode")
+
+    def pm(p) -> bool:
+        for ip in ignore:
+            if len(p) >= len(ip) and p[: len(ip)] == ip:
+                return True
+        return False
+
+    with anyio.move_on_after(timeout) if timeout else nullcontext():
+        async with obj.conn.d_watch(
+            obj.path,
+            state=state,
+            mark=mark,
+            meta=True,
+            subtree=subtree,
+            max_length=max_length,
+            min_length=min_length,
+            age=max_age,
+        ) as mon:
+            async for res in mon:
+                if res is None:
+                    res = "*** Snapshot data ends ***"  # noqa:PLW2901
+                yprint(res, stream=obj.stdout)
+                print("---", file=obj.stdout)
+                obj.stdout.flush()

moat_link_error-0.0.1.dist-info/METADATA

@@ -0,0 +1,57 @@
+Metadata-Version: 2.4
+Name: moat-link-error
+Version: 0.0.1
+Summary: Error trace monitor for MoaT-Link
+Author-email: Matthias Urlichs <matthias@urlichs.de>
+Project-URL: homepage, https://m-o-a-t.org
+Project-URL: repository, https://github.com/M-o-a-T/moat
+Keywords: MoaT
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Framework :: AsyncIO
+Classifier: Framework :: Trio
+Classifier: Framework :: AnyIO
+Classifier: Development Status :: 4 - Beta
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: anyio~=4.2
+Requires-Dist: asyncclick~=8.3
+Requires-Dist: attrs
+Requires-Dist: moat-util~=0.64.0
+Requires-Dist: moat-link~=0.12.0
+Dynamic: license-file
+
+# Error traces
+
+% start synopsis
+% start main
+
+Remember the backtraces of MoaT-Link errors.
+
+The server stores error records without their backtrace. The full record
+is only published, without retaining it, to `run.error.*`. This package
+adds the `moat link error` command and a daemon (`moat link error monitor`)
+that keeps the first and the last backtrace it sees for each error, for a
+day by default, and hands them to `moat link error PATH get`, along with the host, PID and
+command line of the process that caused the error.
+
+% end synopsis
+
+The daemon keeps its data in memory. A restart forgets the traces of
+errors that don't recur.
+
+% end main
+
+## Usage
+
+Run `moat link error monitor` (the `moat-link-error-monitor` systemd unit
+does that) on one host. Then `moat link error PATH get` adds the stored
+traces to the error's record, as `trace.first` and `trace.last`.
+
+How long traces are kept is configured with `link.error.monitor.max_age`.
+See `docs/moat-link/errors.md` for details.
+
+## License
+
+This project is licensed under the same terms as MoaT.

moat_link_error-0.0.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+moat/link/error/__init__.py,sha256=o10-A8NZviQrmTBAboxqVlU-ZzaY98ds0oGto4H4Q-k,552
+moat/link/error/_cfg.yaml,sha256=mjSs37GsoYPoFe5sDBkdA7zXJPforBBjsNwsDUFFlio,238
+moat/link/error/_impl.py,sha256=DZEWNQVKqXqAJ2qAcT7_m2DpQLiqPnB9Vv9ZkQKTTzc,6346
+moat/link/error/_main.py,sha256=KZxw0WEOglojCnj6HlnFO3iB5NWRVIXujEEcYUfVLiA,8051
+moat_link_error-0.0.1.dist-info/licenses/LICENSE.txt,sha256=L5vKJLVOg5t0CEEPpW9-O_0vzbP0PEjEF06tLvnIDuk,541
+moat_link_error-0.0.1.dist-info/METADATA,sha256=I6Ba59PE4skIlXq3zgSJ7OFA2M9zWRfLRzwUg2wwWg8,1817
+moat_link_error-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+moat_link_error-0.0.1.dist-info/top_level.txt,sha256=pcs9fl5w5AB5GVi4SvBqIVmFrkRwQkVw_dEvW0Q0cSA,5
+moat_link_error-0.0.1.dist-info/RECORD,,

moat_link_error-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

moat_link_error-0.0.1.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,14 @@
+The code in this repository, and all MoaT submodules it refers to,
+is part of the MoaT project.
+
+Unless a submodule's LICENSE.txt states otherwise, all included files are
+licensed under the LGPL V3, as published by the FSF at
+https://www.gnu.org/licenses/lgpl-3.0.html .
+
+In addition to the LGPL's terms, the author(s) respectfully ask all users of
+this code to contribute any bug fixes or enhancements. Also, please link back to
+https://M-o-a-T.org.
+
+Thank you.
+
+Copyright © 2021 ff.: the MoaT contributor(s), as per the git changelog(s).

moat_link_error-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+moat