sgnmon 0.1.0__py3-none-any.whl

sgnmon/__init__.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2026, California Institute of Technology and contributors
+#
+# You should have received a copy of the licensing terms for this
+# software included in the file "LICENSE" located in the top-level
+# directory of this package. If you did not, you can view a copy at
+# https://git.ligo.org/greg/sgnmon/-/raw/main/LICENSE
+
+"""sgnmon --- monitoring and metrics for SGN streaming pipelines."""
+
+from sgnmon.elements import EventLatency, MonitorSink, MonitorTransform
+from sgnmon.health import (
+    CheckResult,
+    HealthReport,
+    MaxFrameAge,
+    MaxGapFraction,
+    MaxLatency,
+    MaxLatencyDrift,
+    MinThroughput,
+    NoEOS,
+)
+from sgnmon.metrics import MetricsMixin, MetricSpec
+from sgnmon.monitor import Monitor
+from sgnmon.probes import (
+    DEFAULT_WINDOWS,
+    FrameStats,
+    PadProbe,
+    ProbeSnapshot,
+    WindowStats,
+    frame_stats,
+)
+from sgnmon.server import MonitorServer, monitor_app
+
+try:
+    from sgnmon._version import __version__
+except ModuleNotFoundError:  # pragma: no cover -- source tree without vcs build
+    __version__ = "0.0.0"
+
+__all__ = [
+    "DEFAULT_WINDOWS",
+    "CheckResult",
+    "EventLatency",
+    "FrameStats",
+    "HealthReport",
+    "MaxFrameAge",
+    "MaxGapFraction",
+    "MaxLatency",
+    "MaxLatencyDrift",
+    "MetricSpec",
+    "MetricsMixin",
+    "MinThroughput",
+    "Monitor",
+    "MonitorServer",
+    "MonitorSink",
+    "MonitorTransform",
+    "NoEOS",
+    "PadProbe",
+    "ProbeSnapshot",
+    "WindowStats",
+    "__version__",
+    "frame_stats",
+    "monitor_app",
+]

sgnmon/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

sgnmon/cli.py

@@ -0,0 +1,256 @@
+# Copyright (c) 2026, California Institute of Technology and contributors
+#
+# You should have received a copy of the licensing terms for this
+# software included in the file "LICENSE" located in the top-level
+# directory of this package. If you did not, you can view a copy at
+# https://git.ligo.org/greg/sgnmon/-/raw/main/LICENSE
+
+"""Command line interface for sgnmon."""
+
+from __future__ import annotations
+
+import json
+import runpy
+import shutil
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import typer
+
+app = typer.Typer(
+    help="Monitoring and metrics for SGN streaming pipelines.",
+    no_args_is_help=True,
+)
+
+
+@app.command()
+def demo(
+    host: str = typer.Option("localhost", help="Interface to bind the server to."),
+    port: int = typer.Option(9090, help="Port for the monitoring server."),
+    rate: int = typer.Option(256, help="Sample rate of the fake data."),
+    gaps: bool = typer.Option(
+        True, help="Randomly emit gap buffers to exercise gap metrics."
+    ),
+    duration: float = typer.Option(
+        0.0, help="Seconds of data to produce; 0 runs until Ctrl+C."
+    ),
+) -> None:
+    """Run a small monitored fake pipeline and serve its metrics.
+
+    Builds a diamond-shaped pipeline --- a fake source fanning out to an
+    amplifier, a resampler, and a monitoring transform, all converging
+    into a null sink --- monitors it with both attachment styles (taps
+    everywhere, plus the spliced MonitorTransform), and serves the live
+    dashboard, /metrics, /health, and /status until the pipeline ends.
+    """
+    from sgn.apps import Pipeline
+    from sgn.sources import SignalEOS
+    from sgnts.sinks.null import NullSeriesSink
+    from sgnts.sources.fake_series import FakeSeriesSource
+    from sgnts.transforms.amplify import Amplify
+    from sgnts.transforms.resampler import Resampler
+
+    from sgnmon.elements import MonitorTransform
+    from sgnmon.monitor import Monitor
+    from sgnmon.server import MonitorServer
+
+    monitor = Monitor()
+    monitor.add_default_checks(
+        max_frame_age=10.0,
+        max_latency=10.0,
+        max_gap_fraction=0.9,
+    )
+
+    source_kwargs: dict = {}
+    if duration:
+        source_kwargs["duration"] = duration
+    source = FakeSeriesSource(
+        name="src",
+        source_pad_names=["H1"],
+        rate=rate,
+        signal_type="white",
+        ngap=-1 if gaps else 0,
+        real_time=True,
+        **source_kwargs,
+    )
+    amp = Amplify(
+        name="amp", sink_pad_names=["H1"], source_pad_names=["H1"], factor=2.0
+    )
+    resamp = Resampler(
+        name="resamp",
+        sink_pad_names=["H1"],
+        source_pad_names=["H1"],
+        inrate=rate,
+        outrate=rate // 2,
+    )
+    mon = MonitorTransform(name="mon", sink_pad_names=["H1"], monitor=monitor)
+    sink = NullSeriesSink(name="snk", sink_pad_names=["amp", "resamp", "mon"])
+
+    pipeline = Pipeline()
+    pipeline.insert(source, amp, resamp, mon, sink)
+    pipeline.link(
+        {
+            "amp:snk:H1": "src:src:H1",
+            "resamp:snk:H1": "src:src:H1",
+            "mon:snk:H1": "src:src:H1",
+            "snk:snk:amp": "amp:src:H1",
+            "snk:snk:resamp": "resamp:src:H1",
+            "snk:snk:mon": "mon:src:H1",
+        }
+    )
+    # the mon element records itself; tap everything else
+    monitor.tap(pipeline, exclude=r"^mon:")
+
+    with MonitorServer(monitor, host=host, port=port) as server:
+        typer.echo(f"dashboard: {server.url}/")
+        typer.echo(f"metrics:  {server.url}/metrics")
+        typer.echo(f"health:   {server.url}/health")
+        typer.echo(f"status:   {server.url}/status")
+        with SignalEOS():
+            pipeline.run()
+
+
+@app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
+)
+def run(
+    ctx: typer.Context,
+    host: str = typer.Option("localhost", help="Interface to bind the server to."),
+    port: int = typer.Option(9090, help="Port for the monitoring server."),
+    pad_types: str = typer.Option(
+        "src,snk,adp",
+        help="Comma-separated observation points to tap: src, snk, adp.",
+    ),
+    include: str | None = typer.Option(None, help="Only tap pads matching this regex."),
+    exclude: str | None = typer.Option(None, help="Skip pads matching this regex."),
+    module: bool = typer.Option(
+        False, "--module", "-m", help="Treat COMMAND as a python module name."
+    ),
+    process_metrics: bool = typer.Option(
+        False,
+        help="Also export process CPU/memory/fd metrics (Linux only).",
+    ),
+    max_frame_age: float = typer.Option(
+        60.0, help="Health: max seconds since last frame; 0 disables."
+    ),
+    max_latency: float = typer.Option(
+        0.0, help="Health: max frame latency in seconds; 0 disables."
+    ),
+    max_gap_fraction: float = typer.Option(
+        0.0, help="Health: max gap fraction over the window; 0 disables."
+    ),
+    min_throughput: float = typer.Option(
+        0.9,
+        help="Health: min fraction of expected throughput; 0 disables.",
+    ),
+    max_latency_drift: float = typer.Option(
+        10.0,
+        help="Health: max latency rise over the long-window baseline; 0 disables.",
+    ),
+    warmup: float = typer.Option(
+        0.0,
+        help="Health: extra grace period before checks apply, in seconds; "
+        "checks already self-pace their startup, so 0 is usually right.",
+    ),
+) -> None:
+    """Run any sgn application with monitoring attached, no code changes.
+
+    Everything after "--" is the application command line, e.g.::
+
+        sgnmon run --port 9090 -- sgn-cal --config c.yaml --model m.npz
+
+    The application runs in this process; when it executes its pipeline,
+    the pipeline is tapped and the dashboard served for the duration of
+    the run. The command must be a python console script on PATH, a
+    path to a python script, or (with -m) a module name.
+    """
+    from sgn.apps import Pipeline
+
+    from sgnmon.monitor import Monitor
+    from sgnmon.server import MonitorServer
+
+    command = list(ctx.args)
+    if not command:
+        typer.echo(
+            "no command given; usage: sgnmon run [options] -- <command...>", err=True
+        )
+        raise typer.Exit(2)
+
+    monitor = Monitor(process_metrics=process_metrics)
+    monitor.add_default_checks(
+        max_frame_age=max_frame_age or None,
+        max_latency=max_latency or None,
+        max_gap_fraction=max_gap_fraction or None,
+        min_throughput=min_throughput or None,
+        max_latency_drift=max_latency_drift or None,
+        warmup=warmup,
+    )
+    types = tuple(t.strip() for t in pad_types.split(",") if t.strip())
+
+    original_run = Pipeline.run
+
+    def monitored_run(self, *args, **kwargs):
+        monitor.tap(self, include=include, exclude=exclude, pad_types=types)
+        with MonitorServer(monitor, host=host, port=port) as server:
+            typer.echo(f"[sgnmon] dashboard: {server.url}/")
+            return original_run(self, *args, **kwargs)
+
+    Pipeline.run = monitored_run  # type: ignore[method-assign]
+
+    sys.argv = command
+    exit_code = 0
+    try:
+        if module:
+            runpy.run_module(command[0], run_name="__main__", alter_sys=True)
+        else:
+            target = shutil.which(command[0]) or command[0]
+            if not Path(target).exists():
+                typer.echo(f"command not found: {command[0]}", err=True)
+                raise typer.Exit(2)
+            runpy.run_path(target, run_name="__main__")
+    except SystemExit as error:  # console scripts exit via sys.exit(main())
+        if isinstance(error.code, int):
+            exit_code = error.code
+        elif error.code is not None:
+            typer.echo(str(error.code), err=True)
+            exit_code = 1
+    finally:
+        Pipeline.run = original_run  # type: ignore[method-assign]
+    raise typer.Exit(exit_code)
+
+
+@app.command()
+def check(
+    url: str = typer.Argument(..., help="Base URL of a running sgnmon server."),
+    timeout: float = typer.Option(10.0, help="HTTP timeout in seconds."),
+) -> None:
+    """Query the health endpoint of a running monitored pipeline.
+
+    Prints the JSON health report and exits 0 when healthy, 1 when any
+    check fails, and 2 when the server cannot be reached, so it can be used
+    directly as a monitoring plugin.
+    """
+    target = url.rstrip("/") + "/health"
+    if not target.startswith(("http://", "https://")):
+        target = "http://" + target
+    try:
+        with urllib.request.urlopen(target, timeout=timeout) as response:  # noqa: S310
+            code, body = response.status, response.read()
+    except urllib.error.HTTPError as error:
+        code, body = error.code, error.read()
+    except (urllib.error.URLError, TimeoutError, OSError) as error:
+        typer.echo(f"cannot reach {target}: {error}", err=True)
+        raise typer.Exit(2) from error
+    try:
+        report = json.loads(body)
+    except json.JSONDecodeError as error:
+        typer.echo(f"invalid response from {target}: {error}", err=True)
+        raise typer.Exit(2) from error
+    typer.echo(json.dumps(report, indent=2))
+    raise typer.Exit(0 if code == 200 and report.get("ok") else 1)
+
+
+if __name__ == "__main__":
+    app()

sgnmon/elements.py

@@ -0,0 +1,201 @@
+# Copyright (c) 2026, California Institute of Technology and contributors
+#
+# You should have received a copy of the licensing terms for this
+# software included in the file "LICENSE" located in the top-level
+# directory of this package. If you did not, you can view a copy at
+# https://git.ligo.org/greg/sgnmon/-/raw/main/LICENSE
+
+"""Pipeline elements that record monitoring metrics.
+
+Two styles are provided:
+
+* :class:`MonitorTransform` --- a transparent pass-through transform to
+  splice between two elements; frames are observed and forwarded unchanged.
+* :class:`MonitorSink` --- a terminal sink for branches whose frames you
+  only want to monitor.
+* :class:`EventLatency` --- a :class:`MonitorTransform` that additionally
+  measures per-event latency from a timestamp field, for event streams.
+
+All work with any frame type. Time-series frames additionally get gap/data
+time accounting and latency metrics (see :func:`sgnmon.probes.frame_stats`).
+For monitoring an existing pipeline without touching its graph, see
+:meth:`sgnmon.monitor.Monitor.tap`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import gpstime
+from sgn.base import SinkElement, SinkPad, SourcePad, TransformElement
+
+from sgnmon.monitor import Monitor
+
+if TYPE_CHECKING:
+    from sgn.frames import Frame
+
+
+@dataclass(kw_only=True)
+class MonitorTransform(TransformElement):
+    """A pass-through transform that records monitoring metrics.
+
+    Frames received on each sink pad are observed and forwarded unchanged
+    on the source pad of the same name. If no source pad names are given,
+    they mirror the sink pad names.
+
+    Parameters
+    ----------
+    monitor : Monitor or None, optional
+        The monitor to record into; defaults to :meth:`Monitor.default`.
+
+    Examples
+    --------
+    Splice into a link between a source and a sink::
+
+        mon = MonitorTransform(name="mon", sink_pad_names=["H1"])
+        pipeline.insert(src, mon, snk)
+        pipeline.link({
+            "mon:snk:H1": "src:src:H1",
+            "snk:snk:H1": "mon:src:H1",
+        })
+    """
+
+    monitor: Monitor | None = None
+
+    def __post_init__(self):
+        if not self.source_pad_names:
+            self.source_pad_names = list(self.sink_pad_names)
+        super().__post_init__()
+        if sorted(self.source_pad_names) != sorted(self.sink_pad_names):
+            msg = (
+                "MonitorTransform forwards frames between identically named "
+                f"pads; got sink pads {list(self.sink_pad_names)} and source "
+                f"pads {list(self.source_pad_names)}"
+            )
+            raise ValueError(msg)
+        monitor = self.monitor if self.monitor is not None else Monitor.default()
+        self.monitor = monitor
+        self._frames: dict[str, Frame] = {}
+        self._probes = {pad: monitor.probe(pad) for pad in self.sink_pads}
+
+    def pull(self, pad: SinkPad, frame: Frame) -> None:
+        """Observe a frame and stage it for forwarding.
+
+        Parameters
+        ----------
+        pad : sgn.base.SinkPad
+            The sink pad receiving the frame.
+        frame : Frame
+            The received frame.
+        """
+        self._probes[pad].observe(frame)
+        self._frames[self.rsnks[pad]] = frame
+
+    def new(self, pad: SourcePad) -> Frame:
+        """Forward the staged frame for the same-named sink pad.
+
+        Parameters
+        ----------
+        pad : sgn.base.SourcePad
+            The source pad producing the frame.
+
+        Returns
+        -------
+        Frame
+            The frame received on the same-named sink pad, unchanged.
+        """
+        return self._frames[self.rsrcs[pad]]
+
+
+@dataclass(kw_only=True)
+class EventLatency(MonitorTransform):
+    """A pass-through transform that measures per-event latency.
+
+    Like :class:`MonitorTransform`, frames are observed and forwarded
+    unchanged. Additionally, each event in a frame's data is inspected
+    for a timestamp field, and current GPS time minus that timestamp is
+    recorded as the pad's latency (histogram, last-value gauge, and
+    windowed statistics). Use for event streams --- frames carrying
+    dicts or objects with a GPS time --- which have no time-series
+    accounting and would otherwise get no latency metrics.
+
+    Parameters
+    ----------
+    time_field : str, optional
+        Event field holding the event's GPS time, ``"time"`` by
+        default; read by key for mappings and by attribute otherwise.
+        Events missing the field, or with a non-numeric value, are
+        skipped.
+    monitor : Monitor or None, optional
+        The monitor to record into; defaults to :meth:`Monitor.default`.
+    """
+
+    time_field: str = "time"
+
+    def pull(self, pad: SinkPad, frame: Frame) -> None:
+        """Observe and stage a frame, recording per-event latencies.
+
+        Parameters
+        ----------
+        pad : sgn.base.SinkPad
+            The sink pad receiving the frame.
+        frame : Frame
+            The received frame; ``frame.data`` is either one event or a
+            list of events.
+        """
+        super().pull(pad, frame)
+        if frame.is_gap or frame.data is None:
+            return
+        events = frame.data if isinstance(frame.data, list) else [frame.data]
+        now = gpstime.gpsnow()
+        probe = self._probes[pad]
+        for event in events:
+            if isinstance(event, Mapping):
+                value = event.get(self.time_field)
+            else:
+                value = getattr(event, self.time_field, None)
+            if value is None:
+                continue
+            try:
+                end_gps = float(value)
+            except (TypeError, ValueError):
+                continue
+            probe.record_latency(now - end_gps, end_gps=end_gps)
+
+
+@dataclass(kw_only=True)
+class MonitorSink(SinkElement):
+    """A terminal sink that only records monitoring metrics.
+
+    Useful to terminate a monitoring branch, e.g. behind a tee, when no
+    other sink consumes the stream.
+
+    Parameters
+    ----------
+    monitor : Monitor or None, optional
+        The monitor to record into; defaults to :meth:`Monitor.default`.
+    """
+
+    monitor: Monitor | None = None
+
+    def __post_init__(self):
+        super().__post_init__()
+        monitor = self.monitor if self.monitor is not None else Monitor.default()
+        self.monitor = monitor
+        self._probes = {pad: monitor.probe(pad) for pad in self.sink_pads}
+
+    def pull(self, pad: SinkPad, frame: Frame) -> None:
+        """Observe a frame and mark end-of-stream when it carries EOS.
+
+        Parameters
+        ----------
+        pad : sgn.base.SinkPad
+            The sink pad receiving the frame.
+        frame : Frame
+            The received frame.
+        """
+        self._probes[pad].observe(frame)
+        if frame.EOS:
+            self.mark_eos(pad)