meegxer-plugins 0.2.1a2__py3-none-any.whl

meegxer/plugins/__init__.py

@@ -0,0 +1,6 @@
+"""Plugin runtime and SDKs: Python and Octave plugins as MeegXer strips.
+
+Loaded only through the node-provider entry point (CLAUDE.md rule 9).
+"""
+
+__version__ = "0.0.0"

meegxer/plugins/_sdk.py

@@ -0,0 +1,310 @@
+"""MeegXer plugin SDK for Python (protocol meegxer.plugin/0.2, spec 0.2.x).
+
+A single-file, numpy-only implementation of spec/plugin-protocol.md for use inside
+plugin images. Copy this file into your image and write::
+
+    import numpy as np
+    from meegxer_plugin import Plugin, run
+
+    class Gain(Plugin):
+        manifest = {
+            "spec_version": "0.2.1", "name": "lab.gain", "version": "0.1.0",
+            "type": "stream", "inputs": [{"name": "in", "type": "signal"}],
+            "outputs": [{"name": "out", "type": "signal"}],
+            "params": [{"name": "k", "type": "float", "default": 2.0}],
+        }
+
+        def transform(self, data, info, model, state, params):
+            return data * params["k"], state or {}, []
+
+    if __name__ == "__main__":
+        run(Gain())
+
+``data`` is a float64 array (n_channels, n_samples) in volts; ``info`` holds
+``ch_labels``, ``sfreq``, ``stream_id``, ``bads`` plus ``sample_index`` and
+``timestamp`` of the block. ``state`` is None on the first block. Return
+``(data, state, events)``; events are dicts with ``sample_index``, ``label`` and
+optionally ``duration`` and ``payload``. A plugin whose output lags its input
+(look-ahead) returns ``(data, state, events, first)`` with ``first`` the sample index
+of its first output sample (output may be shorter than the input), and returns what
+it still holds at the end of the stream from ``flush(info, model, state, params)``.
+Keep everything that must survive between blocks in ``state``: the host owns it,
+saves it and restores it after a restart.
+Never print to stdout (it carries the protocol); log to stderr.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import struct
+import sys
+import traceback
+from typing import Any, BinaryIO, ClassVar
+
+import numpy as np
+
+SDK = "meegxer-python/0.1.0"
+SPEC_VERSION = "0.2.5"
+MAGIC = b"MXP1"
+_DTYPES = {"float32", "float64", "int32", "int64", "uint8", "bool"}
+
+
+class Plugin:
+    """Subclass and set ``manifest``; override ``transform`` (and ``fit``)."""
+
+    manifest: ClassVar[dict[str, Any]] = {}
+
+    def fit(
+        self, data: np.ndarray, info: dict[str, Any], params: dict[str, Any]
+    ) -> dict:
+        raise NotImplementedError("this plugin does not implement fit")
+
+    def transform(
+        self,
+        data: np.ndarray,
+        info: dict[str, Any],
+        model: dict | None,
+        state: dict | None,
+        params: dict[str, Any],
+    ) -> tuple[Any, ...]:
+        raise NotImplementedError
+
+    def flush(
+        self,
+        info: dict[str, Any],
+        model: dict | None,
+        state: dict | None,
+        params: dict[str, Any],
+    ) -> tuple[Any, ...] | None:
+        """End of stream: what a lagging plugin still holds, as transform's
+        4-tuple; None (the default) when nothing is held."""
+        return None
+
+
+# -- wire format ----------------------------------------------------------------
+
+
+def _read_exact(stream: BinaryIO, n: int) -> bytes:
+    buf = b""
+    while len(buf) < n:
+        chunk = stream.read(n - len(buf))
+        if not chunk:
+            raise EOFError
+        buf += chunk
+    return buf
+
+
+def read_frame(stream: BinaryIO) -> tuple[dict, dict[str, np.ndarray]]:
+    magic = stream.read(4)
+    if not magic:
+        raise EOFError
+    magic += _read_exact(stream, 4 - len(magic)) if len(magic) < 4 else b""
+    if magic != MAGIC:
+        raise ValueError(f"bad magic {magic!r}")
+    (hlen,) = struct.unpack("<I", _read_exact(stream, 4))
+    header = json.loads(_read_exact(stream, hlen))
+    (plen,) = struct.unpack("<Q", _read_exact(stream, 8))
+    payload = _read_exact(stream, plen)
+    arrays = {}
+    for d in header.pop("arrays", []):
+        if d["dtype"] not in _DTYPES:
+            raise ValueError(f"unknown dtype {d['dtype']}")
+        raw = payload[d["offset"] : d["offset"] + d["nbytes"]]
+        dtype = np.dtype(d["dtype"]).newbyteorder("<")
+        arrays[d["name"]] = np.frombuffer(raw, dtype=dtype).reshape(d["shape"]).copy()
+    return header, arrays
+
+
+def write_frame(stream: BinaryIO, header: dict, arrays: dict[str, np.ndarray]) -> None:
+    descs, chunks, offset = [], [], 0
+    for name, arr in arrays.items():
+        arr = np.asarray(arr)
+        dtype = "bool" if arr.dtype == np.bool_ else arr.dtype.name
+        if dtype not in _DTYPES:
+            raise TypeError(f"array {name!r} has unsupported dtype {arr.dtype}")
+        raw = np.ascontiguousarray(
+            arr, dtype=np.dtype(dtype).newbyteorder("<")
+        ).tobytes()
+        descs.append(
+            {
+                "name": name,
+                "dtype": dtype,
+                "shape": list(arr.shape),
+                "offset": offset,
+                "nbytes": len(raw),
+            }
+        )
+        chunks.append(raw)
+        offset += len(raw)
+    head = json.dumps({**header, "arrays": descs}, allow_nan=False).encode()
+    payload = b"".join(chunks)
+    stream.write(MAGIC + struct.pack("<I", len(head)) + head)
+    stream.write(struct.pack("<Q", len(payload)) + payload)
+    stream.flush()
+
+
+def to_wire(tree: Any, arrays: dict[str, np.ndarray], prefix: str) -> Any:
+    if isinstance(tree, np.ndarray):
+        name = f"{prefix}{len(arrays)}"
+        arrays[name] = tree
+        return {"mxarray": name}
+    if isinstance(tree, np.generic):
+        return to_wire(tree.item(), arrays, prefix)
+    if isinstance(tree, float) and not math.isfinite(tree):
+        return {
+            "mxfloat": "nan" if math.isnan(tree) else ("inf" if tree > 0 else "-inf")
+        }
+    if isinstance(tree, (list, tuple)):
+        return [to_wire(v, arrays, prefix) for v in tree]
+    if isinstance(tree, dict):
+        return {str(k): to_wire(v, arrays, prefix) for k, v in tree.items()}
+    return tree
+
+
+def from_wire(tree: Any, arrays: dict[str, np.ndarray]) -> Any:
+    if isinstance(tree, dict):
+        if set(tree) == {"mxarray"}:
+            return arrays[tree["mxarray"]]
+        if set(tree) == {"mxfloat"}:
+            return float(tree["mxfloat"])
+        return {k: from_wire(v, arrays) for k, v in tree.items()}
+    if isinstance(tree, list):
+        return [from_wire(v, arrays) for v in tree]
+    return tree
+
+
+# -- main loop ------------------------------------------------------------------
+
+
+def run(
+    plugin: Plugin, stdin: BinaryIO | None = None, stdout: BinaryIO | None = None
+) -> None:
+    """Serve requests until ``shutdown`` or end of input."""
+    inp = stdin or sys.stdin.buffer
+    out = stdout or sys.stdout.buffer
+    sys.stdout = sys.stderr  # stray print() calls must not corrupt the protocol
+    models: dict[str, dict] = {}
+
+    while True:
+        try:
+            header, arrays = read_frame(inp)
+        except EOFError:
+            return
+        except Exception as exc:  # unreadable frame: we cannot resynchronise
+            print(f"meegxer_plugin: bad frame: {exc}", file=sys.stderr)
+            return
+        rid = header.get("id")
+        kind = header.get("type")
+        try:
+            if kind == "hello":
+                write_frame(
+                    out,
+                    {
+                        "type": "hello",
+                        "id": rid,
+                        "spec_version": SPEC_VERSION,
+                        "manifest": plugin.manifest,
+                        "sdk": SDK,
+                    },
+                    {},
+                )
+            elif kind == "shutdown":
+                write_frame(out, {"type": "ok", "id": rid}, {})
+                return
+            elif kind == "load_model":
+                models[header["model_id"]] = from_wire(header["model"], arrays)
+                write_frame(out, {"type": "ok", "id": rid}, {})
+            elif kind == "fit":
+                info = dict(
+                    header["info"],
+                    sample_index=header.get("sample_index", 0),
+                    timestamp=header.get("timestamp", 0.0),
+                )
+                model = plugin.fit(
+                    arrays["data"].astype(np.float64), info, header["params"]
+                )
+                model_arrays: dict[str, np.ndarray] = {}
+                tree = to_wire(model, model_arrays, "m")
+                write_frame(
+                    out, {"type": "model", "id": rid, "model": tree}, model_arrays
+                )
+            elif kind in ("transform", "flush"):
+                model_id = header.get("model_id")
+                if model_id is not None and model_id not in models:
+                    write_frame(
+                        out,
+                        {
+                            "type": "error",
+                            "id": rid,
+                            "error": "unknown_model",
+                            "message": f"model {model_id} not loaded",
+                        },
+                        {},
+                    )
+                    continue
+                info = dict(header["info"])
+                if "sample_index" in header:
+                    info.update(
+                        sample_index=header["sample_index"],
+                        timestamp=header["timestamp"],
+                    )
+                state = header.get("state")
+                state = None if state is None else from_wire(state, arrays)
+                model = models.get(model_id) if model_id else None
+                if kind == "transform":
+                    got = plugin.transform(
+                        arrays["data"].astype(np.float64),
+                        info,
+                        model,
+                        state,
+                        header["params"],
+                    )
+                    first = header["sample_index"]
+                else:
+                    got = plugin.flush(info, model, state, header["params"])
+                    if got is None:
+                        got = (np.zeros((len(info["ch_labels"]), 0)), state, [], 0)
+                    first = 0
+                data, new_state, events = got[0], got[1], got[2]
+                if len(got) > 3:
+                    first = int(got[3])
+                out_arrays: dict[str, np.ndarray] = {
+                    "data": np.asarray(data, dtype=np.float32)
+                }
+                tree = to_wire(new_state or {}, out_arrays, "s")
+                evs = [{"duration": 0, "payload": {}, **e} for e in events]
+                write_frame(
+                    out,
+                    {
+                        "type": "result",
+                        "id": rid,
+                        "state": tree,
+                        "events": evs,
+                        "sample_index": first,
+                    },
+                    out_arrays,
+                )
+            else:
+                write_frame(
+                    out,
+                    {
+                        "type": "error",
+                        "id": rid,
+                        "error": "bad_request",
+                        "message": f"unknown message type {kind!r}",
+                    },
+                    {},
+                )
+        except Exception as exc:
+            traceback.print_exc(file=sys.stderr)
+            write_frame(
+                out,
+                {
+                    "type": "error",
+                    "id": rid,
+                    "error": "plugin_error",
+                    "message": f"{type(exc).__name__}: {exc}",
+                },
+                {},
+            )

meegxer/plugins/adapter.py

@@ -0,0 +1,195 @@
+"""Expose a plugin as an ordinary Processor node.
+
+The adapter class is created from the plugin's handshake manifest, so the rest of
+MeegXer (graph, schedulers, conformance) cannot tell a plugin from a built-in. Each
+node instance owns one worker; the host keeps model and state.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, ClassVar
+
+import numpy as np
+
+from meegxer.core.block import Block, Event, SignalInfo
+from meegxer.core.manifest import Manifest
+from meegxer.core.node import Params, Processor, TransformResult
+from meegxer.core.serialization import State, hash_state
+from meegxer.plugins.wire import tree_from_wire, tree_to_wire
+from meegxer.plugins.worker import Launcher, PluginError, PluginWorker
+
+RESERVED_PREFIXES = ("meegxer.",)
+
+
+def _info_dict(info: SignalInfo) -> dict[str, Any]:
+    return {
+        "ch_labels": list(info.ch_labels),
+        "sfreq": info.sfreq,
+        "stream_id": info.stream_id,
+        "bads": list(info.bads),
+    }
+
+
+class PluginProcessor(Processor):
+    """Base class of generated plugin adapters."""
+
+    launcher: ClassVar[Launcher]
+    worker_options: ClassVar[dict[str, Any]] = {}
+    origin: ClassVar[dict[str, Any]] = {}  # image or command (sessions)
+
+    @classmethod
+    def provenance(cls) -> dict[str, Any]:
+        return dict(cls.origin)
+
+    def __init__(self) -> None:
+        self._worker: PluginWorker | None = None
+        self._model_ids: dict[int, tuple[State, str]] = {}
+
+    @property
+    def worker(self) -> PluginWorker:
+        if self._worker is None:
+            self._worker = PluginWorker(self.launcher, **self.worker_options)
+        return self._worker
+
+    def close(self) -> None:
+        if self._worker is not None:
+            self._worker.close()
+            self._worker = None
+
+    def _model_id(self, model: State) -> str:
+        cached = self._model_ids.get(id(model))
+        if cached is not None and cached[0] is model:
+            return cached[1]
+        model_id = hash_state(model)
+        self._model_ids = {id(model): (model, model_id)}  # keep only the current model
+        return model_id
+
+    def fit(self, calibration: Block, params: Params) -> State:
+        reply = self.worker.request(
+            {
+                "type": "fit",
+                "params": dict(params),
+                "info": _info_dict(calibration.info),
+                "sample_index": calibration.sample_index,
+                "timestamp": calibration.timestamp,
+            },
+            {"data": calibration.data.astype(np.float32)},
+        )
+        if reply.type != "model":
+            raise PluginError(f"expected a model reply, got {reply.type!r}")
+        model = tree_from_wire(reply.header.get("model", {}), reply.arrays)
+        if not isinstance(model, dict):
+            raise PluginError("model must be an object")
+        return model
+
+    def transform(
+        self, block: Block, model: State, state: State | None, params: Params
+    ) -> TransformResult:
+        result = self._exchange("transform", block.info, model, state, params, block)
+        assert result is not None
+        return result
+
+    def flush(
+        self, info: SignalInfo, model: State, state: State | None, params: Params
+    ) -> TransformResult | None:
+        """End of stream: ask a lagging plugin for what it still holds. Plugins
+        built for protocol 0.1 refuse the message: they hold nothing."""
+        try:
+            result = self._exchange("flush", info, model, state, params, None)
+        except PluginError as exc:
+            if "bad_request" in str(exc):
+                return None
+            raise
+        if result is None or not isinstance(result.block, Block):
+            return result
+        return result if result.block.n_samples else None
+
+    def _exchange(
+        self,
+        kind: str,
+        info: SignalInfo,
+        model: State,
+        state: State | None,
+        params: Params,
+        block: Block | None,
+    ) -> TransformResult | None:
+        arrays: dict[str, Any] = {}
+        if block is not None:
+            arrays["data"] = block.data.astype(np.float32)
+        state_tree = None if state is None else tree_to_wire(state, arrays, "s")
+        models: dict[str, tuple[Any, dict[str, Any]]] = {}
+        model_id = None
+        if self.manifest.requires_fit:
+            model_id = self._model_id(model)
+            model_arrays: dict[str, Any] = {}
+            models[model_id] = (tree_to_wire(model, model_arrays, "m"), model_arrays)
+        header: dict[str, Any] = {
+            "type": kind,
+            "params": dict(params),
+            "info": _info_dict(info),
+            "model_id": model_id,
+            "state": state_tree,
+        }
+        if block is not None:
+            header.update(sample_index=block.sample_index, timestamp=block.timestamp)
+        reply = self.worker.request(header, arrays, models=models)
+        if reply.type != "result" or "data" not in reply.arrays:
+            raise PluginError(f"expected a result with data, got {reply.header}")
+        if isinstance(reply.header.get("info"), dict):
+            new = reply.header["info"]
+            info = SignalInfo(
+                tuple(new["ch_labels"]),
+                float(new.get("sfreq", info.sfreq)),
+                new.get("stream_id", info.stream_id),
+                tuple(new.get("bads", ())),
+            )
+        data = np.asarray(reply.arrays["data"], dtype=np.float64)
+        if data.ndim == 1:  # an empty result may lose its channel axis
+            data = data.reshape(info.n_channels, -1)
+        first = int(reply.header.get("sample_index",
+                                     block.sample_index if block else 0))  # fmt: skip
+        if block is not None:
+            timestamp = block.timestamp + (first - block.sample_index) / info.sfreq
+        else:
+            timestamp = 0.0
+        out = Block(data, info, first, timestamp)
+        new_state = tree_from_wire(reply.header.get("state") or {}, reply.arrays)
+        events = tuple(Event(**e) for e in reply.header.get("events", []))
+        return TransformResult(out, new_state, events)
+
+
+def plugin_class(
+    manifest: dict[str, Any],
+    launcher: Launcher,
+    *,
+    worker_options: dict[str, Any] | None = None,
+    origin: dict[str, Any] | None = None,
+) -> type[PluginProcessor]:
+    """Build the node class for a plugin from its (handshake) manifest."""
+    parsed = Manifest.from_dict(manifest)
+    if parsed.role != "processor":
+        raise PluginError(f"plugins must be processors, not {parsed.role}s")
+    if parsed.name.startswith(RESERVED_PREFIXES):
+        raise PluginError(f"plugin name {parsed.name!r} uses a reserved prefix")
+    attrs: dict[str, Any] = {
+        "manifest": parsed,
+        "launcher": launcher,
+        "worker_options": dict(worker_options or {}),
+        "origin": dict(origin or {}),
+        "__doc__": f"Plugin {parsed.name} {parsed.version} ({launcher.describe()}).",
+    }
+    name = "Plugin_" + parsed.name.replace(".", "_")
+    return type(name, (PluginProcessor,), attrs)
+
+
+def probe_manifest(launcher: Launcher, **worker_options: Any) -> dict[str, Any]:
+    """Start a plugin once to read its manifest."""
+    worker = PluginWorker(launcher, **worker_options)
+    try:
+        return worker.start()
+    finally:
+        worker.close()
+
+
+PluginFactory = Callable[[], type[PluginProcessor]]

meegxer/plugins/cli.py

@@ -0,0 +1,132 @@
+"""``meegxer-plugin``: install, list, remove and check plugins."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from meegxer.core.notices import LOCAL_CODE_NOTICE
+from meegxer.plugins import index
+from meegxer.plugins.conformance import format_report, run_conformance
+from meegxer.plugins.worker import ContainerLauncher, LocalLauncher
+
+
+def _params(items: list[str]) -> dict[str, object]:
+    params: dict[str, object] = {}
+    for item in items:
+        name, _, value = item.partition("=")
+        try:
+            params[name] = json.loads(value)
+        except ValueError:
+            params[name] = value
+    return params
+
+
+def _accepted(args: argparse.Namespace) -> bool:
+    """Show the local-code notice; accepted with --accept-risk or a typed yes."""
+    print(LOCAL_CODE_NOTICE, file=sys.stderr)
+    if args.accept_risk:
+        return True
+    if not sys.stdin.isatty():
+        print("\nRead the notice, then run again with --accept-risk.", file=sys.stderr)
+        return False
+    answer = input("\nType yes to add this plugin: ")
+    return answer.strip().lower() == "yes"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="meegxer-plugin", description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    check = sub.add_parser("check", help="run the conformance suite")
+    check.add_argument("image", nargs="?", help="container image")
+    check.add_argument("--engine", help="docker or podman")
+    check.add_argument(
+        "--script", type=Path, help="a custom script (process(raw, ...))"
+    )
+    check.add_argument("--param", action="append", default=[], help="name=value")
+    check.add_argument(
+        "--local",
+        nargs=argparse.REMAINDER,
+        help="run a local command instead (unsandboxed)",
+    )
+
+    add = sub.add_parser(
+        "add", help="install a plugin: an image, a custom script or a local command"
+    )
+    add.add_argument("image", nargs="?", help="container image")
+    add.add_argument("--engine")
+    add.add_argument("--script", type=Path, help="a custom script (process(raw, ...))")
+    add.add_argument(
+        "--accept-risk",
+        action="store_true",
+        help="accept the local-code notice (local plugins run unsandboxed)",
+    )
+    add.add_argument(
+        "--local",
+        nargs=argparse.REMAINDER,
+        help="a local command speaking the plugin protocol (unsandboxed)",
+    )
+
+    sub.add_parser("list", help="list installed plugins")
+    rm = sub.add_parser("remove", help="uninstall a plugin by name")
+    rm.add_argument("name")
+    args = parser.parse_args(argv)
+
+    if args.command == "check":
+        if args.script:
+            launcher: ContainerLauncher | LocalLauncher = LocalLauncher(
+                tuple(index.script_command(args.script)), allow=True
+            )
+        elif args.local:
+            launcher = LocalLauncher(tuple(args.local), allow=True)
+        elif args.image:
+            launcher = ContainerLauncher(args.image, args.engine)
+        else:
+            parser.error("give an image, --script FILE or --local COMMAND")
+        checks = run_conformance(launcher, _params(args.param))
+        print(format_report(checks))
+        return 0 if all(c.passed for c in checks) else 1
+    if args.command == "add" and (args.script or args.local):
+        if args.script and not args.script.is_file():
+            parser.error(f"no such script: {args.script}")
+        if not _accepted(args):
+            return 1
+        command = index.script_command(args.script) if args.script else args.local
+        entry = index.add_local(
+            command, accept_risk=True, script=args.script if args.script else None
+        )
+        print(
+            f"installed {entry['manifest']['name']} {entry['manifest']['version']} "
+            f"(local, unsandboxed) into {index.index_path()}"
+        )
+        return 0
+    if args.command == "add":
+        if not args.image:
+            parser.error("give an image, --script FILE or --local COMMAND")
+        entry = index.add_image(args.image, args.engine)
+        print(
+            f"installed {entry['manifest']['name']} {entry['manifest']['version']} "
+            f"({entry['image_id'][:19]}) into {index.index_path()}"
+        )
+        return 0
+    if args.command == "list":
+        for entry in index.load_index():
+            where = entry.get("image") or " ".join(entry.get("command", []))
+            flag = "  local, unsandboxed" if entry.get("unsandboxed") else ""
+            name, version = entry["manifest"]["name"], entry["manifest"]["version"]
+            print(f"{name:40} {version:8} {where}{flag}")
+        return 0
+    if args.command == "remove":
+        if not index.remove(args.name):
+            print(f"no plugin named {args.name}", file=sys.stderr)
+            return 1
+        print(f"removed {args.name}")
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())