mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/ui_stream/artefact_bind.py

@@ -0,0 +1,308 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Canonical artefact ↔ pipeline I/O ↔ WS stream binder.
+
+One name, four projections:
+
+* ``name`` / ``stream`` — WS topic owner (``custom.chat``, ``native``)
+* ``uri`` — ``artifact://experiment/scenario/item/run/name#attempt``
+* ``address`` — ``/api/artefacts/{library}/…/{name}``
+* ``path`` — ui-trace relative file (``pipeline/custom.chat.jsonl``)
+
+``inputs`` / ``outputs`` / ``producers`` / ``consumers`` are the DAG edges
+in both directions. ``complete`` is sealed: a cache hit is only legal on a
+complete artefact whose input digest still matches.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from mas.lab.lab.config.artifact_types import artefact_address, artefact_uri
+
+# Catalog shortcuts (`chat`) map onto the stream-owner identity (`custom.chat`).
+SHORTCUTS = {
+    "chat": "custom.chat",
+    "topology": "custom.topology",
+    "loop": "series",
+    "trace": "events",
+    "events": "events",
+    "metrics": "metric",
+}
+
+# Live folds consume the events artefact and produce one named snapshot each.
+# YAML pipeline steps merge on top via :func:`graph_from_steps`.
+LIVE_GRAPH: dict[str, dict[str, Any]] = {
+    "events": {
+        "stream": "native",
+        "inputs": [],
+        "outputs": ["events"],
+        "producers": ["runtime"],
+        "consumers": ["fold.chat", "fold.topology"],
+    },
+    "custom.chat": {
+        "stream": "custom.chat",
+        "plot": "chat",
+        "inputs": ["events"],
+        "outputs": ["custom.chat"],
+        "producers": ["fold.chat"],
+        "consumers": ["plot.chat"],
+    },
+    "custom.topology": {
+        "stream": "custom.topology",
+        "plot": "topology",
+        "inputs": ["events"],
+        "outputs": ["custom.topology"],
+        "producers": ["fold.topology"],
+        "consumers": ["plot.topology"],
+    },
+    "series": {
+        "stream": "series",
+        "plot": "loop",
+        "inputs": ["events"],
+        "outputs": ["series"],
+        "producers": ["fold.series"],
+        "consumers": ["plot.loop"],
+    },
+    "metric": {
+        "stream": "metric",
+        "plot": "metrics",
+        "inputs": ["events"],
+        "outputs": ["metric"],
+        "producers": ["fold.metrics"],
+        "consumers": ["plot.metrics"],
+    },
+}
+
+
+def canonical_artefact(name: str) -> str:
+    """Shortcut (`chat`) or topic (`custom.chat.snapshot`) → stream owner."""
+    key = str(name or "").strip()
+    if key in SHORTCUTS:
+        return SHORTCUTS[key]
+    if key in LIVE_GRAPH:
+        return key
+    return name_of_stream(key) or key
+
+
+def _node(name: str) -> dict[str, Any]:
+    declared = LIVE_GRAPH.get(name) or {}
+    return {
+        "stream": str(declared.get("stream") or name),
+        "plot": declared.get("plot"),
+        "inputs": list(declared.get("inputs") or ([] if name == "events" else ["events"])),
+        "outputs": list(declared.get("outputs") or [name]),
+        "producers": list(declared.get("producers") or []),
+        "consumers": list(declared.get("consumers") or []),
+    }
+
+
+def stream_of(name: str) -> str:
+    return str(_node(name).get("stream") or name)
+
+
+_STREAM_TICK_SUFFIXES = frozenset({"snapshot", "plot", "tick", "sealed", "html", "complete"})
+
+
+def _stream_matches(declared: str, owner: str) -> bool:
+    """Exact owner, or ``declared.snapshot`` / ``.sealed`` — not ``declared.foo``."""
+    if not declared or owner == declared:
+        return bool(declared) and owner == declared
+    prefix = declared + "."
+    if not owner.startswith(prefix):
+        return False
+    rest = owner[len(prefix) :]
+    first = rest.split(".", 1)[0]
+    return first in _STREAM_TICK_SUFFIXES
+
+
+def name_of_stream(stream: str) -> str:
+    owner = str(stream or "").strip()
+    if not owner:
+        return ""
+    for name, node in LIVE_GRAPH.items():
+        declared = str(node.get("stream") or name)
+        if _stream_matches(declared, owner):
+            return name
+    for prefix in ("custom.", "series.", "artifact."):
+        if owner.startswith(prefix):
+            parts = owner.split(".")
+            return ".".join(parts[:2]) if len(parts) >= 2 else owner
+    if owner.startswith("metric"):
+        return "metric"
+    if owner.startswith(("native.", "spec.", "job.", "command.", "result.", "control.", "governance.", "io.")) or owner in {
+        "native",
+        "job",
+        "spec",
+    }:
+        return "events"
+    return owner
+
+
+def is_sealed_topic(topic: str) -> bool:
+    name = str(topic or "")
+    return name.endswith(".sealed") or name.endswith(".complete") or name in {"job.completed"}
+
+
+@dataclass
+class ArtefactLink:
+    name: str
+    stream: str
+    uri: str = ""
+    address: str = ""
+    path: str = ""
+    complete: bool | None = None
+    digest: str = ""
+    key: str = ""
+    inputs: list[str] = field(default_factory=list)
+    outputs: list[str] = field(default_factory=list)
+    producers: list[str] = field(default_factory=list)
+    consumers: list[str] = field(default_factory=list)
+    plot: str | None = None
+    tree: str = "pipeline"
+
+    def as_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "artefact": self.name,
+            "stream": self.stream,
+            "uri": self.uri,
+            "address": self.address,
+            "path": self.path,
+            "inputs": list(self.inputs),
+            "outputs": list(self.outputs),
+            "producers": list(self.producers),
+            "consumers": list(self.consumers),
+            "tree": self.tree,
+        }
+        if self.complete is not None:
+            data["complete"] = self.complete
+        if self.digest:
+            data["digest"] = self.digest
+        if self.key:
+            data["key"] = self.key
+        if self.plot:
+            data["plot"] = self.plot
+        return data
+
+    def envelope_fields(self) -> dict[str, Any]:
+        data = {
+            "artefact": self.name,
+            "stream": self.stream,
+            "uri": self.uri,
+            "path": self.path,
+            "inputs": list(self.inputs),
+            "outputs": list(self.outputs),
+        }
+        if self.complete is not None:
+            data["complete"] = self.complete
+        return data
+
+
+def link_for(
+    name: str,
+    ref: dict[str, Any] | None = None,
+    *,
+    complete: bool | None = None,
+    tree: str | None = None,
+    digest: str = "",
+    key: str = "",
+) -> ArtefactLink:
+    name = canonical_artefact(name)
+    node = _node(name)
+    stream = str(node.get("stream") or name)
+    resolved_tree = tree or ("runtime" if name == "events" else "pipeline")
+    if resolved_tree == "runtime":
+        rel = "runtime.jsonl"
+    else:
+        rel = f"pipeline/{name}.jsonl"
+    data = dict(ref or {})
+    return ArtefactLink(
+        name=name,
+        stream=stream,
+        uri=artefact_uri(name, data),
+        address=artefact_address(name, data),
+        path=rel,
+        complete=complete,
+        digest=digest,
+        key=key,
+        inputs=list(node.get("inputs") or []),
+        outputs=list(node.get("outputs") or [name]),
+        producers=list(node.get("producers") or []),
+        consumers=list(node.get("consumers") or []),
+        plot=node.get("plot"),
+        tree=resolved_tree,
+    )
+
+
+def link_for_topic(
+    topic: str,
+    ref: dict[str, Any] | None = None,
+    *,
+    complete: bool | None = None,
+) -> ArtefactLink:
+    name = canonical_artefact(name_of_stream(topic))
+    sealed = is_sealed_topic(topic)
+    flag = True if sealed else complete
+    tree = "runtime" if name == "events" else "pipeline"
+    return link_for(name, ref, complete=flag, tree=tree)
+
+
+def _step_io(spec: Any) -> tuple[str, list[str], list[str]]:
+    step_name = str(getattr(spec, "name", None) or getattr(spec, "type", "") or "")
+    inputs = list(getattr(spec, "inputs", None) or [])
+    outputs = list(getattr(spec, "outputs", None) or [])
+    if isinstance(spec, dict):
+        if not step_name:
+            step_name = str(spec.get("name") or spec.get("type") or "")
+        if not inputs:
+            raw = spec.get("inputs") or spec.get("in")
+            if raw:
+                inputs = [raw] if isinstance(raw, str) else list(raw)
+        if not outputs:
+            raw = spec.get("outputs") or spec.get("out")
+            if raw:
+                outputs = [raw] if isinstance(raw, str) else list(raw)
+    return (
+        step_name,
+        [canonical_artefact(str(item)) for item in inputs if str(item).strip()],
+        [canonical_artefact(str(item)) for item in outputs if str(item).strip()],
+    )
+
+
+def graph_from_steps(steps: Iterable[Any] | None = None) -> dict[str, dict[str, Any]]:
+    """Merge YAML ``in:`` / ``out:`` onto the live-fold graph (bidirectional)."""
+    graph: dict[str, dict[str, Any]] = {name: dict(node) for name, node in LIVE_GRAPH.items()}
+    for spec in steps or []:
+        step_name, inputs, outputs = _step_io(spec)
+        for artefact in outputs:
+            node = graph.setdefault(artefact, _node(artefact))
+            producers = list(node.get("producers") or [])
+            if step_name and step_name not in producers:
+                producers.append(step_name)
+            node["producers"] = producers
+            node["outputs"] = list(dict.fromkeys([*(node.get("outputs") or []), artefact]))
+            node["inputs"] = list(dict.fromkeys([*(node.get("inputs") or []), *inputs]))
+        for artefact in inputs:
+            node = graph.setdefault(artefact, _node(artefact))
+            consumers = list(node.get("consumers") or [])
+            if step_name and step_name not in consumers:
+                consumers.append(step_name)
+            node["consumers"] = consumers
+            node.setdefault("outputs", [artefact])
+            node.setdefault("inputs", [])
+            node.setdefault("producers", [])
+            node.setdefault("stream", _node(artefact).get("stream"))
+    return graph
+
+
+def shard_is_complete(rows: list[dict[str, Any]], *, tree: str) -> bool:
+    if not rows:
+        return False
+    if any(row.get("complete") is True or is_sealed_topic(str(row.get("topic") or "")) for row in rows):
+        return True
+    if tree == "runtime":
+        kinds = {str((row.get("payload") or {}).get("kind") if isinstance(row.get("payload"), dict) else "") for row in rows}
+        kinds.update(str(row.get("topic") or "").rsplit(".", 1)[-1] for row in rows)
+        return bool(kinds & {"mas_call_end", "execution_end", "job.completed"})
+    return False

mas/lab/controller/ui_stream/auth.py

@@ -0,0 +1,136 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Token + TLS gate for the demo-ui stream plane."""
+
+from __future__ import annotations
+
+import hmac
+import os
+from typing import Any
+
+from fastapi import HTTPException, Request, WebSocket
+from starlette.datastructures import Headers
+
+_LOOPBACK = frozenset({"127.0.0.1", "localhost", "::1", "0:0:0:0:0:0:0:1", "testclient", "testserver"})
+_PROTECTED_PREFIXES = (
+    "/api/ingest",
+    "/api/commands",
+    "/api/stream",
+    "/api/plots",
+    "/ws",
+)
+CSP = (
+    "default-src 'self'; "
+    "script-src 'self' 'unsafe-inline' 'unsafe-eval'; "
+    "style-src 'self' 'unsafe-inline'; "
+    "img-src 'self' data:; "
+    "connect-src 'self' ws: wss:; "
+    "object-src 'none'; "
+    "frame-src 'none'; "
+    "frame-ancestors 'none'; "
+    "base-uri 'self'"
+)
+
+
+def ui_token() -> str:
+    return str(os.environ.get("MAS_DEMO_UI_TOKEN") or "").strip()
+
+
+def is_loopback(host: str) -> bool:
+    name = str(host or "").split("%", 1)[0].strip().lower().strip("[]")
+    if name.startswith("127."):
+        return True
+    return name in _LOOPBACK
+
+
+def ssl_enabled() -> bool:
+    cert = str(os.environ.get("MAS_DEMO_UI_SSL_CERTFILE") or "").strip()
+    key = str(os.environ.get("MAS_DEMO_UI_SSL_KEYFILE") or "").strip()
+    return bool(cert and key)
+
+
+def ssl_uvicorn_kwargs() -> dict[str, str]:
+    if not ssl_enabled():
+        return {}
+    return {
+        "ssl_certfile": str(os.environ.get("MAS_DEMO_UI_SSL_CERTFILE") or "").strip(),
+        "ssl_keyfile": str(os.environ.get("MAS_DEMO_UI_SSL_KEYFILE") or "").strip(),
+    }
+
+
+def assert_ui_bind(host: str) -> None:
+    """Refuse a non-loopback bind that has no TLS or no token."""
+    if is_loopback(host):
+        return
+    if not ssl_enabled():
+        raise RuntimeError("TLS is required for a non-loopback mas-lab serve bind")
+    if not ui_token():
+        raise RuntimeError("MAS_DEMO_UI_TOKEN is required for a non-loopback mas-lab serve bind")
+
+
+def token_from_headers(headers: Headers | dict[str, str] | None, extra: str | None = None) -> str:
+    if extra:
+        return str(extra)
+    if headers is None:
+        return ""
+    raw = str(headers.get("authorization") or headers.get("Authorization") or "")
+    if raw.lower().startswith("bearer "):
+        raw = raw[7:]
+    raw = raw.strip()
+    if raw:
+        return raw
+    return str(headers.get("x-mas-token") or headers.get("X-Mas-Token") or "").strip()
+
+
+def token_from_request(request: Request) -> str:
+    query = str(request.query_params.get("token") or "").strip()
+    return token_from_headers(request.headers, query or None)
+
+
+def check_token(provided: str, expected: str) -> bool:
+    if not expected:
+        return False
+    return hmac.compare_digest(str(provided or ""), str(expected))
+
+
+def authorize_http(request: Request) -> None:
+    expected = ui_token()
+    if not expected:
+        return
+    if not check_token(token_from_request(request), expected):
+        raise HTTPException(status_code=401, detail="unauthorized")
+
+
+def authorize_ws(ws: WebSocket, hello: dict[str, Any] | None = None) -> None:
+    expected = ui_token()
+    if not expected:
+        return
+    query = str(ws.query_params.get("token") or "").strip()
+    provided = token_from_headers(ws.headers, query or None)
+    if hello and not provided:
+        provided = str(hello.get("token") or hello.get("authorization") or "").strip()
+        if provided.lower().startswith("bearer "):
+            provided = provided[7:].strip()
+    if not check_token(provided, expected):
+        raise HTTPException(status_code=401, detail="unauthorized")
+
+
+def path_is_protected(path: str) -> bool:
+    raw = str(path or "")
+    return any(raw == prefix or raw.startswith(prefix + "/") or raw.startswith(prefix + "?") for prefix in _PROTECTED_PREFIXES)
+
+
+def ingest_headers() -> dict[str, str]:
+    headers = {"Content-Type": "application/json"}
+    token = ui_token()
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
+def security_headers() -> dict[str, str]:
+    return {
+        "Content-Security-Policy": CSP,
+        "X-Content-Type-Options": "nosniff",
+        "Referrer-Policy": "no-referrer",
+    }

mas/lab/controller/ui_stream/bus.py

@@ -0,0 +1,156 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Topic-scoped in-memory pub/sub for one gateway process."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+from mas.lab.controller.ui_stream.refs import DataSourceRef, EventEnvelope, topic_matches
+
+
+@dataclass
+class Subscription:
+    topics: tuple[str, ...]
+    ref: DataSourceRef
+    nonce: str | None = None
+    queue: asyncio.Queue[EventEnvelope] = field(default_factory=lambda: asyncio.Queue(maxsize=2000))
+
+
+class EventBus:
+    def __init__(self, *, max_history: int | None = 4000) -> None:
+        self._subs: list[Subscription] = []
+        self._history: list[EventEnvelope] = []
+        self._max_history = max_history
+
+    def publish(self, envelope: EventEnvelope) -> None:
+        self._history.append(envelope)
+        if self._max_history and len(self._history) > self._max_history:
+            self._history = self._history[-self._max_history :]
+        for sub in list(self._subs):
+            if not topic_matches(sub.topics, envelope.topic):
+                continue
+            if not sub.ref.matches(envelope.ref):
+                continue
+            if envelope.nonce and sub.nonce and envelope.nonce != sub.nonce:
+                if not envelope.replay and envelope.topic != "view.reset":
+                    continue
+            try:
+                sub.queue.put_nowait(envelope)
+            except asyncio.QueueFull:
+                try:
+                    sub.queue.get_nowait()
+                except Exception:
+                    pass
+                try:
+                    sub.queue.put_nowait(envelope)
+                except Exception:
+                    pass
+
+    def subscribe(
+        self, topics: Iterable[str], ref: DataSourceRef, *, nonce: str | None = None
+    ) -> Subscription:
+        sub = Subscription(topics=tuple(topics), ref=ref, nonce=str(nonce) if nonce else None)
+        self._subs.append(sub)
+        return sub
+
+    def unsubscribe(self, sub: Subscription) -> None:
+        try:
+            self._subs.remove(sub)
+        except ValueError:
+            return
+
+    def drop(self, *, run: str | None = None, batch: str | None = None, origin: str | None = None) -> int:
+        kept: list[EventEnvelope] = []
+        removed = 0
+        for item in self._history:
+            ref = item.ref
+            match = False
+            if run and ref.run == run:
+                match = True
+            if batch and ref.batch == batch:
+                match = True
+            if origin and ref.origin == origin:
+                match = True
+            if match:
+                removed += 1
+                continue
+            kept.append(item)
+        self._history = kept
+        return removed
+
+    def clear(self) -> None:
+        self._history.clear()
+        for sub in list(self._subs):
+            while True:
+                try:
+                    sub.queue.get_nowait()
+                except Exception:
+                    break
+
+    def wants_topic(self, topic: str, ref: DataSourceRef) -> bool:
+        """True when a live WS/plugin subscription would receive ``topic`` for ``ref``."""
+        for sub in self._subs:
+            if not topic_matches(sub.topics, topic):
+                continue
+            if sub.ref.matches(ref):
+                return True
+        return False
+
+    def demands_topic(self, topic: str, ref: DataSourceRef) -> bool:
+        """True when a subscription named this topic (not a catch-all ``*``)."""
+        for sub in self._subs:
+            globs = tuple(glob for glob in sub.topics if glob not in {"", "*"})
+            if not globs:
+                continue
+            if not topic_matches(globs, topic):
+                continue
+            if sub.ref.matches(ref):
+                return True
+        return False
+
+    def snapshot(self, topics: Iterable[str], ref: DataSourceRef) -> list[EventEnvelope]:
+        globs = tuple(topics)
+        out: list[EventEnvelope] = []
+        for item in self._history:
+            if not topic_matches(globs, item.topic):
+                continue
+            if not ref.matches(item.ref):
+                continue
+            out.append(item)
+        return out
+
+    def catchup(
+        self,
+        topics: Iterable[str],
+        ref: DataSourceRef,
+        *,
+        nonce: str | None = None,
+    ) -> list[EventEnvelope]:
+        """Latest envelope per topic after the last ``view.reset``.
+
+        A tab that connects after ingest has finished still paints from this
+        list. Empty ``topics`` means nothing (do not dump the whole bus).
+        """
+        globs = tuple(str(item) for item in topics if str(item or "").strip())
+        if not globs:
+            return []
+        items = self.snapshot(globs, ref)
+        last_reset: EventEnvelope | None = None
+        latest: dict[str, EventEnvelope] = {}
+        for item in items:
+            if item.nonce and nonce and item.nonce != nonce:
+                if not item.replay and item.topic != "view.reset":
+                    continue
+            if item.topic == "view.reset":
+                last_reset = item
+                latest.clear()
+                continue
+            latest[item.topic] = item
+        out: list[EventEnvelope] = []
+        if last_reset is not None:
+            out.append(last_reset)
+        out.extend(latest.values())
+        return out