mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/ui_stream/stream.py

@@ -0,0 +1,95 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""One stream of envelopes. Backend protocols are adapters that publish or consume frames.
+
+A frame is always ``{topic, ts, ref, source, payload, replay, id}``. Topics are
+the dispatch key for plugins (``command.*``, ``native.*``, ``artifact.*``, …).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mas.lab.controller.ui_stream.refs import EventEnvelope, envelope, ref_from_mapping
+
+_SOURCES = {
+    "command",
+    "result",
+    "control",
+    "native",
+    "telemetry",
+    "metric",
+    "job",
+    "catalog",
+    "spec",
+    "artifact",
+    "io",
+    "governance",
+}
+
+
+def parse_frame(data: dict[str, Any] | None) -> EventEnvelope:
+    raw = dict(data or {})
+    payload = raw.get("payload")
+    topic = str(raw.get("topic") or "").strip()
+    source = str(raw.get("source") or "").strip()
+    if source not in _SOURCES:
+        if topic.startswith("command."):
+            source = "command"
+        elif topic.startswith("result."):
+            source = "result"
+        elif topic.startswith("native.") or topic.startswith("telemetry."):
+            source = "native"
+        elif topic.startswith("metric."):
+            source = "metric"
+        elif topic.startswith("control."):
+            source = "control"
+        elif topic.startswith("job."):
+            source = "job"
+        elif topic.startswith("spec."):
+            source = "spec"
+        elif topic.startswith("io."):
+            source = "io"
+        elif topic.startswith("governance."):
+            source = "governance"
+        else:
+            source = "artifact"
+    if not topic:
+        kind = ""
+        if isinstance(payload, dict):
+            kind = str(payload.get("kind") or payload.get("verb") or source)
+        topic = f"{source}.{kind or 'frame'}"
+    ref = ref_from_mapping(raw.get("ref") if isinstance(raw.get("ref"), dict) else {})
+    if source not in _SOURCES:
+        source = "artifact"
+    return envelope(
+        topic,
+        ref,
+        source,  # type: ignore[arg-type]
+        payload,
+        replay=bool(raw.get("replay")),
+        event_id=str(raw.get("id") or "") or None,
+        ts=str(raw.get("ts") or "") or None,
+    )
+
+
+def is_command_frame(frame: EventEnvelope, raw: dict[str, Any] | None = None) -> bool:
+    if frame.source == "command" or frame.topic.startswith("command."):
+        return True
+    payload = frame.payload if isinstance(frame.payload, dict) else {}
+    if payload.get("verb"):
+        return True
+    return bool(raw and raw.get("verb"))
+
+
+def command_payload(frame: EventEnvelope, raw: dict[str, Any] | None = None) -> dict[str, Any]:
+    payload = dict(frame.payload) if isinstance(frame.payload, dict) else {}
+    extra = dict(raw or {})
+    for key in ("topic", "source", "ts", "replay", "id", "payload"):
+        extra.pop(key, None)
+    merged = {**extra, **payload}
+    if not merged.get("verb") and frame.topic.startswith("command."):
+        merged["verb"] = frame.topic.split(".", 1)[1]
+    if not merged.get("ref"):
+        merged["ref"] = frame.ref.as_dict()
+    return merged

mas/lab/controller/ui_stream/streams.py

@@ -0,0 +1,42 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Named telemetry streams plugins get() / subscribe() on."""
+
+from __future__ import annotations
+
+# Logical stream → topic globs. Plugins may also pass raw topic globs.
+# Keep in sync with demo-ui/src/streams.js (same keys and topic lists).
+STREAM_TOPICS: dict[str, tuple[str, ...]] = {
+    "spec": ("spec.snapshot", "spec.*"),
+    "observability": ("native.*",),
+    "metrics": ("metrics.snapshot", "metric.*", "native.metric", "metric.scope"),
+    "artifacts": ("artifact.*", "artifact.metric"),
+    "governance": ("governance.*", "control.context"),
+    "control": ("control.*", "control.structure", "control.context"),
+    "hitl": ("control.*",),
+    "job": ("job.*", "job.progress"),
+    "custom": ("custom.*",),
+    "series": ("series.*",),
+}
+
+
+def expand_stream_names(names: list[str] | tuple[str, ...] | None) -> list[str]:
+    """Map ``spec`` / ``observability`` / topic globs onto unique topic globs."""
+    out: list[str] = []
+    seen: set[str] = set()
+    for raw in names or []:
+        text = str(raw or "").strip()
+        if not text or text == "*":
+            continue
+        if text in STREAM_TOPICS:
+            parts = STREAM_TOPICS[text]
+        elif text.startswith(("custom.", "series.")) and "*" not in text:
+            parts = (text, f"{text}.*")
+        else:
+            parts = (text,)
+        for item in parts:
+            if item in seen:
+                continue
+            seen.add(item)
+            out.append(item)
+    return out

mas/lab/controller/ui_stream/timeline.py

@@ -0,0 +1,183 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Discrete-event replay clock.
+
+All sources are preloaded, merged by timestamp, then emitted on one clock.
+Lookahead is the whole timeline (not a streaming parse of several files).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections import Counter
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(order=True)
+class Scheduled:
+    ts: float
+    seq: int
+    tree: str = field(compare=False, default="runtime")
+    ref: Any = field(compare=False, default=None)
+    record: dict[str, Any] = field(compare=False, default_factory=dict)
+    envelope: dict[str, Any] | None = field(compare=False, default=None)
+
+
+def event_ts_s(record: dict[str, Any] | None) -> float | None:
+    if not isinstance(record, dict):
+        return None
+    raw = record.get("timestamp", record.get("ts", record.get("t")))
+    if isinstance(raw, (int, float)):
+        value = float(raw)
+        return value / 1000.0 if value > 1e12 else value
+    if isinstance(raw, str) and raw.strip():
+        try:
+            return float(raw)
+        except ValueError:
+            from datetime import datetime
+
+            try:
+                return datetime.fromisoformat(raw.replace("Z", "+00:00")).timestamp()
+            except ValueError:
+                return None
+    return None
+
+
+def envelope_ts_s(row: dict[str, Any]) -> float | None:
+    ts = event_ts_s(row)
+    if ts is not None:
+        return ts
+    payload = row.get("payload")
+    if isinstance(payload, dict):
+        return event_ts_s(payload)
+    return None
+
+
+def merge_timeline(lanes: list[list[Scheduled]]) -> list[Scheduled]:
+    """Stable merge of preloaded lanes ordered by (ts, seq)."""
+    merged: list[Scheduled] = []
+    seq = 0
+    for lane in lanes:
+        for item in lane:
+            merged.append(
+                Scheduled(
+                    ts=item.ts,
+                    seq=seq,
+                    tree=item.tree,
+                    ref=item.ref,
+                    record=item.record,
+                    envelope=item.envelope,
+                )
+            )
+            seq += 1
+    merged.sort()
+    return merged
+
+
+def _coherent_fallback(times: list[float | None]) -> float:
+    """Prefer unix event time over epoch-0 placeholders mixed into the same file."""
+    unix = [t for t in times if t is not None and t > 1e9]
+    if unix:
+        return unix[0]
+    relative = [t for t in times if t is not None and t != 0.0]
+    return relative[0] if relative else 0.0
+
+
+def _fill_ts(raw: float | None, fallback: float, *, unix: bool) -> float:
+    if raw is None:
+        return fallback
+    if unix and raw < 1e9:
+        return fallback
+    return raw
+
+
+def records_to_scheduled(
+    records: list[dict[str, Any]],
+    *,
+    ref: Any = None,
+    tree: str = "runtime",
+) -> list[Scheduled]:
+    times = [event_ts_s(record) for record in records]
+    unix = any(t is not None and t > 1e9 for t in times)
+    fallback = _coherent_fallback(times)
+    out: list[Scheduled] = []
+    for i, record in enumerate(records):
+        ts = _fill_ts(times[i], fallback, unix=unix)
+        fallback = ts
+        out.append(Scheduled(ts=ts, seq=i, tree=tree, ref=ref, record=record))
+    return out
+
+
+def envelopes_to_scheduled(
+    rows: list[dict[str, Any]],
+    *,
+    tree: str,
+) -> list[Scheduled]:
+    times = [envelope_ts_s(row) if isinstance(row, dict) else None for row in rows]
+    unix = any(t is not None and t > 1e9 for t in times)
+    fallback = _coherent_fallback(times)
+    out: list[Scheduled] = []
+    for i, row in enumerate(rows):
+        ts = _fill_ts(times[i], fallback, unix=unix)
+        fallback = ts
+        out.append(Scheduled(ts=ts, seq=i, tree=tree, envelope=row if isinstance(row, dict) else {}))
+    return out
+
+
+async def pump_timeline(
+    items: list[Scheduled],
+    *,
+    speed: float,
+    emit: Callable[[Scheduled], Awaitable[None] | None],
+    cancelled: Callable[[], bool] | None = None,
+    on_progress: Callable[[dict[str, Any]], None] | None = None,
+) -> None:
+    """Emit preloaded events on a shared clock. ``speed`` 0 is instant, 1 realtime."""
+    if not items:
+        return
+    t0 = items[0].ts
+    t1 = items[-1].ts
+    n_total = len(items)
+    tree_totals = Counter(item.tree for item in items)
+    tree_done: Counter[str] = Counter()
+    clock = asyncio.get_running_loop()
+    wall0 = clock.time()
+    prev_ts = t0
+    jump = 3600.0
+    for i, item in enumerate(items, 1):
+        if cancelled and cancelled():
+            return
+        if speed > 0:
+            gap = item.ts - prev_ts
+            clock_break = gap > jump or gap < -1e-9 or (item.ts > 1e9 and prev_ts < 1e7)
+            if clock_break:
+                t0 = item.ts - (clock.time() - wall0) * speed
+                delay = 0.0
+            else:
+                delay = (item.ts - t0) / speed - (clock.time() - wall0)
+            if delay > 0:
+                await asyncio.sleep(delay)
+        prev_ts = item.ts
+        result = emit(item)
+        if asyncio.iscoroutine(result):
+            await result
+        await asyncio.sleep(0)
+        tree_done[item.tree] += 1
+        if on_progress is not None:
+            on_progress(
+                {
+                    "t": item.ts,
+                    "t0": t0,
+                    "t1": t1,
+                    "n": i,
+                    "n_total": n_total,
+                    "tree": item.tree,
+                    "speed": speed,
+                    "trees": {
+                        name: {"n": tree_done[name], "n_total": tree_totals[name]}
+                        for name in tree_totals
+                    },
+                }
+            )

mas/lab/controller/ui_stream/ui_trace.py

@@ -0,0 +1,258 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""On-disk UI trace: the WS envelope stream, split into runtime vs pipeline.
+
+A replay directory looks like::
+
+    ui-trace/
+      manifest.json                 # kind, files[], runtime_digest
+      runtime.jsonl                 # native / spec / job  (MAS telemetry)
+      pipeline/<artefact>.jsonl     # one materialized artefact (chat, series, …)
+
+``runtime.jsonl`` is the input log. Each ``pipeline/<artefact>.jsonl`` is the
+realtime counterpart of a pipeline-step fingerprint: a content-addressed
+snapshot keyed by ``hash(artefact, version, runtime_digest)``. Replay skips a
+shard when that key still matches (interrupt/restart, or adding a new fold
+that only misses its own file). ``files[]`` is the extension point.
+
+The batch executor keeps the same contract on
+``<output_dir>/.cache/<step>.fingerprint`` + ``artefacts/<aa>/<key>/``.
+This tree is the WS/live view of those artefacts, not a second cache.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+KIND = "mas-lab.ui-trace"
+VERSION = 1
+ARTEFACT_VERSION = "2"
+
+RUNTIME_PREFIXES = (
+    "native.",
+    "spec.",
+    "job.",
+    "command.",
+    "result.",
+    "control.",
+    "governance.",
+    "io.",
+)
+PIPELINE_PREFIXES = ("custom.", "series.", "artifact.", "metric.", "metrics.")
+
+
+def artefact_owner(topic: str) -> str:
+    name = str(topic or "")
+    parts = name.split(".")
+    if name.startswith(("custom.", "series.", "artifact.")) and len(parts) >= 2:
+        return ".".join(parts[:2])
+    if name.startswith("metric"):
+        return "metric"
+    return name or "pipeline"
+
+
+def artefact_filename(owner: str) -> str:
+    safe = "".join(ch if ch.isalnum() or ch in ".-" else "." for ch in owner).strip(".")
+    return f"pipeline/{safe or 'pipeline'}.jsonl"
+
+
+def envelopes_digest(rows: list[dict[str, Any]]) -> str:
+    digest = hashlib.sha256()
+    for row in rows:
+        payload = row.get("payload") if isinstance(row, dict) else row
+        digest.update(json.dumps(payload, sort_keys=True, default=str).encode("utf-8"))
+        digest.update(b"\n")
+    return digest.hexdigest()
+
+
+def artefact_key(owner: str, runtime_digest: str, *, version: str = ARTEFACT_VERSION) -> str:
+    blob = f"{owner}|{version}|{runtime_digest}".encode()
+    return hashlib.sha256(blob).hexdigest()
+
+
+def tree_of(topic: str) -> str:
+    name = str(topic or "")
+    if name.startswith(PIPELINE_PREFIXES):
+        return "pipeline"
+    if name.startswith(RUNTIME_PREFIXES):
+        return "runtime"
+    if name.startswith("view."):
+        return "control"
+    return "pipeline" if name else "runtime"
+
+
+def is_ui_trace_dir(path: Path) -> bool:
+    manifest = path / "manifest.json"
+    if not manifest.is_file():
+        return False
+    try:
+        doc = json.loads(manifest.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return False
+    return isinstance(doc, dict) and doc.get("kind") == KIND
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    if not path.is_file():
+        return []
+    rows: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        row = json.loads(line)
+        if isinstance(row, dict):
+            rows.append(row)
+    return rows
+
+
+def split_envelopes(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
+    out: dict[str, list[dict[str, Any]]] = {"runtime": [], "pipeline": [], "control": []}
+    for row in rows:
+        tree = tree_of(str(row.get("topic") or ""))
+        out.setdefault(tree, []).append(row)
+    return out
+
+
+def _stamp_rows(rows: list[dict[str, Any]], *, tree: str) -> list[dict[str, Any]]:
+    from mas.lab.controller.ui_stream.artefact_bind import link_for_topic
+
+    stamped: list[dict[str, Any]] = []
+    for row in rows:
+        item = dict(row)
+        topic = str(item.get("topic") or "")
+        link = link_for_topic(topic, item.get("ref") if isinstance(item.get("ref"), dict) else {}, complete=item.get("complete") if isinstance(item.get("complete"), bool) else None)
+        owner = link.name if tree == "pipeline" else "events"
+        item.setdefault("artefact", owner)
+        item.setdefault("stream", link.stream)
+        item.setdefault("uri", link.uri)
+        item.setdefault("path", link.path)
+        item.setdefault("inputs", list(link.inputs))
+        item.setdefault("outputs", list(link.outputs))
+        if item.get("complete") is None and link.complete is not None:
+            item["complete"] = link.complete
+        stamped.append(item)
+    return stamped
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        "".join(json.dumps(row, ensure_ascii=False, default=str) + "\n" for row in rows),
+        encoding="utf-8",
+    )
+
+
+def write_ui_trace(dest: Path, envelopes: list[dict[str, Any]], *, extra: dict[str, Any] | None = None) -> Path:
+    dest.mkdir(parents=True, exist_ok=True)
+    buckets = split_envelopes(envelopes)
+    files = []
+    from mas.lab.controller.ui_stream.artefact_bind import graph_from_steps, link_for, shard_is_complete
+
+    runtime = _stamp_rows(buckets.get("runtime") or [], tree="runtime")
+    runtime_digest = envelopes_digest(runtime)
+    extra_doc = dict(extra or {})
+    ref = extra_doc.get("ref") if isinstance(extra_doc.get("ref"), dict) else {}
+    runtime_link = link_for("events", ref, tree="runtime", digest=runtime_digest, complete=shard_is_complete(runtime, tree="runtime"))
+    _write_jsonl(dest / "runtime.jsonl", runtime)
+    files.append(
+        {
+            **runtime_link.as_dict(),
+            "path": "runtime.jsonl",
+            "n": len(runtime),
+            "digest": runtime_digest,
+        }
+    )
+    by_owner: dict[str, list[dict[str, Any]]] = {}
+    for row in _stamp_rows(buckets.get("pipeline") or [], tree="pipeline"):
+        by_owner.setdefault(str(row.get("artefact") or "pipeline"), []).append(row)
+    for owner, rows in sorted(by_owner.items()):
+        rel = artefact_filename(owner)
+        _write_jsonl(dest / rel, rows)
+        complete = shard_is_complete(rows, tree="pipeline")
+        link = link_for(
+            owner,
+            ref,
+            tree="pipeline",
+            complete=complete,
+            digest=runtime_digest,
+            key=artefact_key(owner, runtime_digest),
+        )
+        files.append({**link.as_dict(), "path": rel, "n": len(rows), "input_digest": runtime_digest})
+    manifest = {
+        "kind": KIND,
+        "version": VERSION,
+        "runtime_digest": runtime_digest,
+        "graph": extra_doc.pop("graph", None) or graph_from_steps(extra_doc.pop("steps", None)),
+        "files": files,
+        **extra_doc,
+    }
+    (dest / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    return dest
+
+
+def read_ui_trace(dest: Path, *, trees: tuple[str, ...] | None = None) -> dict[str, list[dict[str, Any]]]:
+    dest = dest.resolve()
+    if dest.is_file() and dest.name == "manifest.json":
+        dest = dest.parent
+    if not is_ui_trace_dir(dest):
+        raise FileNotFoundError(f"not a ui-trace directory: {dest}")
+    wanted = set(trees) if trees else {"runtime", "pipeline"}
+    manifest = json.loads((dest / "manifest.json").read_text(encoding="utf-8"))
+    out: dict[str, list[dict[str, Any]]] = {}
+    for spec in manifest.get("files") or []:
+        tree = str(spec.get("tree") or "")
+        if tree not in wanted:
+            continue
+        rel = str(spec.get("path") or "")
+        if not rel:
+            continue
+        out.setdefault(tree, []).extend(load_jsonl(dest / rel))
+    for tree in wanted:
+        out.setdefault(tree, [])
+    return out
+
+
+def cached_pipeline_files(dest: Path, runtime_digest: str) -> list[dict[str, Any]]:
+    """Pipeline shards whose node key still matches ``runtime_digest``."""
+    dest = dest.resolve()
+    if dest.is_file() and dest.name == "manifest.json":
+        dest = dest.parent
+    manifest = json.loads((dest / "manifest.json").read_text(encoding="utf-8"))
+    hits: list[dict[str, Any]] = []
+    for spec in manifest.get("files") or []:
+        if str(spec.get("tree") or "") != "pipeline":
+            continue
+        owner = str(spec.get("artefact") or "")
+        if not owner:
+            continue
+        if spec.get("complete") is not True:
+            continue
+        if spec.get("input_digest") != runtime_digest:
+            continue
+        if spec.get("key") != artefact_key(owner, runtime_digest):
+            continue
+        hits.append(spec)
+    return hits
+
+
+def discover_ui_traces(target: Path) -> list[tuple[str, Path]]:
+    """Return (scenario, ui-trace dir) pairs under a lab or a single dir."""
+    target = target.expanduser().resolve()
+    if is_ui_trace_dir(target):
+        return [(target.name, target)]
+    if target.is_file():
+        return []
+    hits: list[tuple[str, Path]] = []
+    seen: set[Path] = set()
+    for manifest in sorted(target.rglob("manifest.json")):
+        parent = manifest.parent
+        if parent in seen or not is_ui_trace_dir(parent):
+            continue
+        seen.add(parent)
+        scenario = parent.parent.name if parent.name in {"ui-trace", "trace"} else parent.name
+        hits.append((scenario, parent))
+    return hits

mas/lab/controller/uploads.py

@@ -0,0 +1,61 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Multipart uploads referenced by opaque upload_id (never a client path)."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class Upload:
+    upload_id: str
+    name: str
+    size: int
+    path: Path
+
+    def public_dict(self) -> dict:
+        return {"upload_id": self.upload_id, "name": self.name, "size": self.size}
+
+
+class UploadStore:
+    def __init__(self, root: Path):
+        self.root = root
+        self.root.mkdir(parents=True, exist_ok=True)
+        self._items: dict[str, Upload] = {}
+
+    def save(self, filename: str, data: bytes) -> Upload:
+        from mas.lab.controller.constants import MAX_UPLOAD_BYTES
+
+        if len(data) > MAX_UPLOAD_BYTES:
+            raise ValueError(f"upload exceeds {MAX_UPLOAD_BYTES} bytes")
+        upload_id = str(uuid.uuid4())
+        safe_name = Path(filename).name or "upload"
+        path = self.root / upload_id
+        path.write_bytes(data)
+        item = Upload(upload_id=upload_id, name=safe_name, size=len(data), path=path)
+        self._items[upload_id] = item
+        return item
+
+    def get(self, upload_id: str) -> Upload | None:
+        return self._items.get(upload_id)
+
+
+_store: UploadStore | None = None
+
+
+def get_upload_store() -> UploadStore:
+    global _store
+    if _store is None:
+        from mas.lab.controller.config import controller_dir
+
+        _store = UploadStore(controller_dir() / "state" / "uploads")
+    return _store
+
+
+def init_upload_store(root: Path) -> UploadStore:
+    global _store
+    _store = UploadStore(root)
+    return _store