mas-lab-component-core 0.2.0__py3-none-any.whl

mas/lab/artifacts.py

@@ -0,0 +1,282 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Typed artifact value objects for the mas-lab processor/pipeline system.
+
+Artifacts are the strongly-typed values that flow between :class:`Processor`
+instances and pipeline steps.  Every artifact has:
+
+* a ``kind`` class-variable — the canonical type name used in YAML / CLI
+* an optional ``path`` — when the artifact is disk-backed (file)
+* an optional ``data`` — when the artifact is in-memory only
+* a ``meta`` dict — provenance, run_id, timestamps, …
+
+Hierarchy::
+
+    Artifact
+    ├── Trajectory               ← raw JSONL event trace
+    │   └── AnnotatedTrajectory  ← + highlights + analyst notes
+    ├── PlotFile                 ← rendered HTML / SVG / PNG
+    ├── NormalizedTrace          ← normalized manifold trace (JSONL)
+    └── KnowledgeGraph           ← JSON-LD graph file
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar, Dict, List, Optional, Type
+
+
+# ---------------------------------------------------------------------------
+# Base
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Artifact:
+    """Base value object for all typed artifacts.
+
+    Parameters
+    ----------
+    path:
+        File path when the artifact lives on disk.
+    data:
+        In-memory payload (list, dict, str, …) when the artifact is not
+        backed by a file, or has already been loaded from one.
+    meta:
+        Arbitrary provenance metadata (run_id, timestamps, labels …).
+    """
+
+    kind: ClassVar[str] = "artifact"
+
+    path: Optional[Path] = None
+    data: Any = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    def is_file(self) -> bool:
+        """True iff ``path`` points to an existing file."""
+        return self.path is not None and Path(self.path).exists()
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "kind": self.kind,
+            "path": str(self.path) if self.path else None,
+            "meta": self.meta,
+        }
+
+    def __repr__(self) -> str:
+        loc = f"path={self.path}" if self.path else "in-memory"
+        return f"<{self.__class__.__name__} {loc}>"
+
+
+# ---------------------------------------------------------------------------
+# Trajectory family
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Trajectory(Artifact):
+    """Raw MAS execution trace — a list of OTEL-style event dicts.
+
+    The ``events`` list is populated lazily: calling :meth:`load` reads the
+    JSONL file at ``path`` if ``events`` is empty.
+    """
+
+    kind: ClassVar[str] = "trajectory"
+
+    events: List[Dict[str, Any]] = field(default_factory=list)
+    run_id: str = ""
+
+    def load(self) -> "Trajectory":
+        """Load events from ``path`` (or ``run_id``) into ``events`` (idempotent)."""
+        if not self.events:
+            source = self.path or (self.run_id or None)
+            if source:
+                from mas.lab.plots.trajectory import load_trace  # lazy import
+                self.events = load_trace(source)
+        return self
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d["run_id"] = self.run_id
+        return d
+
+
+@dataclass
+class AnnotatedTrajectory(Trajectory):
+    """Trajectory enriched with analyst highlights and free-form notes.
+
+    Parameters
+    ----------
+    highlights:
+        Each entry is a correlation-id prefix (e.g. ``"f19445b6"``) **or** a
+        1-based delegation index (e.g. ``"3"``).
+    notes:
+        List of ``{"index": int, "note": str}`` dicts — one per highlighted
+        delegation.  Rendered as tooltips or annotations in plots.
+    """
+
+    kind: ClassVar[str] = "annotated_trajectory"
+
+    highlights: List[str] = field(default_factory=list)
+    notes: List[Dict[str, Any]] = field(default_factory=list)
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d["highlights"] = self.highlights
+        d["notes"] = self.notes
+        return d
+
+
+# ---------------------------------------------------------------------------
+# Plot artifacts
+# ---------------------------------------------------------------------------
+
+@dataclass
+class PlotFile(Artifact):
+    """A rendered visualisation written to disk (HTML, SVG, PNG …)."""
+
+    kind: ClassVar[str] = "plot_file"
+
+    format: str = "html"
+    """One of: html, svg, mermaid, png."""
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d["format"] = self.format
+        return d
+
+
+# ---------------------------------------------------------------------------
+# Knowledge-graph family
+# ---------------------------------------------------------------------------
+
+@dataclass
+class NormalizedTrace(Artifact):
+    """Normalized manifold trace (JSONL) ready for KG construction."""
+
+    kind: ClassVar[str] = "normalized_trace"
+
+    run_id: str = ""
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d["run_id"] = self.run_id
+        return d
+
+
+@dataclass
+class KnowledgeGraph(Artifact):
+    """JSON-LD knowledge graph produced by the manifold normalizer."""
+
+    kind: ClassVar[str] = "knowledge_graph"
+
+    run_id: str = ""
+
+    def load_json(self) -> Any:
+        """Read and return the JSON-LD dict from ``path``."""
+        if self.path:
+            return json.loads(Path(self.path).read_text(encoding="utf-8"))
+        return self.data
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d["run_id"] = self.run_id
+        return d
+
+
+# ---------------------------------------------------------------------------
+# MCE session-metrics artefacts
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SessionMetrics(Artifact):
+    """MCE metric scores for one MAS session.
+
+    Written as ``metrics.json`` next to ``run_info.json`` in each run folder
+    (``item{N}/r{K}/metrics.json``) by :class:`EvalMceBatchStep`.
+
+    Parameters
+    ----------
+    item_id:
+        Dataset item identifier (e.g. ``"1"``).
+    scenario:
+        Benchmark scenario name (e.g. ``"baseline"``).
+    session:
+        Session-level scores keyed by metric_id.
+        Each value is ``{"value": float|None, "reasoning": str, "error": str|None}``.
+    agents:
+        Per-agent scores (empty in schema_version="1").
+        Reserved for future extension.
+    schema_version:
+        Artefact format version.  Increment on breaking changes.
+    """
+
+    kind: ClassVar[str] = "session_metrics"
+
+    item_id:        str = ""
+    scenario:       str = ""
+    session:        Dict[str, Any] = field(default_factory=dict)
+    agents:         Dict[str, Any] = field(default_factory=dict)
+    schema_version: str = "1"
+
+    def as_dict(self) -> Dict[str, Any]:
+        d = super().as_dict()
+        d.update({
+            "schema_version": self.schema_version,
+            "item_id":  self.item_id,
+            "scenario": self.scenario,
+            "session":  self.session,
+            "agents":   self.agents,
+        })
+        return d
+
+    @classmethod
+    def from_json(cls, path: Path) -> "SessionMetrics":
+        """Load a ``metrics.json`` file from *path*."""
+        doc = json.loads(Path(path).read_text(encoding="utf-8"))
+        return cls(
+            path=path,
+            item_id=doc.get("item_id", ""),
+            scenario=doc.get("scenario", ""),
+            session=doc.get("session", {}),
+            agents=doc.get("agents", {}),
+            schema_version=doc.get("schema_version", "1"),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+
+_ARTIFACT_REGISTRY: Dict[str, Type[Artifact]] = {
+    "artifact":              Artifact,
+    "trajectory":            Trajectory,
+    "annotated_trajectory":  AnnotatedTrajectory,
+    "plot_file":             PlotFile,
+    "normalized_trace":      NormalizedTrace,
+    "knowledge_graph":       KnowledgeGraph,
+    "session_metrics":       SessionMetrics,
+}
+
+
+def register_artifact(cls: Type[Artifact]) -> Type[Artifact]:
+    """Class decorator — register a custom Artifact subclass by its ``kind``."""
+    _ARTIFACT_REGISTRY[cls.kind] = cls
+    return cls
+
+
+def artifact_from_dict(d: Dict[str, Any]) -> Artifact:
+    """Reconstruct an Artifact from a dict produced by :meth:`Artifact.as_dict`."""
+    cls = _ARTIFACT_REGISTRY.get(d.get("kind", "artifact"), Artifact)
+    path = Path(d["path"]) if d.get("path") else None
+    inst = cls.__new__(cls)
+    # call dataclass __init__ with just the base fields; subclass extras default
+    Artifact.__init__(inst, path=path, meta=d.get("meta", {}))
+    if hasattr(inst, "run_id") and "run_id" in d:
+        inst.run_id = d["run_id"]
+    if hasattr(inst, "highlights") and "highlights" in d:
+        inst.highlights = d["highlights"]
+    if hasattr(inst, "notes") and "notes" in d:
+        inst.notes = d["notes"]
+    if hasattr(inst, "format") and "format" in d:
+        inst.format = d["format"]
+    return inst

mas/lab/cli/__init__.py

@@ -0,0 +1,115 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""mas-lab click CLI — ``app`` is the single entry point.
+
+Import and use programmatically::
+
+    from mas.lab.cli import app
+    from click.testing import CliRunner
+
+    result = CliRunner().invoke(app, ["benchmark", "list"])
+"""
+from __future__ import annotations
+
+import importlib
+import logging
+
+import click
+import mas.third_party_otel  # noqa: F401
+from dotenv import find_dotenv, load_dotenv
+from mas.lab.cli.commands import check
+from mas.lab.cli.commands import config as _config_mod
+from mas.lab.cli.commands import init as _init_mod
+from mas.lab.cli.commands import validate as _validate_mod
+from mas.lab.cli.extensions import register_extension_components
+
+
+@click.group()
+@click.option("-v", "--verbose", count=True, default=0,
+              help="Verbosity: -v exchange log, -vv debug, -vvv full")
+@click.pass_context
+def app(ctx: click.Context, verbose: int) -> None:
+    """MAS Lab — evaluation, experimentation, and benchmarking tools.
+
+    \b
+    Command groups:
+      benchmark     Run experiments, list results, manage steps
+      control       Start/stop the controller daemon (UI + workers)
+      worker        List and manage background workers
+      plot          Generate trajectory and communication diagrams
+      run           Execute processors and pipelines
+      pipe          Stream-oriented processing
+
+    \b
+    Utilities:
+      check         Validate a MAS configuration
+      config        Show resolved paths and environment configuration
+      serve         Start the benchmark UI server
+      telemetry     Convert, verify, push, and dump traces (library-telemetry)
+      kg            Normalize, validate, push, and dump graphs (library-kg)
+      eval          Score traces with MCE metrics (library-eval)
+      validate      Validate YAML manifests
+
+        \b
+        Optional extensions are discovered via the ``mas.lab.cli.components``
+        entry-point group and loaded automatically when their packages are installed.
+
+    \b
+    Global options:
+      -v            Show exchange log
+      -vv           Debug output
+      -vvv          Full trace output
+    """
+    ctx.ensure_object(dict)
+    ctx.obj["verbose"] = verbose
+
+    try:
+        from mas.ctl.logging_setup import setup_logging
+        setup_logging(verbosity=verbose)
+    except ImportError:
+        level = (logging.DEBUG if verbose >= 2
+                 else logging.INFO if verbose >= 1
+                 else logging.WARNING)
+        logging.basicConfig(level=level, format="%(message)s")
+
+    # Workspace config — loaded once, shared via ctx.obj
+    try:
+        from mas.lab.workspace import WorkspaceConfig
+        ctx.obj["workspace"] = WorkspaceConfig.load()
+    except ImportError:
+        ctx.obj["workspace"] = None
+
+    load_dotenv(find_dotenv(usecwd=True))
+
+
+def _add_if_importable(name: str, module: str, attr: str) -> None:
+    """Attach a command that lives in an optional lab component."""
+    try:
+        mod = importlib.import_module(module)
+    except ImportError:
+        return
+    app.add_command(getattr(mod, attr), name=name)
+
+
+app.add_command(check.check_cmd, name="check")
+app.add_command(check.check_config_cmd, name="check-config")
+app.add_command(_config_mod.config_cmd, name="config")
+app.add_command(_init_mod.init_cmd, name="init")
+app.add_command(_validate_mod.validate_cmd, name="validate")
+_add_if_importable("serve", "mas.lab.cli.commands.serve", "serve_cmd")
+_add_if_importable("benchmark", "mas.lab.cli.commands.benchmark", "group")
+_add_if_importable("plot", "mas.lab.cli.commands.plot", "plot_group")
+_add_if_importable("telemetry", "mas.lab.cli.commands.telemetry", "telemetry_group")
+_add_if_importable("run", "mas.lab.cli.commands.run", "run_group")
+_add_if_importable("pipe", "mas.lab.cli.commands.pipe", "pipe_group")
+
+# Register optional command groups from installed extension packages.
+register_extension_components(app)
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    app()

mas/lab/cli/commands/__init__.py

@@ -0,0 +1,3 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Click command modules for mas-lab."""

mas/lab/cli/commands/benchmark/__init__.py

@@ -0,0 +1,73 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Click wrappers for ``mas-lab benchmark`` subcommands."""
+
+from __future__ import annotations
+
+import click
+
+from .analyze import analyze_cmd
+from .artifacts import artifact_group
+from .clean import clean_cmd
+from .daemon import daemon_group
+from .export_cmd import export_cmd
+from .follow import follow_cmd
+from .import_cmd import import_cmd
+from .import_hf import import_hf_cmd
+from .list_cmd import list_cmd
+from .migrate import migrate_cmd
+from .pipeline import pipeline_group
+from .publish_hf import publish_hf_cmd
+from .rename import rename_cmd
+from .run import run_cmd
+from .show import show_cmd
+from .step import step_group
+from .step_info import step_info_cmd
+from .update import update_cmd
+
+
+@click.group("benchmark")
+def group() -> None:
+    """Benchmark management — run, list, show, update, analyze, step."""
+
+
+group.add_command(run_cmd)
+group.add_command(list_cmd)
+group.add_command(show_cmd)
+group.add_command(artifact_group)
+group.add_command(update_cmd)
+group.add_command(rename_cmd)
+group.add_command(migrate_cmd)
+group.add_command(analyze_cmd)
+group.add_command(export_cmd)
+group.add_command(import_cmd)
+group.add_command(publish_hf_cmd)
+group.add_command(import_hf_cmd)
+group.add_command(follow_cmd)
+group.add_command(clean_cmd)
+group.add_command(step_info_cmd)
+group.add_command(step_group)
+group.add_command(pipeline_group)
+group.add_command(daemon_group)
+
+__all__ = [
+    "group",
+    "run_cmd",
+    "list_cmd",
+    "show_cmd",
+    "artifact_group",
+    "update_cmd",
+    "rename_cmd",
+    "migrate_cmd",
+    "analyze_cmd",
+    "export_cmd",
+    "import_cmd",
+    "publish_hf_cmd",
+    "import_hf_cmd",
+    "follow_cmd",
+    "clean_cmd",
+    "step_info_cmd",
+    "step_group",
+    "pipeline_group",
+    "daemon_group",
+]

mas/lab/cli/commands/benchmark/analyze.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""``mas-lab benchmark analyze`` command."""
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import click
+
+
+@click.command("analyze")
+@click.argument("benchmark_id")
+@click.option("--experiment-yaml", type=Path, default=None,
+              help="Path to experiment.yaml (auto-detected if omitted).")
+@click.option("-o", "--output-dir", type=Path, default=None)
+def analyze_cmd(benchmark_id: str, experiment_yaml: Path | None,
+                output_dir: Path | None) -> None:
+    """Regenerate statistics and plots from existing results."""
+    from mas.lab.benchmark.cli import analyze_command
+
+    args = SimpleNamespace(benchmark_id=benchmark_id,
+                           experiment_yaml=experiment_yaml,
+                           output_dir=output_dir)
+    raise SystemExit(analyze_command(args))

mas/lab/cli/commands/benchmark/artifacts.py

@@ -0,0 +1,57 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""``mas-lab benchmark artifact`` commands."""
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import click
+
+
+@click.group("artifact")
+def artifact_group() -> None:
+    """List and show artifacts for an experiment."""
+
+
+_LEVEL = click.Choice(["application", "scenario", "test", "run"], case_sensitive=False)
+
+
+@artifact_group.command("list")
+@click.argument("target")
+@click.option("-t", "--type", "artifact_type", default=None, metavar="TYPE",
+              help="Filter by artifact type.")
+@click.option("-l", "--level", "level", type=_LEVEL, default=None,
+              help="Filter to one hierarchy level.")
+@click.option("-v", "--verbose", is_flag=True, default=False,
+              help="Include the type description.")
+def artifact_list_cmd(
+    target: str,
+    artifact_type: str | None,
+    level: str | None,
+    verbose: bool,
+) -> None:
+    """List artifacts declared in the experiment, and whether each was generated.
+
+    TARGET is ``last``, a benchmark id, or an experiment YAML. The run directory
+    is resolved from the last-run pointer or the experiment identity — not passed
+    as an argument.
+    """
+    from mas.lab.benchmark.cli import artifact_list_command
+
+    raise SystemExit(artifact_list_command(SimpleNamespace(
+        target=target,
+        artifact_type=artifact_type,
+        level=level,
+        verbose=verbose,
+    )))
+
+
+@artifact_group.command("show")
+@click.argument("artifact_id")
+def artifact_show_cmd(artifact_id: str) -> None:
+    """Show one generated artifact by its 8-character id."""
+    from mas.lab.benchmark.cli import show_artifact_by_id_command
+
+    raise SystemExit(show_artifact_by_id_command(SimpleNamespace(
+        artifact_id=artifact_id,
+    )))

mas/lab/cli/commands/benchmark/clean.py

@@ -0,0 +1,56 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""``mas-lab benchmark clean`` command."""
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import click
+
+
+@click.command("clean")
+@click.argument("experiment_yaml", type=Path, metavar="EXPERIMENT_YAML")
+@click.option("--scenario", "scenarios", multiple=True, metavar="SCENARIO_ID",
+              help="Scenario ID(s) to clean. Repeatable. If omitted, ALL scenarios are cleaned.")
+@click.option("--dry-run", is_flag=True, default=False,
+              help="Print what would be deleted without deleting anything.")
+@click.option("--keep-traces", is_flag=True, default=False,
+              help="Delete only the scenario run dirs; keep the global trace-cache entries.")
+@click.option("-o", "--output-dir", type=Path, default=None,
+              help="Override the output dir directly (use when experiment.yaml is unavailable; pass '-' as EXPERIMENT_YAML).")  # noqa: E501
+def clean_cmd(
+    experiment_yaml: Path,
+    scenarios: tuple[str, ...],
+    dry_run: bool,
+    keep_traces: bool,
+    output_dir: Path | None,
+) -> None:
+    """Delete cached runs for selected scenarios so they are re-executed on the next run.
+
+    Examples:
+
+    \b
+    # Clean only the ToT scenario
+    mas-lab benchmark clean labs/design-space.lab/01-design-patterns/experiment.yaml \\
+        --scenario pattern-tree-of-thoughts
+
+    \b
+    # Dry-run: see what would be removed
+    mas-lab benchmark clean labs/design-space.lab/01-design-patterns/experiment.yaml \\
+        --scenario pattern-tree-of-thoughts --dry-run
+
+    \b
+    # Clean all scenarios in an experiment
+    mas-lab benchmark clean labs/design-space.lab/01-design-patterns/experiment.yaml
+    """
+    from mas.lab.benchmark.cli import clean_command
+
+    args = SimpleNamespace(
+        experiment_yaml=experiment_yaml,
+        scenarios=list(scenarios) if scenarios else None,
+        dry_run=dry_run,
+        keep_traces=keep_traces,
+        output_dir=output_dir,
+    )
+    raise SystemExit(clean_command(args))