mas-lab-library-lab 0.2.0__tar.gz

mas_lab_library_lab-0.2.0/.gitignore

@@ -0,0 +1,86 @@
+*.pyc
+__pycache__/
+*.egg-info/
+.DS_Store
+dist/
+build/
+.venv/
+.demo-ui/
+.demo-ui-mock/
+.env
+.env.local
+.env.*.local
+config.yaml
+!lab/src/mas/lab/templates/**/config.yaml
+!docs/tutorials/**/config.yaml
+!labs/**/config.yaml
+!library-samples/sample-workspace/config.yaml
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.test-venv/
+node_modules/
+*.log
+docs/tutorials/**/logs/
+.hypothesis/
+compliance-report.json
+COMPLIANCE_REPORT.md
+compliance/results/
+
+# MkDocs build output
+site/
+
+.vscode/
+.cursor/
+.claude/
+
+# Test / coverage artifacts
+.coverage
+coverage.xml
+coverage.json
+**/coverage.json
+
+# LaTeX build artifacts
+*.fdb_latexmk
+*.fls
+*.aux
+*.out
+*.toc
+*.synctex.gz
+
+# Run output / artifacts
+**/output/*.jsonl
+**/output/benchmark
+**/output/meta.json
+**/output/current_run_id.txt
+**/output/*/
+.tutorial-cache/
+tests/.tutorial-engine-cache/
+.demo-cache/
+# Local mas-ctl chat traces next to an agent (fixtures live under fixtures/ or T01 traces/)
+docs/tutorials/**/agents/**/traces/
+
+# Backup files
+*.bak
+*.bak2
+
+# Legacy dead code (removed from repo)
+lab/src/mas/lab/_cli_legacy.py
+.deepeval/
+~/.mas/
+
+
+# Docker local secrets + data (never commit)
+docker/.env
+docker/data/*
+!docker/data/.gitkeep
+
+# Local protocol compliance checkouts and reports
+compliance/vendor/
+
+# Local harness playbooks (not published)
+AGENTS.md
+# .agents/
+COVERAGE_REPORT.md
+MANIFEST_SCHEMA_MAPPING.md
+.lycheecache

mas_lab_library_lab-0.2.0/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.5
+Name: mas-lab-library-lab
+Version: 0.2.0
+Summary: MAS Lab provider library with public eval plugins.
+Author: MAS Framework Contributors
+License: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: mas-lab-component-bench
+Requires-Dist: mas-lab-component-core
+Description-Content-Type: text/markdown
+
+<!--
+  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+  SPDX-License-Identifier: Apache-2.0
+-->
+# library-lab
+
+Public plugin library for MAS Lab extensions.
+
+## Registered eval providers
+
+- `mcev1` -> `mas.library.lab.eval.mcev1:MCEv1Provider`
+
+## Documentation
+
+- User guide: `docs/user-guide.md`
+- Developer guide: `docs/developer-guide.md`

mas_lab_library_lab-0.2.0/README.md

@@ -0,0 +1,16 @@
+<!--
+  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+  SPDX-License-Identifier: Apache-2.0
+-->
+# library-lab
+
+Public plugin library for MAS Lab extensions.
+
+## Registered eval providers
+
+- `mcev1` -> `mas.library.lab.eval.mcev1:MCEv1Provider`
+
+## Documentation
+
+- User guide: `docs/user-guide.md`
+- Developer guide: `docs/developer-guide.md`

mas_lab_library_lab-0.2.0/docs/developer-guide.md

@@ -0,0 +1,33 @@
+<!--
+  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+  SPDX-License-Identifier: Apache-2.0
+-->
+# library-lab Developer Guide
+
+This guide covers extending public lab-oriented providers and utilities.
+
+## Package layout
+
+- `src/mas/library/lab/eval/`: eval providers
+- `src/mas/library/lab/plots.py`: plotting helpers
+- `src/mas/library/lab/__init__.py`: public exports
+
+## Extend eval providers
+
+1. Add provider module under `eval/`.
+2. Expose stable registration identifier.
+3. Keep metric output schema explicit and documented.
+4. Add tests validating provider registration and score shape.
+
+## Extend plotting utilities
+
+- Keep plotting helpers pure and reusable.
+- Avoid side effects that break notebook or headless usage.
+- Document required dataframe columns and expected types.
+
+## Release checklist
+
+- Public exports updated
+- Provider IDs documented
+- Backward compatibility impact reviewed
+- Examples updated to include new provider usage

mas_lab_library_lab-0.2.0/docs/user-guide.md

@@ -0,0 +1,34 @@
+<!--
+  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+  SPDX-License-Identifier: Apache-2.0
+-->
+# library-lab User Guide
+
+`library-lab` ships public extensions for `mas-lab`, including eval providers and plotting helpers.
+
+## Install
+
+```bash
+uv pip install -e library-lab
+```
+
+## What is included
+
+- Public eval provider registration (`mcev1`)
+- Plot helper exports such as `execution_chain_graph`
+
+## Typical usage
+
+- Use this package when your experiments depend on public eval plugins.
+- Keep experiment config aligned with provider names exported by this package.
+
+## Quick verification
+
+1. Install package in editable mode.
+2. Run a benchmark referencing the provider.
+3. Confirm provider-specific metrics are present in artifacts.
+
+## Troubleshooting
+
+- Provider not detected: verify package is installed in active env.
+- Missing metric output: check provider identifier and experiment pipeline step config.

mas_lab_library_lab-0.2.0/library.yaml

@@ -0,0 +1,284 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+# MAS Library Metadata — Lab
+# Pipeline steps, artifact types, and artifact codecs used by the
+# lab/components/bench pipeline engine. Bench ships the engine only (Pipeline,
+# PipelineExecutor, PipelineStep, caching, dependency resolution); every
+# concrete step, artifact type, and codec implementation lives here and
+# is registered through the runtime plugin registry — see
+# runtime/docs/plugin-registry-manifests.md.
+#
+# This file is both the library manifest (kind: Library — metadata read by
+# mas.library_catalog: discover_apps/discover_datasets/discover_tools) and,
+# via the types:/plugins: keys below, a plugin manifest fed directly into
+# mas.runtime.registry.bootstrap.register_manifest_data(). A library only
+# needs a separate plugins/*.plugins.yaml file (referenced from a
+# plugin_manifests: catalog here, or picked up by the plugins/ directory
+# scan) if it wants to split its plugin declarations across multiple files.
+# See docs/schemas/library.schema.yaml for the full schema.
+
+apiVersion: mas/v1
+kind: Library
+
+name: lab
+description: >
+  Pipeline step and artifact codec plugins for lab/components/bench, plus the
+  MCE v1 eval provider and the standard plot library.
+version: "0.1.0"
+
+module_base: mas.library.lab
+
+types:
+  - step
+  - codec
+  - design_pattern
+  - artifact
+
+plugins:
+  - type: step
+    name: analysis
+    module: mas.library.lab.steps.data.analysis
+    class: AnalysisStep
+  - type: step
+    name: annotate_metrics
+    module: mas.library.lab.steps.eval.annotate_metrics
+    class: AnnotateMetricsStep
+  - type: step
+    name: ci_plot
+    module: mas.library.lab.steps.viz.ci_plot
+    class: CIPlotStep
+  - type: step
+    name: collect_dataframe
+    module: mas.library.lab.steps.data.collect_dataframe
+    class: CollectDataFrameStep
+  - type: step
+    name: collect_metrics
+    module: mas.library.lab.steps.eval.collect_metrics
+    class: CollectMetricsStep
+  - type: step
+    name: compute_ci
+    module: mas.library.lab.steps.eval.compute_ci
+    class: ComputeCIStep
+  - type: step
+    name: compute_drift
+    module: mas.library.lab.steps.eval.compute_drift
+    class: ComputeDriftStep
+  - type: step
+    name: dataset
+    module: mas.library.lab.steps.data.dataset
+    class: DatasetStep
+  - type: step
+    name: deserialize
+    module: mas.library.lab.steps.data.deserialize
+    class: DeserializeStep
+  - type: step
+    name: diff_trajectories
+    module: mas.library.lab.steps.data.diff_trajectories
+    class: DiffTrajectoriesStep
+  - type: step
+    name: embed_trajectories
+    module: mas.library.lab.steps.data.embed_trajectories
+    class: EmbedStep
+  - type: step
+    name: eval_mce
+    module: mas.library.lab.steps.eval.mce
+    class: EvalMceStep
+  - type: step
+    name: events_to_otel
+    module: mas.library.lab.steps.services.events_to_otel
+    class: EventsToOtelStep
+  - type: step
+    name: experiment
+    module: mas.library.lab.steps.data.experiment
+    class: ExperimentStep
+  - type: step
+    name: export_otel
+    module: mas.library.lab.steps.services.export_otel
+    class: ExportOtelStep
+  - type: step
+    name: extract_mealy_stats
+    module: mas.library.lab.steps.extract.mealy_stats
+    class: ExtractMealyStatsStep
+  - type: step
+    name: extract_sys_stats
+    module: mas.library.lab.steps.extract.sys_stats
+    class: ExtractSysStatsStep
+  - type: step
+    name: extract_trace_stats
+    module: mas.library.lab.steps.extract.trace_stats
+    class: ExtractTraceStatsStep
+  - type: step
+    name: file_tail
+    module: mas.library.lab.steps.services.file_tail
+    class: FileTailStep
+  - type: step
+    name: shell
+    module: mas.library.lab.steps.services.shell
+    class: ShellStep
+  - type: step
+    name: extract_trajectories
+    module: mas.library.lab.steps.extract.trajectories
+    class: ExtractTrajectoriesStep
+  - type: step
+    name: gather_level
+    module: mas.library.lab.steps.data.gather_level
+    class: GatherLevelStep
+  - type: step
+    name: generate_dataset
+    module: mas.library.lab.steps.data.generate_dataset
+    class: GenerateDatasetStep
+  - type: step
+    name: ggplot
+    module: mas.library.lab.steps.viz.plot_ggplot
+    class: PlotPipelineStep
+  - type: step
+    name: join_dataframe
+    module: mas.library.lab.steps.data.join_dataframe
+    class: JoinDataFrameStep
+  - type: step
+    name: metrics_comparison_plot
+    module: mas.library.lab.steps.viz.metrics_comparison_plot
+    class: MetricsComparisonPlotStep
+  - type: step
+    name: metrics_to_dataframe
+    module: mas.library.lab.steps.eval.metrics_to_dataframe
+    class: MetricsToDataFrameStep
+  - type: step
+    name: pipeline_diagram
+    module: mas.library.lab.steps.viz.pipeline_diagram
+    class: PipelineDiagramStep
+  - type: step
+    name: plot
+    module: mas.library.lab.steps.viz.plot
+    class: PlotStep
+  - type: step
+    name: plot_communication_flow
+    module: mas.library.lab.steps.viz.plot_communication_flow
+    class: PlotCommunicationFlowStep
+  - type: step
+    name: plot_message_graph
+    module: mas.library.lab.steps.viz.plot_message_graph
+    class: PlotMessageGraphStep
+  - type: step
+    name: plot_multilevel_trajectory
+    module: mas.library.lab.steps.viz.plot_multilevel_trajectory
+    class: PlotMultilevelTrajectoryStep
+  - type: step
+    name: plot_trajectory
+    module: mas.library.lab.steps.viz.plot_trajectory
+    class: PlotTrajectoryStep
+  - type: step
+    name: plotnine
+    module: mas.library.lab.steps.viz.plotnine
+    class: PlotNineStep
+  - type: step
+    name: processor
+    module: mas.library.lab.steps.data.processor
+    class: ProcessorStep
+  - type: step
+    name: serialize
+    module: mas.library.lab.steps.data.serialize
+    class: SerializeStep
+  - type: step
+    name: service_start
+    module: mas.library.lab.steps.services.service_start
+    class: ServiceStartStep
+  - type: step
+    name: service_stop
+    module: mas.library.lab.steps.services.service_stop
+    class: ServiceStopStep
+  - type: step
+    name: to_dataframe
+    module: mas.library.lab.steps.data.to_dataframe
+    class: ToDataFrameStep
+  - type: step
+    name: validate_outputs
+    module: mas.library.lab.steps.eval.validate_outputs
+    class: ValidateOutputsStep
+  - type: codec
+    name: otel_traces-otlp
+    urn: mas.codec.otel_traces_otlp
+    module: mas.library.lab.codecs.otel_codecs
+    class: OtlpOtelCodec
+    attributes:
+      artifact_kind: otel_traces
+      store_type: otlp
+  - type: codec
+    name: otel_traces-clickhouse
+    urn: mas.codec.otel_traces_clickhouse
+    module: mas.library.lab.codecs.otel_codecs
+    class: ClickHouseOtelCodec
+    attributes:
+      artifact_kind: otel_traces
+      store_type: clickhouse
+  - type: codec
+    name: otel_traces-filesystem
+    urn: mas.codec.otel_traces_filesystem
+    module: mas.library.lab.codecs.otel_codecs
+    class: FilesystemOtelCodec
+    attributes:
+      artifact_kind: otel_traces
+      store_type: filesystem
+  - type: codec
+    name: events-filesystem
+    urn: mas.codec.events_filesystem
+    module: mas.library.lab.codecs.otel_codecs
+    class: FilesystemEventsCodec
+    attributes:
+      artifact_kind: events
+      store_type: filesystem
+
+  # Experimental deterministic design patterns (local, not OSS)
+  - type: design_pattern
+    name: deterministic_voting
+    urn: mas.dp.deterministic_voting
+    module: mas.library.lab.plugins.design_patterns.voting
+    class: DeterministicVotingPlugin
+  - type: design_pattern
+    name: deterministic_staged_debate
+    urn: mas.dp.deterministic_staged_debate
+    module: mas.library.lab.plugins.design_patterns.staged_debate
+    class: DeterministicStagedDebatePlugin
+  - type: design_pattern
+    name: deterministic_supervised
+    urn: mas.dp.deterministic_supervised
+    module: mas.library.lab.plugins.design_patterns.supervised
+    class: DeterministicSupervisedPlugin
+  - type: design_pattern
+    name: deterministic_verifier
+    urn: mas.dp.deterministic_verifier
+    module: mas.library.lab.plugins.design_patterns.verifier
+    class: DeterministicVerifierPlugin
+
+  - type: artifact
+    name: dataframe
+    module: mas.library.lab.artifact_types
+    class: DataFrame
+  - type: artifact
+    name: metrics
+    module: mas.library.lab.artifact_types
+    class: Metrics
+  - type: artifact
+    name: plot
+    module: mas.library.lab.artifact_types
+    class: Plot
+  - type: artifact
+    name: run_info
+    module: mas.library.lab.artifact_types
+    class: RunInfo
+  - type: artifact
+    name: trace
+    module: mas.library.lab.artifact_types
+    class: Trace
+  - type: artifact
+    name: chat
+    module: mas.library.lab.artifact_types
+    class: ChatSnapshot
+  - type: artifact
+    name: topology
+    module: mas.library.lab.artifact_types
+    class: TopologySnapshot
+  - type: artifact
+    name: series
+    module: mas.library.lab.artifact_types
+    class: LoopSeries

mas_lab_library_lab-0.2.0/pyproject.toml

@@ -0,0 +1,33 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mas-lab-library-lab"
+version = "0.2.0"
+description = "MAS Lab provider library with public eval plugins."
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "Apache-2.0"}
+authors = [{name = "MAS Framework Contributors"}]
+dependencies = [
+    "mas-lab-component-core",
+    "mas-lab-component-bench",
+]
+
+[project.entry-points."mas.lab.eval.plugins"]
+mcev1 = "mas.library.lab.eval.mcev1:MCEv1Provider"
+
+[project.entry-points."mas.lab.ui.plots"]
+series = "mas.library.lab.ui.series:SeriesPlots"
+
+[project.entry-points."mas.runtime.manifest_libraries"]
+lab = "mas.library.lab"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/mas"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"library.yaml" = "mas/library/lab/library.yaml"

mas_lab_library_lab-0.2.0/src/mas/library/lab/__init__.py

@@ -0,0 +1,23 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""mas.library.lab package."""
+
+from pathlib import Path
+
+from mas._dist import installed_version
+
+
+def package_root() -> Path:
+    """Return the directory containing ``library.yaml`` for this manifest library."""
+    here = Path(__file__).resolve().parent
+    for parent in [here, *here.parents]:
+        if (parent / "library.yaml").is_file():
+            return parent
+    return here
+
+
+__version__ = installed_version(file=__file__)
+
+from mas.library.lab.plots import execution_chain_graph  # noqa: E402
+
+__all__ = ["execution_chain_graph"]

mas_lab_library_lab-0.2.0/src/mas/library/lab/artifact_types.py

@@ -0,0 +1,73 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Artifact kinds declared by the lab library.
+
+Each class is a library.yaml ``type: artifact`` plugin
+(``type`` / ``name`` / ``module`` / ``class``).
+"""
+
+from __future__ import annotations
+
+from mas.lab.lab.config.artifact_types import ArtifactType
+
+
+class Trace(ArtifactType):
+    path = "{run_dir}/traces/events.jsonl"
+    format = "jsonl"
+    stream = "native"
+    realtime = True
+    description = "Raw observability event stream from a single run."
+
+
+class RunInfo(ArtifactType):
+    path = "{run_dir}/run_info.json"
+    format = "json"
+    description = "Run metadata (hash, model, timing, status)."
+
+
+class Plot(ArtifactType):
+    path = "{level_dir}/plot.png"
+    format = "png"
+    description = "Visualization output (PNG, SVG, HTML, or PDF)."
+
+
+class Metrics(ArtifactType):
+    path = "{level_dir}/metrics.json"
+    format = "json"
+    schema = "artefacts/metrics.schema.json"
+    stream = "metric"
+    realtime = True
+    description = "Quality metrics computed by evaluation providers."
+
+
+class ChatSnapshot(ArtifactType):
+    path = "{run_dir}/custom.chat.jsonl"
+    format = "jsonl"
+    stream = "custom.chat"
+    plot = "chat"
+    realtime = True
+    description = "Folded chat cards for the chat HTML plot."
+
+
+class TopologySnapshot(ArtifactType):
+    path = "{run_dir}/custom.topology.jsonl"
+    format = "jsonl"
+    stream = "custom.topology"
+    plot = "topology"
+    realtime = True
+    description = "Folded graph for the topology HTML plot."
+
+
+class LoopSeries(ArtifactType):
+    path = "{run_dir}/plots/series.svg"
+    format = "svg"
+    stream = "series"
+    plot = "loop"
+    realtime = True
+    description = "Generic timeseries (html live; svg/png end-state)."
+
+
+class DataFrame(ArtifactType):
+    path = "{level_dir}/data.csv"
+    format = "csv"
+    description = "Tidy CSV dataframe for analysis and plotting."