mas-lab-component-bench 0.2.0__py3-none-any.whl

mas/lab/benchmark/README.md

@@ -0,0 +1,323 @@
+<!--
+  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+  SPDX-License-Identifier: Apache-2.0
+-->
+# Benchmark Framework - Reusable Components
+
+**Composable primitives for agent evaluation.**
+
+## Philosophy
+
+This framework follows these principles:
+
+1. **Declarative over imperative** - Configure via YAML/JSON, not code
+2. **Incremental over monolithic** - Add runs without rerunning everything
+3. **Composable over all-in-one** - Mix and match components
+4. **Human-readable over optimized** - Folder names you can understand
+5. **DataFrame-native over custom formats** - Leverage pandas/polars ecosystem
+
+## Components
+
+### 1. Dataset (`dataset.py`)
+
+Loads and manipulates evaluation datasets.
+
+```python
+from mas.lab.benchmark import Dataset
+
+# Load from JSON
+dataset = Dataset.from_json("math_problems.json")
+
+# Filter by category
+arithmetic = dataset.filter(category="arithmetic")
+
+# Iterate
+for item in dataset:
+    print(item.id, item.prompt)
+```
+
+**JSON Format:**
+```json
+{
+  "name": "dataset_name",
+  "version": "v1",
+  "items": [
+    {
+      "id": "001",
+      "prompt": "Question text",
+      "category": "category_name",
+      "expected_answer": "Answer",
+      "metadata_key": "metadata_value"
+    }
+  ]
+}
+```
+
+### 2. ResultStorage (`storage.py`)
+
+Manages human-readable folder structure.
+
+```python
+from mas.lab.benchmark import ResultStorage, RunMetadata
+
+storage = ResultStorage("./results")
+
+# Generate run ID
+run_id = storage.generate_run_id("cot", run_number=1)
+# → "cot_run_001_20260218_143022"
+
+# Save run
+metadata = RunMetadata(
+    dataset_name="math",
+    item_id="001",
+    run_id=run_id,
+    pattern="cot",
+    timestamp="2026-02-18T14:30:22",
+    config={"model": "gpt-4"},
+    success=True,
+    latency_ms=1234.5
+)
+
+storage.save_run(
+    dataset_name="math",
+    item_id="001",
+    run_id=run_id,
+    metadata=metadata,
+    events_file=Path("events.jsonl"),
+    artifacts={"log": Path("agent.log")}
+)
+
+# List all runs
+runs = storage.list_runs("math")
+# → [("math", "001", "cot_run_001_20260218_143022"), ...]
+```
+
+**Folder Structure:**
+```
+results/
+  {dataset_name}/
+    {item_id}/
+      {run_id}/
+        metadata.json
+        events.jsonl
+        artifacts/
+    consolidated.parquet
+```
+
+### 3. MultiRunOrchestrator (`runner.py`)
+
+Coordinates N runs per item across patterns.
+
+```python
+from mas.lab.benchmark import MultiRunOrchestrator
+
+orchestrator = MultiRunOrchestrator(
+    storage=storage,
+    n_runs=3,
+    pause_between_runs=1.0
+)
+
+# Define agent factory
+async def agent_factory(pattern_name, pattern_config):
+    # Your agent creation logic
+    return agent
+
+# Run benchmark
+summary = await orchestrator.run_dataset(
+    dataset=dataset,
+    patterns={
+        "cot": {"manifest": "cot.yaml"},
+        "react": {"manifest": "react.yaml"}
+    },
+    agent_factory=agent_factory
+)
+```
+
+### 4. ResultAnalyzer (`analysis.py`)
+
+Consolidates results into DataFrame.
+
+```python
+from mas.lab.benchmark import ResultAnalyzer
+
+analyzer = ResultAnalyzer(storage)
+
+# Consolidate all runs
+df = analyzer.consolidate_results(
+    dataset_name="math",
+    include_events=True,  # Parse event logs
+    cache=True            # Save as Parquet
+)
+
+# Compute statistics
+stats = analyzer.compute_statistics(df)
+#    pattern  latency_ms_mean  success_rate  mean_tokens
+# 0  cot      1234.5           0.95          567
+# 1  react    890.2            0.92          423
+
+# Compare patterns
+comparison = analyzer.compare_patterns(df, ["cot", "react"], "latency_ms")
+```
+
+**DataFrame Schema:**
+```python
+df.columns
+# ['dataset_name', 'item_id', 'run_id', 'pattern',
+#  'timestamp', 'success', 'latency_ms', 'error',
+#  'total_tokens', 'prompt_tokens', 'completion_tokens',
+#  'tool_calls', 'llm_calls',
+#  'config_*']
+```
+
+### 5. Pipeline plots (`plotnine` / `plot` steps)
+
+Paper figures use **`type: plotnine`** in `experiment.yaml` experiment-level `post:`.
+Ad-hoc charts from step data use **`type: plot`** with **`config.spec`** (plot library).
+
+See [pipeline-steps.md](../../../../../../docs/pipeline-steps.md) and `labs/*/experiment.yaml` for examples.
+
+## Complete Example
+
+```python
+import asyncio
+from pathlib import Path
+from mas.lab.benchmark import (
+    Dataset, ResultStorage, MultiRunOrchestrator,
+    ResultAnalyzer,
+)
+
+async def my_agent_factory(pattern_name, pattern_config):
+    # Your agent creation logic
+    pass
+
+async def main():
+    # 1. Load dataset
+    dataset = Dataset.from_yaml("dataset.yaml")
+    
+    # 2. Setup storage
+    storage = ResultStorage("./results")
+    
+    # 3. Run benchmark
+    orchestrator = MultiRunOrchestrator(storage, n_runs=3)
+    await orchestrator.run_dataset(
+        dataset,
+        patterns={"cot": {}, "react": {}},
+        agent_factory=my_agent_factory
+    )
+    
+    # 4. Analyze
+    analyzer = ResultAnalyzer(storage)
+    df = analyzer.consolidate_results(dataset.name)
+    stats = analyzer.compute_statistics(df)
+    print(stats)
+    # Plots: declare plotnine / plot steps in experiment.yaml post:
+
+asyncio.run(main())
+```
+
+## Incremental Updates
+
+### Add Items
+```python
+# Load existing dataset
+dataset = Dataset.from_json("dataset.json")
+
+# Add new items
+dataset.items.append(DatasetItem(id="006", prompt="New question"))
+
+# Save
+dataset.to_json("dataset.json")
+
+# Run benchmark (only new items executed)
+await orchestrator.run_dataset(dataset, patterns, agent_factory)
+```
+
+### Add Runs
+```python
+# Increase n_runs
+orchestrator = MultiRunOrchestrator(storage, n_runs=5)  # Was 3
+
+# Run again - adds run_004, run_005 to existing items
+await orchestrator.run_dataset(dataset, patterns, agent_factory)
+```
+
+### Add Patterns
+```python
+# Add new pattern
+patterns = {
+    "cot": {...},
+    "react": {...},
+    "planner": {...}  # NEW
+}
+
+# Run again - only planner executed on all items
+await orchestrator.run_dataset(dataset, patterns, agent_factory)
+```
+
+### Regenerate Plots
+```python
+# Re-run pipeline post steps only (content-addressed trace cache skips LLM calls)
+# mas-lab benchmark run experiment.yaml --force-post
+```
+
+## Advanced Usage
+
+### Custom Metrics
+
+Extend `ResultAnalyzer._extract_event_metrics()`:
+
+```python
+def _extract_event_metrics(self, events_file: Path) -> Dict[str, Any]:
+    metrics = super()._extract_event_metrics(events_file)
+    
+    # Add custom metrics
+    metrics["my_custom_metric"] = ...
+    
+    return metrics
+```
+
+### Custom Plots
+
+Add a `plotnine` step in experiment-level `post:` or a plot-library YAML under
+`pipeline/steps/plot_library/`. See `labs/design-space.lab/01-design-patterns/experiment.yaml`.
+
+### Parallel Execution
+
+```python
+# Run patterns in parallel
+import asyncio
+
+async def run_pattern(pattern_name, pattern_config):
+    await orchestrator.run_dataset(
+        dataset,
+        patterns={pattern_name: pattern_config},
+        agent_factory=agent_factory
+    )
+
+await asyncio.gather(
+    run_pattern("cot", {...}),
+    run_pattern("react", {...}),
+)
+```
+
+## Dependencies
+
+```bash
+pip install pandas pyarrow plotnine pyyaml
+```
+
+## Testing
+
+```bash
+cd mas-lab
+pytest tests/benchmark/
+```
+
+## Contributing
+
+When adding new components:
+
+1. Follow existing patterns (DataClass for data, Class for behavior)
+2. Add docstrings with examples
+3. Write tests in `tests/benchmark/`
+4. Update this documentation

mas/lab/benchmark/__init__.py

@@ -0,0 +1,65 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""
+Reusable benchmark infrastructure for MAS evaluations.
+
+Provides composable primitives for:
+- Dataset management
+- Multi-run orchestration  
+- Incremental results storage
+- DataFrame consolidation
+- Declarative evaluation configuration
+"""
+
+from mas.lab.benchmark.dataset import Dataset, DatasetItem
+from mas.lab.benchmark.runner import MultiRunOrchestrator
+from mas.lab.benchmark.storage import ResultStorage, RunMetadata
+from mas.lab.benchmark.metadata import (
+    BenchmarkMetadata,
+    BenchmarkState,
+    BenchmarkStatus,
+    ScenarioResult,
+    ScenarioState,
+)
+from mas.lab.benchmark.state import EnhancedBenchmarkState
+from mas.lab.benchmark.lock import BenchmarkLock, LockInfo
+
+__all__ = [
+    "Dataset",
+    "DatasetItem",
+    "MultiRunOrchestrator",
+    "ResultStorage",
+    "RunMetadata",
+    "ResultAnalyzer",
+    "BenchmarkMetadata",
+    "BenchmarkState",
+    "BenchmarkStatus",
+    "ScenarioResult",
+    "ScenarioState",
+    "EnhancedBenchmarkState",
+    "BenchmarkLock",
+    "LockInfo",
+    "ExperimentResults",
+    "ScenarioView",
+    "ItemView",
+    "RunView",
+]
+
+
+def __getattr__(name: str):
+    # Defer pandas imports so coverage tracing of pipeline subpackages does not
+    # reload numpy during test collection (pytest-cov / coverage.py on Py 3.11+).
+    if name == "ResultAnalyzer":
+        from mas.lab.benchmark.analysis import ResultAnalyzer
+
+        return ResultAnalyzer
+    if name in ("ExperimentResults", "ScenarioView", "ItemView", "RunView"):
+        from mas.lab.benchmark.results import (
+            ExperimentResults,
+            ItemView,
+            RunView,
+            ScenarioView,
+        )
+
+        return locals()[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

mas/lab/benchmark/analysis.py

@@ -0,0 +1,199 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""
+Result analysis and DataFrame consolidation.
+
+Loads all run metadata and creates consolidated DataFrames for analysis.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+import pandas as pd
+
+from mas.lab.benchmark.storage import ResultStorage
+
+
+class ResultAnalyzer:
+    """
+    Analyze benchmark results.
+    
+    Features:
+    - Consolidate all runs into DataFrame
+    - Compute statistics per pattern
+    - Extract metrics from event logs
+    - Cache results as Parquet
+    """
+    
+    def __init__(self, storage: ResultStorage):
+        self.storage = storage
+    
+    def consolidate_results(
+        self,
+        dataset_name: str,
+        include_events: bool = False,
+        cache: bool = True
+    ) -> pd.DataFrame:
+        """
+        Consolidate all runs into DataFrame.
+        
+        Args:
+            dataset_name: Dataset name
+            include_events: Parse event logs for detailed metrics
+            cache: Save as Parquet for incremental updates
+            
+        Returns:
+            DataFrame with columns:
+            - dataset_name, item_id, run_id, pattern
+            - timestamp, success, latency_ms, error
+            - (if include_events) total_tokens, tool_calls, etc.
+        """
+        runs = self.storage.list_runs(dataset_name)
+        
+        if not runs:
+            return pd.DataFrame()
+        
+        rows = []
+        for ds_name, item_id, run_id in runs:
+            metadata = self.storage.load_run_metadata(ds_name, item_id, run_id)
+            if not metadata:
+                continue
+            
+            row = {
+                "dataset_name": metadata.dataset_name,
+                "item_id": metadata.item_id,
+                "run_id": metadata.run_id,
+                "pattern": metadata.pattern,
+                "timestamp": metadata.timestamp,
+                "success": metadata.success,
+                "latency_ms": metadata.latency_ms,
+                "error": metadata.error,
+            }
+            
+            # Add config fields
+            for key, value in metadata.config.items():
+                row[f"config_{key}"] = value
+            
+            # Parse events if requested
+            if include_events:
+                run_dir = self.storage.get_run_dir(ds_name, item_id, run_id)
+                events_file = run_dir / "events.jsonl"
+                if events_file.exists():
+                    event_metrics = self._extract_event_metrics(events_file)
+                    row.update(event_metrics)
+            
+            rows.append(row)
+        
+        df = pd.DataFrame(rows)
+        
+        # Cache as Parquet
+        if cache and not df.empty:
+            cache_path = self.storage.get_consolidated_path(dataset_name)
+            df.to_parquet(cache_path, index=False)
+        
+        return df
+    
+    def load_cached(self, dataset_name: str) -> Optional[pd.DataFrame]:
+        """Load cached consolidated DataFrame."""
+        cache_path = self.storage.get_consolidated_path(dataset_name)
+        if not cache_path.exists():
+            return None
+        return pd.read_parquet(cache_path)
+    
+    def compute_statistics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Compute aggregate statistics per pattern.
+        
+        Returns:
+            DataFrame with pattern-level statistics:
+            - mean_latency_ms, std_latency_ms
+            - success_rate
+            - mean_tokens, mean_tool_calls (if available)
+        """
+        if df.empty:
+            return pd.DataFrame()
+        
+        # Basic statistics
+        stats = df.groupby("pattern").agg({
+            "latency_ms": ["mean", "std", "min", "max"],
+            "success": "mean",
+        })
+        
+        # Flatten column names
+        stats.columns = ["_".join(col).strip("_") for col in stats.columns]
+        stats = stats.rename(columns={"success_mean": "success_rate"})
+        
+        # Token and tool metrics if available
+        if "total_tokens" in df.columns:
+            token_stats = df.groupby("pattern")["total_tokens"].agg(["mean", "std"])
+            token_stats.columns = ["mean_tokens", "std_tokens"]
+            stats = stats.join(token_stats)
+        
+        if "tool_calls" in df.columns:
+            tool_stats = df.groupby("pattern")["tool_calls"].agg(["mean", "std"])
+            tool_stats.columns = ["mean_tool_calls", "std_tool_calls"]
+            stats = stats.join(tool_stats)
+        
+        return stats.reset_index()
+    
+    def _extract_event_metrics(self, events_file: Path) -> Dict[str, Any]:
+        """Extract metrics from event log."""
+        metrics = {
+            "total_tokens": 0,
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "tool_calls": 0,
+            "llm_calls": 0,
+        }
+        
+        with open(events_file) as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                try:
+                    event = json.loads(line)
+                    
+                    # Count LLM calls
+                    if event.get("kind") == "llm_response":
+                        metrics["llm_calls"] += 1
+                        usage = event.get("usage", {})
+                        metrics["total_tokens"] += usage.get("total_tokens", 0)
+                        metrics["prompt_tokens"] += usage.get("prompt_tokens", 0)
+                        metrics["completion_tokens"] += usage.get("completion_tokens", 0)
+                    
+                    # Count tool calls
+                    elif event.get("kind") == "tool_start":
+                        metrics["tool_calls"] += 1
+                    
+                except json.JSONDecodeError:
+                    continue
+        
+        return metrics
+    
+    def get_item_runs(self, df: pd.DataFrame, item_id: str) -> pd.DataFrame:
+        """Get all runs for specific item."""
+        return df[df["item_id"] == item_id].copy()
+    
+    def get_pattern_runs(self, df: pd.DataFrame, pattern: str) -> pd.DataFrame:
+        """Get all runs for specific pattern."""
+        return df[df["pattern"] == pattern].copy()
+    
+    def compare_patterns(
+        self,
+        df: pd.DataFrame,
+        patterns: List[str],
+        metric: str = "latency_ms"
+    ) -> pd.DataFrame:
+        """
+        Compare patterns on specific metric.
+        
+        Returns:
+            DataFrame with item_id, pattern, metric columns
+        """
+        filtered = df[df["pattern"].isin(patterns)]
+        return filtered.pivot_table(
+            index="item_id",
+            columns="pattern",
+            values=metric,
+            aggfunc="mean"
+        ).reset_index()

mas/lab/benchmark/cache/__init__.py

@@ -0,0 +1,28 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Content-addressed trace cache helpers."""
+from mas.lab.benchmark.cache.trace_store import (
+    compute_run_hash,
+    resolve_run_events_path,
+    trace_cache_roots,
+    extract_flavour_info,
+    get_trace_cache_dir,
+    link_trace_to_cache_entry,
+    materialize_config,
+    write_cache_inputs,
+    write_run_info,
+    write_run_result,
+)
+
+__all__ = [
+    "compute_run_hash",
+    "extract_flavour_info",
+    "get_trace_cache_dir",
+    "link_trace_to_cache_entry",
+    "materialize_config",
+    "resolve_run_events_path",
+    "trace_cache_roots",
+    "write_cache_inputs",
+    "write_run_info",
+    "write_run_result",
+]