mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/schemas/pipeline-step-types-post.json

@@ -0,0 +1,764 @@
+{
+  "$comment": "Registry of all pipeline step types with their config schemas. Use this to build a dynamic pipeline editor in React.",
+  "step_types": [
+    {
+      "type": "service_stop",
+      "label": "Stop Service",
+      "description": "Stop an infrastructure service (Neo4j, OTel collector, etc.) after all scenarios complete.",
+      "phase": "post",
+      "category": "infrastructure",
+      "config": {
+        "service": {
+          "type": "string",
+          "required": true,
+          "description": "Service name declared in the infra bundle (e.g. neo4j, otel-collector)"
+        }
+      }
+    },
+    {
+      "type": "extract_trajectories",
+      "label": "Extract Trajectories",
+      "description": "Parse benchmark run traces into a structured trajectories.jsonl file. Extracts the input prompt, final output, per-agent messages, and session metadata from each run's events.jsonl. This is typically the first step in a pipeline — downstream steps like plot_multilevel_trajectory and annotate_metrics consume its output.",
+      "phase": "post",
+      "category": "extraction",
+      "config": {
+        "csv_file": {
+          "type": "string",
+          "default": "results.csv",
+          "description": "Benchmark results CSV filename"
+        },
+        "runs_dir": {
+          "type": "string",
+          "default": "runs",
+          "description": "Relative dir for orphan run scan"
+        },
+        "overwrite": {
+          "type": "boolean",
+          "default": false,
+          "description": "Overwrite existing extractions"
+        }
+      }
+    },
+    {
+      "type": "extract_trace_stats",
+      "label": "Extract Trace Stats",
+      "description": "Compute structural metrics from raw trace files — counts LLM calls, tool invocations, governance events, and measures total duration per run. Outputs a tidy CSV (one row per run) for cost analysis and efficiency comparisons across scenarios. Reads events.jsonl directly from the benchmark output directory.",
+      "phase": "post",
+      "category": "extraction",
+      "config": {
+        "output": {
+          "type": "string",
+          "default": "results/trace_stats.csv",
+          "description": "Output CSV path, relative to the level folder (no absolute paths, no '..')"
+        },
+        "scenarios": {
+          "type": "array",
+          "description": "Optional list of scenario IDs to include (all if omitted)"
+        },
+        "item_glob": {
+          "type": "string",
+          "default": "item*",
+          "description": "Glob pattern for item directories"
+        },
+        "run_glob": {
+          "type": "string",
+          "default": "r*",
+          "description": "Glob pattern for run directories"
+        }
+      }
+    },
+    {
+      "type": "analysis",
+      "label": "Consolidate Results",
+      "description": "Consolidate experiment results into DataFrame + optional parquet cache.",
+      "phase": "post",
+      "category": "analysis",
+      "config": {
+        "include_events": {
+          "type": "boolean",
+          "default": true,
+          "description": "Include event data in consolidation"
+        },
+        "cache": {
+          "type": "boolean",
+          "default": true,
+          "description": "Write consolidated.parquet"
+        }
+      }
+    },
+    {
+      "type": "annotate_metrics",
+      "label": "Annotate Metrics",
+      "description": "Session-level evaluation metrics via EvalMetric class (e.g., AnswerRelevancyMetric).",
+      "phase": "post",
+      "category": "evaluation",
+      "config": {
+        "metric_class": {
+          "type": "string",
+          "default": "mas.lab.components.evaluation.deepeval_wrapper.AnswerRelevancyMetric",
+          "description": "Fully qualified metric class"
+        },
+        "metric_kwargs": {
+          "type": "object",
+          "default": {"model": "azure/gpt-4o"},
+          "description": "Kwargs passed to metric constructor (model, threshold, etc.)"
+        },
+        "level": {
+          "type": "string",
+          "default": "session",
+          "enum": [
+            "session",
+            "turn"
+          ],
+          "description": "Evaluation granularity"
+        },
+        "max_items": {
+          "type": "integer",
+          "default": null,
+          "description": "Limit items to evaluate"
+        },
+        "overwrite": {
+          "type": "boolean",
+          "default": false,
+          "description": "Overwrite existing metrics"
+        }
+      }
+    },
+    {
+      "type": "embed_trajectories",
+      "label": "Embed Trajectories",
+      "description": "Compute embedding vectors for trajectory text fields.",
+      "phase": "post",
+      "category": "analysis",
+      "config": {
+        "model": {
+          "type": "string",
+          "default": "text-embedding-3-small",
+          "description": "Embedding model"
+        },
+        "api_base": {
+          "type": "string",
+          "default": "",
+          "description": "API base URL"
+        },
+        "api_key_env": {
+          "type": "string",
+          "default": "OPENAI_API_KEY",
+          "description": "Env var for API key"
+        },
+        "levels": {
+          "type": "array",
+          "default": [
+            "session"
+          ],
+          "description": "Granularity levels"
+        },
+        "fields": {
+          "type": "array",
+          "default": [
+            "input",
+            "output"
+          ],
+          "description": "Fields to embed"
+        },
+        "max_items": {
+          "type": "integer",
+          "default": null,
+          "description": "Limit items"
+        },
+        "overwrite": {
+          "type": "boolean",
+          "default": false,
+          "description": "Overwrite existing"
+        },
+        "batch_size": {
+          "type": "integer",
+          "default": 64,
+          "description": "Batch size"
+        }
+      }
+    },
+    {
+      "type": "ggplot",
+      "label": "Plot (Custom Class)",
+      "description": "Render a publication-quality figure from a MASPlot subclass. Uses experiment results directory for data.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "class": {
+          "type": "string",
+          "required": true,
+          "description": "Plot library YAML name under plot_library/ (e.g. ablation_heatmap)",
+          "default": "ablation_heatmap"
+        },
+        "formats": {
+          "type": "array",
+          "default": [
+            "svg"
+          ],
+          "description": "Output formats"
+        }
+      }
+    },
+    {
+      "type": "plot",
+      "label": "Plot (Legacy)",
+      "description": "Generate visualizations from analysis DataFrame (legacy or declarative).",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "type": {
+          "type": "string",
+          "default": null,
+          "enum": [
+            "latency_distribution",
+            "success_rate",
+            "pattern_comparison",
+            "pareto_frontier"
+          ],
+          "description": "Legacy plot type"
+        },
+        "params": {
+          "type": "object",
+          "default": {},
+          "description": "Plot parameters"
+        },
+        "spec": {
+          "type": "object",
+          "default": null,
+          "description": "Declarative plot spec (alternative to type)"
+        },
+        "data_key": {
+          "type": "string",
+          "default": null,
+          "description": "Key in dependency data for declarative mode"
+        },
+        "filename": {
+          "type": "string",
+          "default": null,
+          "description": "Output filename for declarative mode"
+        }
+      }
+    },
+    {
+      "type": "plot_trajectory",
+      "label": "Plot Trajectory",
+      "description": "Render agent trajectory diagram from a single trace.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "log_path": {
+          "type": "string",
+          "default": null,
+          "description": "Path to events.jsonl"
+        },
+        "format": {
+          "type": "string",
+          "default": "mermaid",
+          "enum": [
+            "mermaid",
+            "html",
+            "svg",
+            "table"
+          ],
+          "description": "Output format"
+        },
+        "include_prompts": {
+          "type": "boolean",
+          "default": true,
+          "description": "Include prompt text"
+        },
+        "filename": {
+          "type": "string",
+          "default": "trajectory",
+          "description": "Output filename"
+        },
+        "highlight_agents": {
+          "type": "array",
+          "default": null,
+          "description": "Agent names to highlight"
+        },
+        "highlight_keywords": {
+          "type": "array",
+          "default": null,
+          "description": "Keywords to highlight"
+        }
+      }
+    },
+    {
+      "type": "plot_trajectory_batch",
+      "label": "Plot Trajectory (Batch)",
+      "description": "Render trajectory diagrams for multiple runs.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "runs": {
+          "type": "array",
+          "required": true,
+          "description": "List of {log_path, filename} entries"
+        },
+        "format": {
+          "type": "string",
+          "default": "html",
+          "enum": [
+            "mermaid",
+            "html",
+            "svg",
+            "table"
+          ],
+          "description": "Output format"
+        },
+        "include_prompts": {
+          "type": "boolean",
+          "default": true,
+          "description": "Include prompt text"
+        },
+        "concurrency": {
+          "type": "integer",
+          "default": 1,
+          "description": "Parallel renders"
+        }
+      }
+    },
+    {
+      "type": "plot_multilevel_trajectory",
+      "label": "Plot Multilevel Trajectory",
+      "description": "Render an interactive timeline showing how agents, LLM calls, and tool invocations interleave across a run. The diagram reveals execution structure: which agent delegated to which, how long each step took, and where parallelism or bottlenecks occur. Outputs HTML (interactive) or SVG (static). Depends on extract_trajectories.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "log_path": {
+          "type": "string",
+          "default": null,
+          "description": "Path to events.jsonl"
+        },
+        "format": {
+          "type": "string",
+          "default": "html",
+          "enum": [
+            "html",
+            "svg"
+          ],
+          "description": "Output format"
+        },
+        "title": {
+          "type": "string",
+          "default": "MAS Multilevel Trajectory",
+          "description": "Diagram title"
+        },
+        "width_mode": {
+          "type": "string",
+          "default": "log",
+          "enum": [
+            "log",
+            "linear"
+          ],
+          "description": "Width scaling mode"
+        },
+        "show_user_actors": {
+          "type": "boolean",
+          "default": true,
+          "description": "Show user actors"
+        },
+        "filename": {
+          "type": "string",
+          "default": "multilevel_trajectory",
+          "description": "Output filename"
+        },
+        "annotations": {
+          "type": "string",
+          "default": null,
+          "description": "Annotations YAML path"
+        },
+        "highlight_agents": {
+          "type": "array",
+          "default": null,
+          "description": "Agents to highlight"
+        },
+        "pins": {
+          "type": "array",
+          "default": null,
+          "description": "Pin markers"
+        }
+      }
+    },
+    {
+      "type": "plot_multilevel_trajectory_batch",
+      "label": "Plot Multilevel Trajectory (Batch)",
+      "description": "Batch multilevel trajectory rendering across runs.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "runs": {
+          "type": "array",
+          "default": null,
+          "description": "Explicit run list (auto-derived if omitted)"
+        },
+        "format": {
+          "type": "string",
+          "default": "html",
+          "enum": [
+            "html",
+            "svg"
+          ],
+          "description": "Output format"
+        },
+        "width_mode": {
+          "type": "string",
+          "default": "log",
+          "enum": [
+            "log",
+            "linear"
+          ],
+          "description": "Width scaling"
+        },
+        "show_user_actors": {
+          "type": "boolean",
+          "default": true,
+          "description": "Show user actors"
+        },
+        "title": {
+          "type": "string",
+          "default": "MAS Multilevel Trajectory",
+          "description": "Diagram title"
+        },
+        "annotations": {
+          "type": "string",
+          "default": null,
+          "description": "Annotations YAML"
+        },
+        "highlight_agents": {
+          "type": "array",
+          "default": null,
+          "description": "Agents to highlight"
+        },
+        "pins": {
+          "type": "array",
+          "default": null,
+          "description": "Pin markers"
+        },
+        "concurrency": {
+          "type": "integer",
+          "default": 1,
+          "description": "Parallel renders"
+        }
+      }
+    },
+    {
+      "type": "plot_communication_flow",
+      "label": "Plot Communication Flow",
+      "description": "Agent communication-flow diagram.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "log_path": {
+          "type": "string",
+          "default": null,
+          "description": "Path to events.jsonl"
+        },
+        "format": {
+          "type": "string",
+          "default": "html",
+          "enum": [
+            "html",
+            "mermaid"
+          ],
+          "description": "Output format"
+        },
+        "title": {
+          "type": "string",
+          "default": "MAS Agent Communication Flow",
+          "description": "Diagram title"
+        },
+        "filename": {
+          "type": "string",
+          "default": "communication_flow",
+          "description": "Output filename"
+        }
+      }
+    },
+    {
+      "type": "collect_metrics",
+      "label": "Collect Metrics",
+      "description": "Aggregate all per-run metrics.json and run_info.json files into a single tidy CSV (metrics_results.csv). Each row is one (scenario, item, run, metric) observation with columns for score, latency, LLM/tool call counts, and cache keys. Optionally enriches rows with dataset item metadata (group, complexity) for richer downstream grouping. Typically depends on eval_mce.",
+      "phase": "post",
+      "category": "evaluation",
+      "config": {
+        "output": {
+          "type": "string",
+          "default": "results.csv",
+          "description": "Output CSV path, relative to the level folder (no absolute paths, no '..')"
+        },
+        "run_glob": {
+          "type": "string",
+          "default": "*/*/r*",
+          "description": "Glob for run directories relative to the level folder"
+        },
+        "scenarios": {
+          "type": "array",
+          "default": [],
+          "description": "Optional scenario filter (all if empty)"
+        },
+        "metrics_filename": {
+          "type": "string",
+          "default": "metrics.json",
+          "description": "Name of the metrics file in each run directory"
+        },
+        "dataset": {
+          "type": "string",
+          "default": "",
+          "description": "Path to dataset YAML/JSON for item enrichment (or dict with path, join_on, columns)"
+        }
+      }
+    },
+    {
+      "type": "compute_ci",
+      "label": "Compute CI",
+      "description": "Compute confidence intervals across repeated runs. Groups the metrics CSV by scenario and metric (or custom groupby columns), then calculates mean, std, and 95% CI bounds (ci_low, ci_high) for each group. Outputs a summary CSV ready for plotting. Use ci_method 't' for small samples (<30 runs), 'bootstrap' for non-normal data. Depends on collect_metrics.",
+      "phase": "post",
+      "category": "analysis",
+      "config": {
+        "data": {
+          "type": "string",
+          "required": true,
+          "default": "@collect-metrics",
+          "description": "Input CSV path or @step-name reference (e.g. @collect-metrics)"
+        },
+        "value_col": {
+          "type": "string",
+          "default": "value",
+          "description": "Column to aggregate"
+        },
+        "metric_col": {
+          "type": "string",
+          "default": "metric",
+          "description": "Column that names each metric"
+        },
+        "metrics": {
+          "type": "array",
+          "default": [],
+          "description": "Optional: restrict to these metric values"
+        },
+        "groupby": {
+          "type": "array",
+          "default": ["scenario", "metric"],
+          "description": "Grouping columns for CI computation"
+        },
+        "ci_method": {
+          "type": "string",
+          "default": "t",
+          "enum": ["t", "bootstrap", "normal"],
+          "description": "CI method: t (default, for n<30), bootstrap, or normal"
+        },
+        "ci_level": {
+          "type": "number",
+          "default": 0.95,
+          "description": "Confidence level (e.g. 0.95 for 95% CI)"
+        },
+        "min_runs": {
+          "type": "integer",
+          "default": 2,
+          "description": "Warn when a group has fewer runs"
+        },
+        "output": {
+          "type": "string",
+          "default": "ci_summary.csv",
+          "description": "Output CSV path, relative to the level folder (no absolute paths, no '..')"
+        }
+      }
+    },
+    {
+      "type": "eval_mce",
+      "label": "Eval MCE",
+      "description": "Run LLM-as-judge evaluation on every benchmark run using MCE (Metrics Computation Engine). Computes session-level quality metrics — goal success, answer relevancy, groundedness, response completeness, and more — by prompting a judge LLM to score each run's conversation. Writes a metrics.json file next to each run's traces. Expensive (one LLM call per metric per run); use max_workers to control concurrency.",
+      "phase": "post",
+      "category": "evaluation",
+      "config": {
+        "runs_dir": {
+          "type": "string",
+          "default": "",
+          "description": "Path to runs root (defaults to pipeline output_dir)"
+        },
+        "response_agent": {
+          "type": "string",
+          "default": "",
+          "description": "Agent ID whose last execution_end is the final response (auto-detected if empty)"
+        },
+        "metrics": {
+          "type": "array",
+          "default": [],
+          "description": "MCE metric IDs to compute (all session-level if empty)"
+        },
+        "overwrite": {
+          "type": "boolean",
+          "default": false,
+          "description": "Re-compute runs that already have metrics.json"
+        },
+        "validate": {
+          "type": "boolean",
+          "default": true,
+          "description": "Validate each metrics.json against its JSON schema"
+        },
+        "max_workers": {
+          "type": "integer",
+          "default": 2,
+          "description": "Concurrent evaluation threads"
+        },
+        "fail_threshold": {
+          "type": "number",
+          "default": 1.0,
+          "description": "Fraction of items that may fail before step raises (0.0 = any, 1.0 = never)"
+        },
+        "model": {
+          "type": "string",
+          "default": "azure/gpt-4o",
+          "description": "Override judge model (re-installs LLM service with this model)"
+        },
+        "metrics_filename": {
+          "type": "string",
+          "default": "metrics.json",
+          "description": "Name of the output metrics file per run"
+        }
+      }
+    },
+    {
+      "type": "plotnine",
+      "label": "Plotnine Chart",
+      "description": "Render a publication-quality chart using plotnine (ggplot2 for Python). Maps CSV columns to aesthetics (x, y, fill, ymin, ymax) and renders bar charts (geom: col), CI rectangles (geom: crossbar), point plots, box plots, and more. Add facet to split into a grid by metric or scenario. Use as the final visualization step after compute_ci to compare scenarios across metrics.",
+      "phase": "post",
+      "category": "visualization",
+      "config": {
+        "data": {
+          "type": "string",
+          "required": true,
+          "default": "@compute-ci",
+          "description": "DataFrame source: file path (CSV/Parquet/JSON) or @step-name reference"
+        },
+        "output": {
+          "type": "string",
+          "default": "plot.png",
+          "description": "Output file path (png/svg/pdf), relative to the level folder (no absolute paths, no '..')"
+        },
+        "mapping": {
+          "type": "object",
+          "required": true,
+          "default": {"x": "scenario", "y": "mean", "fill": "scenario", "ymin": "ci_low", "ymax": "ci_high"},
+          "description": "aes() mapping: {x, y, fill, color, ymin, ymax, ...}. ymin+ymax auto-trigger error bars."
+        },
+        "geom": {
+          "type": "string",
+          "default": "crossbar",
+          "enum": ["bar", "col", "crossbar", "point", "boxplot", "jitter", "line", "tile"],
+          "description": "Geometry type (crossbar draws CI rectangles from ymin to ymax; col draws bars from 0 to y)"
+        },
+        "stat": {
+          "type": "string",
+          "default": "",
+          "enum": ["", "summary", "mean", "sum"],
+          "description": "Stat summary function (omit for pre-aggregated data)"
+        },
+        "facet": {
+          "type": "string",
+          "default": "~ metric",
+          "description": "Facet formula: '~ metric' or 'metric ~ .' (facet by metric is recommended for multi-metric data)"
+        },
+        "labels": {
+          "type": "object",
+          "default": {"title": "Scenario Comparison by Metric (95% CI)", "x": "Scenario", "y": "MCE Score (mean ± 95% CI)"},
+          "description": "Plot labels: {title, subtitle, x, y, fill, ...}"
+        },
+        "width": {
+          "type": "number",
+          "default": 14,
+          "description": "Figure width in inches"
+        },
+        "height": {
+          "type": "number",
+          "default": 10,
+          "description": "Figure height in inches"
+        },
+        "dpi": {
+          "type": "integer",
+          "default": 150,
+          "description": "DPI for raster output"
+        },
+        "filter": {
+          "type": "string",
+          "default": "",
+          "description": "Optional row filter (pandas query string or dict)"
+        },
+        "legend_position": {
+          "type": "string",
+          "default": "bottom",
+          "description": "Legend position: right, left, top, bottom, none, or [x,y] coords"
+        }
+      }
+    },
+    {
+      "type": "list_clickhouse_sessions",
+      "label": "List ClickHouse Sessions",
+      "description": "List session IDs from ClickHouse sessions_mapping.",
+      "phase": "post",
+      "category": "graph",
+      "config": {
+        "dataset_id": {
+          "type": "string",
+          "required": true,
+          "description": "Dataset ID"
+        },
+        "limit": {
+          "type": "integer",
+          "default": 0,
+          "description": "Max sessions (0 = unlimited)"
+        }
+      }
+    }
+  ],
+  "categories": [
+    {
+      "id": "data",
+      "label": "Data",
+      "color": "#3b82f6"
+    },
+    {
+      "id": "execution",
+      "label": "Execution",
+      "color": "#8b5cf6"
+    },
+    {
+      "id": "extraction",
+      "label": "Extraction",
+      "color": "#06b6d4"
+    },
+    {
+      "id": "normalization",
+      "label": "Normalization",
+      "color": "#14b8a6"
+    },
+    {
+      "id": "analysis",
+      "label": "Analysis",
+      "color": "#f59e0b"
+    },
+    {
+      "id": "evaluation",
+      "label": "Evaluation",
+      "color": "#ef4444"
+    },
+    {
+      "id": "visualization",
+      "label": "Visualization",
+      "color": "#22c55e"
+    },
+    {
+      "id": "graph",
+      "label": "Graph DB",
+      "color": "#a855f7"
+    },
+    {
+      "id": "infrastructure",
+      "label": "Infrastructure",
+      "color": "#64748b"
+    }
+  ]
+}