mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/routes/eval_output.py

@@ -0,0 +1,31 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""LLM-as-judge eval-output endpoint."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from mas.lab.controller.models import EvalOutputRequest
+from mas.lab.controller.routes._api import deps, jobs, LIBRARIES_DIR, validate_pipeline_yaml
+
+router = APIRouter()
+
+
+@router.post("/api/libraries/{library_name}/eval-output", tags=["Libraries"], status_code=202)
+async def eval_output(library_name: str, req: EvalOutputRequest):
+    """Run LLM-as-judge evaluation on an events file. Returns a job_id for polling."""
+    lib_dir = deps.get_library_path(library_name)
+
+    cmd = ["mas-lab", "eval", req.events_file]
+    for metric in req.metrics:
+        cmd.extend(["--metric", metric])
+    cmd.extend(["--model", req.model])
+    job = jobs.submit_job(
+        endpoint=f"/api/libraries/{library_name}/eval-output",
+        cmd=cmd,
+        cwd=lib_dir,
+        timeout=req.timeout,
+        request_body=req.model_dump(),
+    )
+    return {"job_id": job.id, "status": job.status.value, "command": job.command}

mas/lab/controller/routes/experiments.py

@@ -0,0 +1,194 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Library-scoped experiment definition CRUD endpoints."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException
+from mas.lab.controller.api import get_controller_api
+from mas.lab.controller.models import ExperimentRunRequest, SaveExperimentRequest
+from mas.lab.controller.routes._api import deps
+from mas.lab.controller.routes.run import _run_pre_steps
+
+router = APIRouter()
+
+
+def resolve_experiment_path(lib_dir: Path, name: str) -> Path:
+    """Resolve an experiment by path, filename, stem, or YAML ``experiment.name``."""
+    raw = Path(name)
+    candidates: list[Path] = []
+    if raw.is_absolute():
+        candidates.append(raw)
+    else:
+        candidates.append(lib_dir / name)
+        candidates.append(lib_dir / "experiments" / name)
+        if not name.endswith((".yaml", ".yml")):
+            candidates.extend(
+                [
+                    lib_dir / f"{name}.yaml",
+                    lib_dir / "experiments" / f"{name}.yaml",
+                    lib_dir / "experiment.yaml",
+                    lib_dir / "experiments" / "experiment.yaml",
+                ]
+            )
+    for path in candidates:
+        if path.is_file():
+            return path
+    wanted = name
+    if wanted.endswith(".yaml"):
+        wanted = wanted[:-5]
+    elif wanted.endswith(".yml"):
+        wanted = wanted[:-4]
+    for path in _experiment_yaml_files(lib_dir):
+        if path.stem == wanted or path.name == name:
+            return path
+        if _experiment_yaml_name(path) == wanted:
+            return path
+    raise HTTPException(status_code=404, detail=f"Experiment '{name}' not found")
+
+
+def _experiment_yaml_files(lib_dir: Path):
+    if not lib_dir.is_dir():
+        return
+    for path in (lib_dir / "experiment.yaml", lib_dir / "experiment.yml"):
+        if path.is_file():
+            yield path
+    experiments = lib_dir / "experiments"
+    if experiments.is_dir():
+        yield from sorted(experiments.glob("*.yaml"))
+        yield from sorted(experiments.glob("*.yml"))
+        yield from sorted(experiments.glob("**/experiment.yaml"))
+        yield from sorted(experiments.glob("**/experiment.yml"))
+
+
+def _experiment_yaml_name(path: Path) -> str:
+    try:
+        from mas.runtime.spec.source import load_yaml_file
+
+        doc = load_yaml_file(path)
+    except Exception:
+        return path.stem
+    block = doc.get("experiment") if isinstance(doc, dict) else None
+    if isinstance(block, dict) and block.get("name"):
+        return str(block["name"])
+    return path.stem
+
+
+@router.post("/api/libraries/{library_name}/experiment/run", tags=["Libraries"], status_code=202)
+async def experiment_run(library_name: str, req: ExperimentRunRequest):
+    """Start an in-process experiment worker that streams live to demo-ui."""
+    lib_dir = deps.get_library_path(library_name)
+    _run_pre_steps(lib_dir, req.pre)
+    experiment_path = resolve_experiment_path(lib_dir, req.experiment_yaml)
+    spec = {
+        "experiment_yaml": str(experiment_path),
+        "progress": False,
+        "live": True,
+        "single_run": True,
+        "force": req.force,
+        "max_runs": req.max_runs or 1,
+        "query": req.query,
+        "infra_refs": list(req.infra_refs or []),
+        "demo_ui_ingest_url": req.demo_ui_ingest_url,
+        "demo_ui_ref": req.demo_ui_ref or {},
+        "endpoint": f"/api/libraries/{library_name}/experiment/run",
+        "command": f"mas-lab experiment run {experiment_path}",
+    }
+    result = get_controller_api().submit_experiment(spec)
+    result["command"] = spec["command"]
+    return result
+
+
+@router.get("/api/libraries/{library_name}/experiments", tags=["Libraries"])
+async def list_library_experiments(library_name: str):
+    """List available experiment definitions with metadata."""
+    experiments = deps.get_manifest_store().list_experiments(library_name)
+    for exp in experiments:
+        exp.setdefault("library", library_name)
+    return {"experiments": experiments}
+
+
+@router.post("/api/libraries/{library_name}/experiments", tags=["Libraries"], status_code=201)
+async def create_experiment(library_name: str, req: SaveExperimentRequest):
+    """Create a new experiment definition in the library's experiments/ directory."""
+    lib_dir = deps.get_library_path(library_name)
+    exp_dir = lib_dir / "experiments"
+    exp_dir.mkdir(parents=True, exist_ok=True)
+
+    filename = f"{req.name}.yaml" if not req.name.endswith(".yaml") else req.name
+    file_path = exp_dir / filename
+
+    if file_path.exists():
+        raise HTTPException(status_code=409, detail=f"Experiment '{req.name}' already exists")
+
+    file_path.write_text(req.content, encoding="utf-8")
+    deps.get_manifest_store().invalidate()
+    return {"name": req.name, "filename": filename}
+
+
+@router.get("/api/libraries/{library_name}/experiments/{experiment_name}", tags=["Libraries"])
+async def get_library_experiment(library_name: str, experiment_name: str):
+    """Return the content of a specific experiment definition."""
+    store = deps.get_manifest_store()
+    lib_dir = deps.get_library_path(library_name)
+    filename = f"{experiment_name}.yaml" if not experiment_name.endswith(".yaml") else experiment_name
+    candidates = [
+        lib_dir / "experiments" / filename,
+        lib_dir / filename,
+    ]
+    candidates.extend(
+        p for p in store._registry._iter_experiment_files(lib_dir)
+        if p.name == filename or p.stem == experiment_name
+    )
+    file_path = next((p for p in candidates if p.exists()), None)
+
+    if file_path is None:
+        raise HTTPException(status_code=404, detail=f"Experiment '{experiment_name}' not found")
+
+    return {
+        "name": experiment_name,
+        "filename": file_path.name,
+        "content": file_path.read_text(encoding="utf-8"),
+    }
+
+
+@router.put("/api/libraries/{library_name}/experiments/{experiment_name}", tags=["Libraries"])
+async def update_experiment(library_name: str, experiment_name: str, req: SaveExperimentRequest):
+    """Update an existing experiment definition. Supports rename via req.name."""
+    lib_dir = deps.get_library_path(library_name)
+    exp_dir = lib_dir / "experiments"
+
+    old_filename = f"{experiment_name}.yaml" if not experiment_name.endswith(".yaml") else experiment_name
+    old_path = exp_dir / old_filename
+
+    if not old_path.exists():
+        raise HTTPException(status_code=404, detail=f"Experiment '{experiment_name}' not found")
+
+    new_filename = f"{req.name}.yaml" if not req.name.endswith(".yaml") else req.name
+    new_path = exp_dir / new_filename
+
+    if req.name != experiment_name and new_path.exists():
+        raise HTTPException(status_code=409, detail=f"Experiment '{req.name}' already exists")
+
+    if old_path != new_path:
+        old_path.unlink()
+    new_path.write_text(req.content, encoding="utf-8")
+    deps.get_manifest_store().invalidate()
+    return {"name": req.name, "filename": new_filename}
+
+
+@router.delete("/api/libraries/{library_name}/experiments/{experiment_name}", tags=["Libraries"])
+async def delete_library_experiment(library_name: str, experiment_name: str):
+    """Delete an experiment definition."""
+    lib_dir = deps.get_library_path(library_name)
+    filename = f"{experiment_name}.yaml" if not experiment_name.endswith(".yaml") else experiment_name
+    file_path = lib_dir / "experiments" / filename
+
+    if not file_path.exists():
+        raise HTTPException(status_code=404, detail=f"Experiment '{experiment_name}' not found")
+
+    file_path.unlink()
+    deps.get_manifest_store().invalidate()
+    return {"deleted": experiment_name}

mas/lab/controller/routes/health.py

@@ -0,0 +1,178 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Health, info, schema, and metrics endpoints."""
+
+from __future__ import annotations
+
+import json as _json
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import JSONResponse, PlainTextResponse
+from mas.lab.controller.constants import (
+    PIPELINE_STEP_TYPES_POST_PATH,
+    PIPELINE_STEP_TYPES_PRE_PATH,
+)
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@router.get("/api/health")
+async def api_health():
+    return {"status": "ok"}
+
+
+@router.get("/api/info")
+async def info(request: Request):
+    """Capabilities the UI uses to bootstrap a connection."""
+    settings = getattr(request.app.state, "settings", None)
+    auth = getattr(settings, "auth_mode", "none") if settings else "none"
+    api_version = getattr(settings, "api_version", "0.3.0") if settings else "0.3.0"
+    cors_ok = True
+    if settings is not None:
+        cors_ok = bool(settings.cors_allow_origins)
+    from mas.lab.controller.deps import store_from_request
+    from mas.lab.controller.routes.scopes import _kinds_for
+    from mas.lab.controller.scope_catalog import get_scope_catalog
+
+    catalog = get_scope_catalog(store_from_request(request))
+    edit_targets = [
+        {
+            "id": scope.id,
+            "kind": scope.kind,
+            "name": scope.name,
+            "origin": scope.origin,
+            "kinds": _kinds_for(scope),
+            "owner": scope.owner,
+        }
+        for scope in catalog.all()
+        if scope.writable
+    ]
+    return {
+        "api_version": api_version,
+        "features": [
+            "scopes",
+            "items",
+            "runs",
+            "artifacts",
+            "uploads",
+            "choices",
+            "sse",
+            "edit_targets",
+            "create_scope",
+        ],
+        "auth": auth,
+        "cors_ok": cors_ok,
+        "writable_default": any(scope.writable for scope in catalog.all()),
+        "edit_targets": edit_targets,
+    }
+
+
+@router.get("/api/schemas", tags=["Schemas"])
+async def list_manifest_schemas():
+    """List manifest schemas resolved from installed packages (runtime, bench, core).
+
+    mas-lab-ui should fetch schemas here instead of bundling copies.
+    """
+    from mas.lab.controller.schema_registry import list_schemas
+
+    return {"schemas": list_schemas()}
+
+
+@router.get("/api/schemas/{schema_id}", tags=["Schemas"])
+async def get_manifest_schema(
+    schema_id: str,
+    format: str | None = None,
+    resolved: bool = Query(False),
+):
+    """Return a manifest schema by registry id.
+
+    Query params:
+      - ``format=json`` — JSON body (required for ``resolved=1``).
+      - ``resolved=1`` — JSON Schema with local ``./`` refs inlined (for UI Ajv).
+        Uses the same loader as ``mas.ctl.validate`` — no filesystem access in the browser.
+    """
+    import yaml as _yaml
+    from mas.ctl.validate.schemas import load_schema, schema_path_for_kind
+    from mas.lab.controller.schema_registry import read_schema_text
+
+    want_resolved = resolved
+    want_json = (format or "").strip().lower() == "json" or want_resolved
+
+    if want_resolved:
+        kind = schema_id.replace("-", "_")
+        if schema_path_for_kind(kind) is None:
+            raise HTTPException(status_code=404, detail=f"No resolved schema for: {schema_id}")
+        try:
+            schema_dict = load_schema(kind)
+        except KeyError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        return JSONResponse(schema_dict)
+
+    try:
+        entry, text = read_schema_text(schema_id)
+    except KeyError:
+        raise HTTPException(status_code=404, detail=f"Unknown schema: {schema_id}") from None
+    except FileNotFoundError as exc:
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+    if want_json and entry.format == "yaml":
+        text = _json.dumps(_yaml.safe_load(text), indent=2)
+        media = "application/json"
+    elif entry.format == "json" or want_json:
+        media = "application/json"
+    else:
+        media = "text/yaml"
+    return PlainTextResponse(content=text, media_type=media)
+
+
+@router.get("/api/pipeline-step-types")
+async def get_pipeline_step_types(phase: str | None = None):
+    """Return the registry of all pipeline step types with config schemas for the React UI.
+
+    Query params:
+        phase: "pre" | "post" — return only pre-phase or post-phase step types.
+               If omitted, returns the post-phase step types.
+    """
+    if phase == "pre":
+        path = PIPELINE_STEP_TYPES_PRE_PATH
+    else:
+        path = PIPELINE_STEP_TYPES_POST_PATH
+    return _json.loads(path.read_text(encoding="utf-8"))
+
+
+@router.get("/api/metrics/eval", tags=["Pipelines"])
+async def get_eval_metrics():
+    """Return available metrics for the annotate_metrics pipeline step.
+
+    Keys are the metric_class values to use in pipeline YAML config.
+    Values are human-readable display names.
+    Used by step type: annotate_metrics (config.metric_class).
+    """
+    return {
+        "mas.lab.components.evaluation.deepeval_wrapper.AnswerRelevancyMetric": "Answer Relevancy",
+        "mas.lab.dataset.evaluator.BiasMetric": "Bias",
+    }
+
+
+@router.get("/api/metrics/mce", tags=["Pipelines"])
+async def get_mce_metrics():
+    """Return available MCE metrics for eval_batch steps.
+
+    Keys are the metric name strings to use in pipeline YAML config.metrics array.
+    Values are human-readable display names.
+    Used by step type: eval_batch (config.metrics).
+    """
+    return {
+        "answer_relevancy": "Answer Relevancy",
+        "goal_success_rate": "Goal Success Rate",
+        "groundedness": "Groundedness",
+        "response_completeness": "Response Completeness",
+        "task_delegation": "Task Delegation Accuracy",
+        "tool_utilization_accuracy": "Tool Utilization Accuracy",
+        "duration": "Duration",
+    }

mas/lab/controller/routes/items.py

@@ -0,0 +1,187 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Item catalog: list/get/create/update/fork/rename/delete/check."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from mas.lab.controller import item_service
+from mas.lab.controller.deps import store_from_request
+from mas.lab.controller.kinds import list_kinds
+from mas.lab.controller.scope_catalog import get_scope_catalog
+from pydantic import BaseModel, Field
+
+router = APIRouter()
+
+
+class CreateItemRequest(BaseModel):
+    kind: str
+    name: str
+    version: Optional[str] = None
+    template: Optional[str] = None
+    parent: Optional[str] = None
+    upload_id: Optional[str] = None
+
+
+class UpdateItemRequest(BaseModel):
+    manifest: str
+    children: dict[str, str] = Field(default_factory=dict)
+    revision: str
+
+
+class ForkItemRequest(BaseModel):
+    version: Optional[str] = None
+    name: Optional[str] = None
+    target_scope: Optional[str] = None
+
+
+class RenameItemRequest(BaseModel):
+    name: str
+
+
+class CheckItemRequest(BaseModel):
+    kind: str
+    name: str
+    version: Optional[str] = None
+
+
+class ValidateScopeRequest(BaseModel):
+    kind: Optional[str] = None
+    manifest: str
+
+
+def _raise(exc: item_service.ItemError) -> None:
+    raise HTTPException(status_code=exc.status, detail=exc.detail) from exc
+
+
+@router.get("/api/kinds", tags=["Kinds"])
+async def get_kinds():
+    return {"kinds": list_kinds()}
+
+
+@router.get("/api/scopes/{scope_id}/items", tags=["Items"])
+async def list_items(
+    scope_id: str,
+    request: Request,
+    kind: str = Query(...),
+    q: Optional[str] = None,
+):
+    store = store_from_request(request)
+    try:
+        scope = get_scope_catalog(store).get(scope_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"scope {scope_id!r} not found") from exc
+    try:
+        groups = item_service.group_items(scope, kind, q=q)
+    except KeyError as exc:
+        raise HTTPException(status_code=400, detail=f"unknown kind {kind!r}") from exc
+    return {"items": groups}
+
+
+@router.get("/api/scopes/{scope_id}/items/{kind}/{item_id}", tags=["Items"])
+async def get_item(scope_id: str, kind: str, item_id: str, request: Request):
+    try:
+        return item_service.get_item(scope_id, kind, item_id, store=store_from_request(request))
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.post("/api/scopes/{scope_id}/items", tags=["Items"], status_code=201)
+async def create_item(scope_id: str, req: CreateItemRequest, request: Request):
+    try:
+        return item_service.create_item(
+            scope_id,
+            kind=req.kind,
+            name=req.name,
+            version=req.version,
+            template=req.template,
+            parent=req.parent,
+            upload_id=req.upload_id,
+            store=store_from_request(request),
+        )
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.put("/api/scopes/{scope_id}/items/{kind}/{item_id}", tags=["Items"])
+async def update_item(scope_id: str, kind: str, item_id: str, req: UpdateItemRequest, request: Request):
+    try:
+        return item_service.update_item(
+            scope_id,
+            kind,
+            item_id,
+            manifest=req.manifest,
+            children=req.children,
+            revision=req.revision,
+            store=store_from_request(request),
+        )
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.post("/api/scopes/{scope_id}/items/{kind}/{item_id}/fork", tags=["Items"], status_code=201)
+async def fork_item(scope_id: str, kind: str, item_id: str, req: ForkItemRequest, request: Request):
+    try:
+        return item_service.fork_item(
+            scope_id,
+            kind,
+            item_id,
+            version=req.version,
+            name=req.name,
+            target_scope=req.target_scope,
+            store=store_from_request(request),
+        )
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.post("/api/scopes/{scope_id}/items/{kind}/{item_id}/rename", tags=["Items"])
+async def rename_item(scope_id: str, kind: str, item_id: str, req: RenameItemRequest, request: Request):
+    try:
+        return item_service.rename_item(scope_id, kind, item_id, name=req.name, store=store_from_request(request))
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.delete("/api/scopes/{scope_id}/items/{kind}/{item_id}", tags=["Items"])
+async def delete_item(scope_id: str, kind: str, item_id: str, request: Request):
+    try:
+        item_service.delete_item(scope_id, kind, item_id, store=store_from_request(request))
+    except item_service.ItemError as exc:
+        _raise(exc)
+    return {"ok": True}
+
+
+@router.post("/api/scopes/{scope_id}/items/check", tags=["Items"])
+async def check_item(scope_id: str, req: CheckItemRequest, request: Request):
+    return item_service.check_name(
+        scope_id, req.kind, req.name, req.version, store=store_from_request(request)
+    )
+
+
+@router.get("/api/resolve", tags=["Items"])
+async def resolve(ref: str, request: Request):
+    try:
+        return item_service.resolve_ref(ref, store=store_from_request(request))
+    except item_service.ItemError as exc:
+        _raise(exc)
+
+
+@router.post("/api/scopes/{scope_id}/validate", tags=["Items"])
+async def validate_in_scope(scope_id: str, req: ValidateScopeRequest, request: Request):
+    from mas.lab.controller.manifest_validation import validate_manifest_yaml_content
+    from mas.lab.controller.scope_catalog import get_scope_catalog as catalog_of
+
+    try:
+        scope = catalog_of(store_from_request(request)).get(scope_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"scope {scope_id!r} not found") from exc
+    result = validate_manifest_yaml_content(
+        req.manifest,
+        base_dir=scope.root,
+        kind=req.kind,
+        resolve_refs=True,
+    )
+    return result