mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/routes/scopes.py

@@ -0,0 +1,139 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Scope listing — libraries and labs as first-class catalogs."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from mas.lab.controller.deps import store_from_request
+from mas.lab.controller.item_service import ItemError, group_items
+from mas.lab.controller.kinds import LAB_KINDS, LIBRARY_BASE_KINDS
+from mas.lab.controller.scope_catalog import get_scope_catalog
+from mas.lab.controller.scope_service import create_lab
+from pydantic import BaseModel, Field
+
+router = APIRouter()
+
+
+class CreateScopeRequest(BaseModel):
+    kind: str = "lab"
+    name: str
+    description: Optional[str] = ""
+    local_library: bool = Field(default=True)
+
+
+@router.get("/api/scopes", tags=["Scopes"])
+async def list_scopes(request: Request):
+    catalog = get_scope_catalog(store_from_request(request))
+    return {"scopes": [_public_scope(scope, counts=True) for scope in catalog.all()]}
+
+
+@router.post("/api/scopes", tags=["Scopes"], status_code=201)
+async def create_scope(req: CreateScopeRequest, request: Request):
+    if req.kind != "lab":
+        raise HTTPException(status_code=400, detail=f"cannot create scope kind {req.kind!r}")
+    store = store_from_request(request)
+    try:
+        scope = create_lab(
+            name=req.name,
+            description=req.description or "",
+            local_library=req.local_library,
+            store=store,
+        )
+    except ItemError as exc:
+        raise HTTPException(status_code=exc.status, detail=exc.detail) from exc
+    return _public_scope(scope, counts=True)
+
+
+@router.get("/api/scopes/{scope_id}", tags=["Scopes"])
+async def get_scope(scope_id: str, request: Request):
+    catalog = get_scope_catalog(store_from_request(request))
+    try:
+        scope = catalog.get(scope_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"scope {scope_id!r} not found") from exc
+    return _public_scope(scope, counts=True)
+
+
+@router.get("/api/scopes/{scope_id}/config", tags=["Scopes"])
+async def scope_config(scope_id: str, request: Request):
+    """Infra / flavour / workspace YAML for the control panel (basename keys only)."""
+    catalog = get_scope_catalog(store_from_request(request))
+    try:
+        scope = catalog.get(scope_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"scope {scope_id!r} not found") from exc
+    return _public_config(scope)
+
+
+@router.get("/api/scopes/{scope_id}/visible-libraries", tags=["Scopes"])
+async def visible_libraries(scope_id: str, request: Request):
+    catalog = get_scope_catalog(store_from_request(request))
+    try:
+        libraries = catalog.visible_libraries(scope_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"scope {scope_id!r} not found") from exc
+    return {"libraries": [s.to_public() for s in libraries]}
+
+
+def _public_scope(scope, *, counts: bool = False) -> dict:
+    payload = scope.to_public()
+    payload["kinds"] = _kinds_for(scope)
+    if not counts:
+        return payload
+    tallies: dict[str, int] = {}
+    for kind in payload["kinds"]:
+        try:
+            tallies[kind] = len(group_items(scope, kind))
+        except KeyError:
+            tallies[kind] = 0
+    payload["counts"] = tallies
+    return payload
+
+
+def _kinds_for(scope) -> list[str]:
+    if scope.kinds:
+        kinds = list(scope.kinds)
+    elif scope.kind == "lab":
+        kinds = list(LAB_KINDS)
+    else:
+        kinds = list(LIBRARY_BASE_KINDS)
+    if scope.kind != "library":
+        return kinds
+    # Advertise experiments/pipelines when the library actually has them.
+    from mas.lab.controller.manifest_layout import layout_for
+
+    for extra in ("experiment", "pipeline"):
+        if extra in kinds:
+            continue
+        try:
+            if layout_for(extra).list_located(scope):
+                kinds.append(extra)
+        except Exception:
+            continue
+    return kinds
+
+
+def _public_config(scope) -> dict[str, dict[str, str]]:
+    from pathlib import Path
+
+    from mas.lab.controller.lab_registry import LabRegistry
+    from mas.runtime.constants import WORKSPACE_CONFIG_FILENAME
+
+    groups: dict[str, dict[str, str]] = {"infra": {}, "flavours": {}, "workspace": {}}
+    root = scope.root
+    for group, folder_name in (("infra", "infra"), ("flavours", "flavours")):
+        folder = root / folder_name
+        if not folder.is_dir():
+            continue
+        for path in sorted(folder.iterdir()):
+            if path.is_file() and path.suffix in {".yaml", ".yml"}:
+                groups[group][path.name] = path.read_text(encoding="utf-8")
+    ws = root / WORKSPACE_CONFIG_FILENAME
+    if ws.is_file():
+        groups["workspace"][ws.name] = ws.read_text(encoding="utf-8")
+    for key, content in LabRegistry.discover_bundled_infra().items():
+        groups["infra"].setdefault(Path(key).name, content)
+    return groups

mas/lab/controller/routes/uploads.py

@@ -0,0 +1,24 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Upload files and get an opaque upload_id back."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, File, HTTPException, UploadFile
+from mas.lab.controller.uploads import get_upload_store
+
+router = APIRouter()
+
+
+@router.post("/api/uploads", tags=["Uploads"], status_code=201)
+async def upload(file: Annotated[UploadFile, File()]):
+    data = await file.read()
+    if not data:
+        raise HTTPException(status_code=400, detail="empty upload")
+    try:
+        item = get_upload_store().save(file.filename or "upload", data)
+    except ValueError as exc:
+        raise HTTPException(status_code=413, detail=str(exc)) from exc
+    return item.public_dict()

mas/lab/controller/routes/validate.py

@@ -0,0 +1,36 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Manifest validation endpoints."""
+
+from __future__ import annotations
+
+import yaml
+from fastapi import APIRouter
+
+from mas.lab.controller.manifest_validation import validate_manifest_yaml_content
+from mas.lab.controller.models import ValidateRequest
+from mas.lab.controller.routes._api import deps
+
+router = APIRouter()
+
+
+@router.post("/api/libraries/{library_name}/validate", tags=["Libraries"])
+async def validate_manifest(library_name: str, req: ValidateRequest):
+    """Validate an agent or MAS manifest in-process. Resolves refs relative to the library dir."""
+    lib_dir = deps.get_library_path(library_name)
+    # Unsaved drafts use __MAS_NAME__ placeholder — skip file-based ref
+    # resolution since the agent files don't exist on disk yet.
+    resolve = "__MAS_NAME__" not in req.manifest_yaml
+
+    base_dir = lib_dir
+    try:
+        data = yaml.safe_load(req.manifest_yaml)
+        kind = (data.get("kind") or "").lower() if isinstance(data, dict) else ""
+        mas_name = data.get("metadata", {}).get("name", "") if isinstance(data, dict) else ""
+        if mas_name:
+            app_dir = lib_dir / "apps" / mas_name
+            base_dir = app_dir / "agents" if kind == "agent" else app_dir
+    except Exception:
+        pass
+
+    return validate_manifest_yaml_content(req.manifest_yaml, base_dir=base_dir, resolve_refs=resolve)

mas/lab/controller/run_bridge.py

@@ -0,0 +1,276 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Mirror controller workers into the HTTP run store so CLI and UI share one catalog."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import threading
+from pathlib import Path
+from typing import Any
+
+from mas.lab.controller.api import peek_controller_api
+from mas.lab.controller.run_store import _now, get_run_store, new_run
+
+_TERMINAL = {"completed", "failed", "cancelled"}
+_dropped: set[str] = set()
+_cleanup_by_run: dict[str, list[Path]] = {}
+_lock = threading.RLock()
+
+_WORKER_KIND_TO_ITEM = {
+    "benchmark": "experiment",
+    "experiment": "experiment",
+    "pipeline": "pipeline",
+    "application": "application",
+}
+
+
+def submit_run(api, req, path: Path, cwd: Path) -> dict[str, Any]:
+    """Submit the same worker the CLI uses for this kind."""
+    ref = {
+        "origin": "ui",
+        "scope": req.scope,
+        "kind": req.kind,
+        "item": req.item,
+    }
+    common = {
+        "cwd": str(cwd),
+        "scope": req.scope,
+        "item": req.item,
+        "demo_ui_ref": ref,
+    }
+    if req.kind == "experiment":
+        spec = {
+            **common,
+            "experiment_yaml": str(path),
+            "progress": True,
+        }
+        return api.submit_benchmark(spec)
+    if req.kind == "pipeline":
+        spec = {**common, "pipeline_yaml": str(path)}
+        return api.submit_pipeline(spec)
+    spec = {
+        **common,
+        "spec_path": str(path),
+        "prompt": req.query or "",
+        "flavour": req.bind,
+        "config": {"overlays": list(req.overlays or [])},
+        "command": f"mas-ctl run-mas {path}",
+    }
+    return api.submit_application(spec)
+
+
+def apply_worker_to_run(run, detail: dict[str, Any], *, cwd: Path | None = None) -> None:  # noqa: ARG001
+    status = str(detail.get("status") or "running")
+    run.status = {
+        "pending": "pending",
+        "running": "running",
+        "completed": "completed",
+        "failed": "failed",
+        "cancelled": "failed",
+    }.get(status, status)
+    run.exit_code = detail.get("exit_code")
+    if detail.get("started_at"):
+        run.started_at = detail["started_at"]
+    elif run.status == "running" and not run.started_at:
+        run.started_at = _now()
+    run.finished_at = detail.get("finished_at")
+    lines: list[str] = []
+    query = (run.request_summary or {}).get("query")
+    if query:
+        lines.append(str(query))
+    stdout = (detail.get("stdout") or "").rstrip()
+    stderr = (detail.get("stderr") or "").rstrip()
+    error = (detail.get("error") or "").rstrip()
+    if stdout:
+        lines.extend(stdout.splitlines())
+    if stderr:
+        lines.extend(stderr.splitlines())
+    if error:
+        lines.append(error)
+    progress = progress_line(detail.get("request_body") or {})
+    if progress:
+        lines = [line for line in lines if not line.startswith("ok=")]
+        lines.append(progress)
+    run.log_lines = lines
+    spec = detail.get("request_body") or {}
+    output_dir = spec.get("output_dir")
+    if output_dir:
+        run.result_dir = str(output_dir)
+
+
+def progress_line(spec: dict[str, Any]) -> str | None:
+    """Same tally ``mas-lab benchmark follow`` prints from ``run_info.json``."""
+    out = spec.get("output_dir")
+    if not out:
+        yaml_path = spec.get("experiment_yaml")
+        if yaml_path:
+            out = str(Path(yaml_path).parent / "output")
+    if not out:
+        return None
+    root = Path(out)
+    if not root.is_dir():
+        return None
+    counts: dict[str, int] = {}
+    for path in root.rglob("run_info.json"):
+        try:
+            info = json.loads(path.read_text(encoding="utf-8"))
+            status = str(info.get("status", "unknown"))
+        except Exception:
+            status = "unknown"
+        counts[status] = counts.get(status, 0) + 1
+    if not counts:
+        return None
+    done = counts.get("ok", 0)
+    err = counts.get("error", 0)
+    return f"ok={done}  err={err}  done={done + err}"
+
+
+def locate_scope_item(path: Path, store, kind: str) -> tuple[str, str]:
+    from mas.lab.controller.manifest_layout import layout_for
+    from mas.lab.controller.scope_catalog import get_scope_catalog
+
+    try:
+        resolved = path.resolve()
+    except OSError:
+        resolved = path
+    try:
+        catalog = get_scope_catalog(store)
+        layout = layout_for(kind)
+    except Exception:
+        return "", path.stem
+    for scope in catalog.all():
+        if kind not in (scope.kinds or []):
+            continue
+        try:
+            located = layout.list_located(scope)
+        except Exception:
+            continue
+        for item in located:
+            try:
+                if item.manifest_path.resolve() == resolved:
+                    return scope.id, item.item_id
+            except OSError:
+                continue
+        try:
+            resolved.relative_to(scope.root.resolve())
+            return scope.id, path.stem
+        except ValueError:
+            continue
+    return "", path.stem
+
+
+def run_from_worker(detail: dict[str, Any], store) -> Any:
+    spec = detail.get("request_body") or {}
+    worker_kind = str(detail.get("kind") or spec.get("kind") or "")
+    kind = _WORKER_KIND_TO_ITEM.get(worker_kind, worker_kind or "experiment")
+    scope = str(spec.get("scope") or "")
+    item = str(spec.get("item") or "")
+    ref = spec.get("demo_ui_ref") if isinstance(spec.get("demo_ui_ref"), dict) else {}
+    if not scope:
+        scope = str(ref.get("scope") or "")
+    if not item:
+        item = str(ref.get("item") or "")
+    if ref.get("kind"):
+        kind = str(ref["kind"])
+    path_key = {
+        "experiment": "experiment_yaml",
+        "pipeline": "pipeline_yaml",
+        "application": "spec_path",
+    }.get(kind)
+    path = Path(spec[path_key]) if path_key and spec.get(path_key) else None
+    if (not scope or not item) and path is not None:
+        found_scope, found_item = locate_scope_item(path, store, kind)
+        scope = scope or found_scope
+        item = item or found_item
+    if not item:
+        item = path.stem if path is not None else (detail.get("id") or "run")
+    if not scope:
+        scope = "library:unknown"
+    source = "ui" if ref.get("origin") == "ui" else "cli"
+    run = new_run(
+        scope=scope,
+        kind=kind,
+        item=item,
+        request_summary={
+            "source": source,
+            "command": detail.get("command") or "",
+            "action": "run",
+        },
+    )
+    worker_id = detail.get("id")
+    if worker_id:
+        run.id = str(worker_id)
+        run.job_id = str(worker_id)
+    apply_worker_to_run(run, detail, cwd=Path(spec["cwd"]) if spec.get("cwd") else None)
+    return run
+
+
+def refresh_run(run) -> None:
+    api = peek_controller_api()
+    if api is None or not run.job_id:
+        return
+    detail = api.get_worker(run.job_id)
+    if detail is None:
+        return
+    cwd = None
+    spec = detail.get("request_body") or {}
+    if spec.get("cwd"):
+        cwd = Path(spec["cwd"])
+    apply_worker_to_run(run, detail, cwd=cwd)
+    if run.status not in {"pending", "running"}:
+        from mas.lab.controller.routes.runs import _attach_artifacts
+
+        _attach_artifacts(run, cwd)
+        _finish_cleanup(run.id)
+
+
+def sync_workers(store) -> None:
+    api = peek_controller_api()
+    if api is None:
+        return
+    runs = get_run_store()
+    with _lock:
+        dropped = set(_dropped)
+    by_job = {row.job_id: row for row in runs.list() if row.job_id}
+    for summary in api.list_workers():
+        worker_id = summary.get("id")
+        if not worker_id or worker_id in dropped:
+            continue
+        detail = api.get_worker(worker_id) or summary
+        # ``list_workers`` summaries omit kind/spec; prefer the detail record.
+        if "kind" not in detail and "kind" in summary:
+            detail = {**detail, "kind": summary["kind"]}
+        run = by_job.get(worker_id)
+        if run is None:
+            run = run_from_worker(detail, store)
+            runs.save(run)
+            by_job[worker_id] = run
+        else:
+            refresh_run(run)
+            runs.save(run)
+
+
+def register_cleanup(run_id: str, paths: list[Path]) -> None:
+    if paths:
+        with _lock:
+            _cleanup_by_run[run_id] = list(paths)
+
+
+def _finish_cleanup(run_id: str) -> None:
+    with _lock:
+        paths = _cleanup_by_run.pop(run_id, [])
+    for path in paths:
+        shutil.rmtree(path, ignore_errors=True)
+
+
+def forget_worker(worker_id: str) -> None:
+    with _lock:
+        _dropped.add(worker_id)
+
+
+def reset_follow_state() -> None:
+    with _lock:
+        _dropped.clear()
+        _cleanup_by_run.clear()

mas/lab/controller/run_store.py

@@ -0,0 +1,190 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Persisted runs (job store rebuilt on the run/artifact model)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+_CREATE = """\
+CREATE TABLE IF NOT EXISTS runs (
+    id               TEXT PRIMARY KEY,
+    scope            TEXT NOT NULL,
+    kind             TEXT NOT NULL,
+    item             TEXT NOT NULL,
+    status           TEXT NOT NULL,
+    started_at       TEXT,
+    finished_at      TEXT,
+    created_at       TEXT NOT NULL,
+    exit_code        INTEGER,
+    request_summary  TEXT NOT NULL DEFAULT '{}',
+    job_id           TEXT,
+    result_dir       TEXT
+);
+"""
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class Run:
+    id: str
+    scope: str
+    kind: str
+    item: str
+    status: str
+    created_at: str
+    started_at: Optional[str] = None
+    finished_at: Optional[str] = None
+    exit_code: Optional[int] = None
+    request_summary: dict[str, Any] = field(default_factory=dict)
+    job_id: Optional[str] = None
+    result_dir: Optional[str] = None
+    log_lines: list[str] = field(default_factory=list)
+
+    def to_public(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "scope": self.scope,
+            "kind": self.kind,
+            "item": self.item,
+            "status": self.status,
+            "created_at": self.created_at,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "exit_code": self.exit_code,
+            "request_summary": self.request_summary,
+            "job_id": self.job_id,
+            "log": "\n".join(self.log_lines),
+        }
+
+
+class RunStore:
+    def __init__(self, db_path: Path | None = None):
+        self._memory: dict[str, Run] = {}
+        self._conn: sqlite3.Connection | None = None
+        if db_path is not None:
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            self._conn = sqlite3.connect(str(db_path), check_same_thread=False)
+            self._conn.execute("PRAGMA journal_mode=WAL")
+            self._conn.execute(_CREATE)
+            self._conn.commit()
+            self._load()
+
+    def _load(self) -> None:
+        if self._conn is None:
+            return
+        rows = self._conn.execute(
+            "SELECT id, scope, kind, item, status, started_at, finished_at, "
+            "created_at, exit_code, request_summary, job_id, result_dir FROM runs"
+        ).fetchall()
+        for row in rows:
+            summary = row[9]
+            try:
+                summary_obj = json.loads(summary) if summary else {}
+            except json.JSONDecodeError:
+                summary_obj = {}
+            run = Run(
+                id=row[0],
+                scope=row[1],
+                kind=row[2],
+                item=row[3],
+                status=row[4],
+                started_at=row[5],
+                finished_at=row[6],
+                created_at=row[7],
+                exit_code=row[8],
+                request_summary=summary_obj,
+                job_id=row[10],
+                result_dir=row[11],
+            )
+            self._memory[run.id] = run
+
+    def save(self, run: Run) -> None:
+        self._memory[run.id] = run
+        if self._conn is None:
+            return
+        self._conn.execute(
+            "INSERT OR REPLACE INTO runs "
+            "(id, scope, kind, item, status, started_at, finished_at, created_at, "
+            "exit_code, request_summary, job_id, result_dir) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                run.id,
+                run.scope,
+                run.kind,
+                run.item,
+                run.status,
+                run.started_at,
+                run.finished_at,
+                run.created_at,
+                run.exit_code,
+                json.dumps(run.request_summary),
+                run.job_id,
+                run.result_dir,
+            ),
+        )
+        self._conn.commit()
+
+    def get(self, run_id: str) -> Run | None:
+        return self._memory.get(run_id)
+
+    def list(self) -> list[Run]:
+        return sorted(self._memory.values(), key=lambda r: r.created_at, reverse=True)
+
+    def delete(self, run_id: str) -> None:
+        self._memory.pop(run_id, None)
+        if self._conn is not None:
+            self._conn.execute("DELETE FROM runs WHERE id = ?", (run_id,))
+            self._conn.commit()
+
+    def close(self) -> None:
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+
+def new_run(*, scope: str, kind: str, item: str, request_summary: dict | None = None) -> Run:
+    return Run(
+        id=str(uuid.uuid4()),
+        scope=scope,
+        kind=kind,
+        item=item,
+        status="pending",
+        created_at=_now(),
+        request_summary=request_summary or {},
+    )
+
+
+_store: RunStore | None = None
+
+
+def get_run_store() -> RunStore:
+    global _store
+    if _store is None:
+        _store = RunStore()
+    return _store
+
+
+def init_run_store(db_path: Path | None) -> RunStore:
+    global _store
+    if _store is not None:
+        _store.close()
+    _store = RunStore(db_path)
+    return _store
+
+
+def set_run_store(store: RunStore | None) -> None:
+    global _store
+    _store = store