mas-lab-component-controller 0.2.0__py3-none-any.whl

mas/lab/controller/deps.py

@@ -0,0 +1,172 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Shared dependencies and helpers for the FastAPI controller API."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import time
+from pathlib import Path
+
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from mas.lab.controller.constants import HIDDEN_FILES, LIBRARIES_DIR
+
+logger = logging.getLogger(__name__)
+
+_manifest_store = None
+_manifest_store_last_refresh = 0.0
+
+
+def _refresh_interval_seconds() -> float:
+    """Return manifest refresh throttle interval.
+
+    Set MAS_CONTROLLER_REFRESH_INTERVAL_S=0 to keep legacy refresh-on-every-call behavior.
+    """
+    raw = os.environ.get("MAS_CONTROLLER_REFRESH_INTERVAL_S", "1.0").strip()
+    try:
+        interval = float(raw)
+    except ValueError:
+        return 1.0
+    return max(0.0, interval)
+
+
+def store_from_request(request: Request):
+    """Prefer the app-injected store (tests); else the process singleton."""
+    return getattr(request.app.state, "manifest_store", None) or get_manifest_store()
+
+
+def get_manifest_store():
+    """Lazy ManifestStore — discovers *.lab dirs and manifest_libraries."""
+    global _manifest_store, _manifest_store_last_refresh
+    if _manifest_store is None:
+        from mas.lab.controller.manifest_store import ManifestStore
+
+        try:
+            from mas.lab.workspace import WorkspaceConfig
+
+            ws = WorkspaceConfig.load()
+        except Exception:
+            ws = None
+        _manifest_store = ManifestStore(ws)
+        _manifest_store_last_refresh = time.monotonic()
+    else:
+        now = time.monotonic()
+        interval = _refresh_interval_seconds()
+        if interval <= 0.0 or (now - _manifest_store_last_refresh) >= interval:
+            _manifest_store.refresh()
+            _manifest_store_last_refresh = now
+    return _manifest_store
+
+
+def get_library_path(library_name: str) -> Path:
+    """Resolve a library name to its directory path."""
+    store = get_manifest_store()
+    try:
+        return store.library_root(library_name)
+    except KeyError:
+        pass
+    lib_dir = LIBRARIES_DIR / library_name
+    if not lib_dir.exists() or not lib_dir.is_dir():
+        raise HTTPException(status_code=404, detail=f"Library '{library_name}' not found")
+    return lib_dir
+
+
+def discover_tools(base_dir: Path, namespaces: list[str] | None = None) -> list[dict]:
+    from mas.lab.controller.artifact_discovery import discover_tools
+
+    return discover_tools(base_dir, namespaces)
+
+
+def discover_skills(base_dir: Path, namespaces: list[str] | None = None) -> list[dict]:
+    from mas.lab.controller.artifact_discovery import discover_skills
+
+    return discover_skills(base_dir, namespaces)
+
+
+async def run_cli(
+    cmd: list[str],
+    cwd: Path,
+    timeout: int = 60,
+    env_override: dict[str, str] | None = None,
+) -> dict:
+    """Execute a CLI command synchronously (for quick operations like validate)."""
+    env = os.environ.copy()
+    if env_override:
+        env.update(env_override)
+
+    logger.info("Running: %s (cwd=%s)", " ".join(cmd), cwd)
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=str(cwd),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+    except asyncio.TimeoutError:
+        proc.kill()
+        await proc.communicate()
+        raise HTTPException(
+            status_code=504,
+            detail=f"Command timed out after {timeout}s: {' '.join(cmd)}",
+        )
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Command not found: {cmd[0]}. Ensure mas-runtime is on PATH.",
+        )
+
+    result = {
+        "exit_code": proc.returncode,
+        "stdout": stdout.decode(errors="replace"),
+        "stderr": stderr.decode(errors="replace"),
+        "command": " ".join(cmd),
+    }
+
+    if proc.returncode != 0:
+        result["error"] = f"Command failed with exit code {proc.returncode}"
+
+    return result
+
+
+def build_tree(root: Path, base: Path) -> list[dict]:
+    """Recursively build a file/folder tree structure."""
+    entries = []
+    try:
+        children_iter = sorted(root.iterdir())
+    except (PermissionError, OSError):
+        return entries
+    for child in children_iter:
+        if child.name in HIDDEN_FILES:
+            continue
+        rel = str(child.relative_to(base))
+        try:
+            if child.is_dir():
+                children = build_tree(child, base)
+                entries.append({"name": child.name, "path": rel, "type": "directory", "children": children})
+            else:
+                entries.append({"name": child.name, "path": rel, "type": "file", "size": child.stat().st_size})
+        except (FileNotFoundError, PermissionError, OSError):
+            entries.append({"name": child.name, "path": rel, "type": "file", "size": 0})
+    return entries
+
+
+def ensure_yaml_ext(name: str) -> str:
+    """Append .yaml extension when none is present."""
+    lower = name.lower()
+    if lower.endswith(".yaml") or lower.endswith(".yml"):
+        return name
+    return f"{name}.yaml"
+
+
+async def validate_overlay_content(content: str, lib_dir: Path) -> list[str] | None:
+    """Validate overlay YAML content. Returns error list or None if valid."""
+    from mas.lab.controller.manifest_validation import validate_overlay_yaml_content
+
+    return validate_overlay_yaml_content(content, base_dir=lib_dir)

mas/lab/controller/fastapi_app.py

@@ -0,0 +1,166 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""
+FastAPI server exposing mas-lab as a scope-based REST API.
+
+Run via controller daemon:
+    mas-lab serve --port 8090
+    mas-lab serve --host 0.0.0.0 --token-file ./token --cors-origin https://ui.example
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import logging
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from mas.lab.controller.constants import LIBRARIES_DIR, MAS_LAB_ROOT
+from mas.lab.controller.deps import _manifest_store, get_manifest_store
+from mas.lab.controller.jobs import Job, JobStatus, _jobs
+from mas.lab.controller.routes import register_routes
+from mas.lab.controller.settings import ControllerSettings, is_loopback_host
+from starlette.middleware.base import BaseHTTPMiddleware
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Probes and the connect screen must work before a token is entered.
+# Swagger and OpenAPI stay behind bearer when a token is configured.
+_OPEN_PATHS = {
+    "/health",
+    "/api/health",
+    "/api/info",
+}
+
+
+def _tokens_match(offered: str, expected: str) -> bool:
+    """Compare bearer secrets without leaking length via ``hmac.compare_digest``."""
+    left = hashlib.sha256(offered.encode("utf-8")).digest()
+    right = hashlib.sha256(expected.encode("utf-8")).digest()
+    return hmac.compare_digest(left, right)
+
+
+class BearerTokenMiddleware(BaseHTTPMiddleware):
+    """Require ``Authorization: Bearer`` when a token was configured."""
+
+    def __init__(self, app: FastAPI, token: str):
+        super().__init__(app)
+        self._token = token
+
+    async def dispatch(self, request: Request, call_next):
+        if request.url.path in _OPEN_PATHS or request.method == "OPTIONS":
+            return await call_next(request)
+        header = request.headers.get("authorization") or ""
+        scheme, _, offered = header.partition(" ")
+        got = offered.strip() if scheme.lower() == "bearer" else ""
+        if not _tokens_match(got, self._token):
+            return JSONResponse({"detail": "unauthorized"}, status_code=401)
+        return await call_next(request)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Emit discovery report once the API process starts."""
+    if os.environ.get("MAS_CONTROLLER_DISABLE_STARTUP_DISCOVERY", "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+    }:
+        yield
+        return
+    try:
+        store = get_manifest_store()
+        report = store._registry.discovery_report()
+        logger.info("MAS Lab API startup discovery: %s", report)
+    except Exception as exc:
+        logger.warning("Startup discovery report failed: %s", exc)
+    yield
+
+
+def create_app(settings: ControllerSettings | None = None) -> FastAPI:
+    """Build and configure the FastAPI application."""
+    settings = settings or ControllerSettings()
+    application = FastAPI(
+        title="MAS Lab API",
+        description="REST interface for mas-lab — scope-based HTTP API with persisted runs",
+        version=settings.api_version,
+        lifespan=lifespan,
+    )
+    application.state.settings = settings
+
+    # Last-added middleware is outermost. Bearer must sit inside CORS so a
+    # 401 JSONResponse still receives Access-Control-Allow-Origin.
+    if settings.token:
+        application.add_middleware(BearerTokenMiddleware, token=settings.token)
+        if not is_loopback_host(settings.host):
+            logger.warning(
+                "Bearer auth is enabled; clients must send Authorization: Bearer"
+            )
+    elif not is_loopback_host(settings.host):
+        raise RuntimeError(
+            f"refusing to bind {settings.host} without --token-file"
+        )
+
+    origins = settings.cors_allow_origins
+    if origins:
+        application.add_middleware(
+            CORSMiddleware,
+            allow_origins=origins,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+    from mas.lab.controller.job_store import init_job_store
+    from mas.lab.controller.run_store import init_run_store
+    from mas.lab.controller.uploads import init_upload_store
+    from mas.lab.controller.workspace_library import init_workspace_library, reset_workspace_library
+
+    project_root = None
+    try:
+        from mas.lab.workspace import project_workspace_root
+
+        project_root = project_workspace_root()
+    except Exception:
+        project_root = None
+
+    if settings.state_dir is not None:
+        state_dir: Path = Path(settings.state_dir)
+        state_dir.mkdir(parents=True, exist_ok=True)
+        init_job_store(db_path=state_dir / "jobs.sqlite")
+        init_run_store(state_dir / "runs.sqlite")
+        init_upload_store(state_dir / "uploads")
+        init_workspace_library(project_root or state_dir)
+    else:
+        import tempfile
+
+        init_job_store(db_path=None)
+        init_run_store(None)
+        init_upload_store(Path(tempfile.mkdtemp(prefix="mas-uploads-")))
+        if project_root is not None:
+            init_workspace_library(project_root)
+        else:
+            reset_workspace_library()
+
+    register_routes(application)
+    return application
+
+
+app = create_app()
+
+
+__all__ = [
+    "app",
+    "create_app",
+    "Job",
+    "JobStatus",
+    "LIBRARIES_DIR",
+    "MAS_LAB_ROOT",
+    "_jobs",
+    "_manifest_store",
+]

mas/lab/controller/http_server.py

@@ -0,0 +1,243 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""HTTP API for mas-lab-ui (stdlib ThreadingHTTPServer)."""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any, Callable, Optional
+from urllib.parse import parse_qs, unquote, urlparse
+
+from mas.lab.controller.api import ControllerAPI
+
+logger = logging.getLogger(__name__)
+
+_CORS = {
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
+    "Access-Control-Allow-Headers": "Content-Type",
+}
+
+
+class ControllerHTTPHandler(BaseHTTPRequestHandler):
+    api: ControllerAPI | None = None
+
+    def log_message(self, fmt: str, *args: Any) -> None:
+        logger.debug(fmt, *args)
+
+    def _read_json(self) -> dict:
+        length = int(self.headers.get("Content-Length", "0"))
+        if length <= 0:
+            return {}
+        return json.loads(self.rfile.read(length).decode("utf-8"))
+
+    def _send(self, status: int, payload: Any) -> None:
+        body = json.dumps(payload).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        for key, value in _CORS.items():
+            self.send_header(key, value)
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _error(self, status: int, detail: str) -> None:
+        self._send(status, {"detail": detail})
+
+    def do_OPTIONS(self) -> None:  # noqa: N802
+        self.send_response(204)
+        for key, value in _CORS.items():
+            self.send_header(key, value)
+        self.end_headers()
+
+    def do_GET(self) -> None:  # noqa: N802
+        api = self.api
+        if api is None:
+            return self._error(500, "API not initialized")
+        path = urlparse(self.path).path
+        query = parse_qs(urlparse(self.path).query)
+
+        if path == "/api/health":
+            return self._send(200, {"status": "ok"})
+        if path == "/api/libraries":
+            return self._send(200, {"libraries": api.list_libraries()})
+        if path == "/api/jobs":
+            status = (query.get("status") or [None])[0]
+            return self._send(200, {"jobs": api.list_jobs(status=status)})
+        if path.startswith("/api/jobs/"):
+            job_id = path.split("/", 3)[-1]
+            job = api.get_job(job_id)
+            if job is None:
+                return self._error(404, f"Job {job_id} not found")
+            return self._send(200, job)
+        if path == "/api/pipeline-step-types":
+            return self._send(200, api.pipeline_step_types())
+        if path == "/api/metrics/eval":
+            return self._send(200, {})
+        if path == "/api/metrics/mce":
+            return self._send(200, {})
+
+        m = re.match(r"^/api/libraries/([^/]+)/tools$", path)
+        if m:
+            return self._send(200, {"tools": api.list_tools(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/skills$", path)
+        if m:
+            return self._send(200, {"skills": api.list_skills(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/experiments$", path)
+        if m:
+            return self._send(200, {"experiments": api.list_experiments(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/experiments/([^/]+)$", path)
+        if m:
+            library, name = unquote(m.group(1)), unquote(m.group(2))
+            try:
+                return self._send(200, api.get_experiment_content(library, name))
+            except FileNotFoundError:
+                return self._error(404, "experiment not found")
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines$", path)
+        if m:
+            return self._send(200, {"pipelines": api.list_pipelines(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines/([^/]+)$", path)
+        if m:
+            library, name = unquote(m.group(1)), unquote(m.group(2))
+            try:
+                return self._send(200, api.get_pipeline_content(library, name))
+            except FileNotFoundError:
+                return self._error(404, "pipeline not found")
+        m = re.match(r"^/api/libraries/([^/]+)/overlays$", path)
+        if m:
+            return self._send(200, {"overlays": api.list_overlays(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/overlays/([^/]+)$", path)
+        if m:
+            library, name = unquote(m.group(1)), unquote(m.group(2))
+            try:
+                return self._send(200, api.get_overlay_content(library, name))
+            except FileNotFoundError:
+                return self._error(404, "overlay not found")
+        m = re.match(r"^/api/libraries/([^/]+)/datasets$", path)
+        if m:
+            return self._send(200, {"datasets": api.list_datasets(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/datasets/([^/]+)$", path)
+        if m:
+            library, name = unquote(m.group(1)), unquote(m.group(2))
+            try:
+                return self._send(200, api.get_dataset_content(library, name))
+            except FileNotFoundError:
+                return self._error(404, "dataset not found")
+        m = re.match(r"^/api/libraries/([^/]+)/scenarios$", path)
+        if m:
+            return self._send(200, {"scenarios": api.list_scenarios(unquote(m.group(1)))})
+        m = re.match(r"^/api/libraries/([^/]+)/config-files$", path)
+        if m:
+            return self._send(200, api.config_files(unquote(m.group(1))))
+        m = re.match(r"^/api/libraries/([^/]+)/apps$", path)
+        if m:
+            return self._send(200, {"mas_resources": {}})
+
+        self._error(404, "not found")
+
+    def do_POST(self) -> None:  # noqa: N802
+        return self._dispatch_write("POST")
+
+    def do_PUT(self) -> None:  # noqa: N802
+        return self._dispatch_write("PUT")
+
+    def do_DELETE(self) -> None:  # noqa: N802
+        api = self.api
+        if api is None:
+            return self._error(500, "API not initialized")
+        path = urlparse(self.path).path
+        m = re.match(r"^/api/libraries/([^/]+)/experiments/([^/]+)$", path)
+        if m:
+            api.delete_experiment(unquote(m.group(1)), unquote(m.group(2)))
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines/([^/]+)$", path)
+        if m:
+            api.delete_pipeline(unquote(m.group(1)), unquote(m.group(2)))
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/overlays/([^/]+)$", path)
+        if m:
+            api.delete_overlay(unquote(m.group(1)), unquote(m.group(2)))
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/datasets/([^/]+)$", path)
+        if m:
+            api.delete_dataset(unquote(m.group(1)), unquote(m.group(2)))
+            return self._send(200, {"ok": True})
+        self._error(404, "not found")
+
+    def _dispatch_write(self, method: str) -> None:
+        api = self.api
+        if api is None:
+            return self._error(500, "API not initialized")
+        path = urlparse(self.path).path
+        body = self._read_json()
+
+        m = re.match(r"^/api/libraries/([^/]+)/validate$", path)
+        if m and method == "POST":
+            return self._send(200, api.validate_manifest_yaml(body.get("manifest_yaml") or ""))
+        m = re.match(r"^/api/libraries/([^/]+)/run$", path)
+        if m and method == "POST":
+            return self._send(200, api.run_agent_job(unquote(m.group(1)), body))
+        m = re.match(r"^/api/libraries/([^/]+)/run-mas$", path)
+        if m and method == "POST":
+            return self._send(200, api.run_mas_job(unquote(m.group(1)), body))
+        m = re.match(r"^/api/libraries/([^/]+)/benchmark/run$", path)
+        if m and method == "POST":
+            return self._send(200, api.run_benchmark_job(unquote(m.group(1)), body))
+        m = re.match(r"^/api/libraries/([^/]+)/pipeline/run$", path)
+        if m and method == "POST":
+            return self._send(200, api.run_pipeline_job(unquote(m.group(1)), body))
+        m = re.match(r"^/api/libraries/([^/]+)/experiments$", path)
+        if m and method == "POST":
+            api.save_experiment(unquote(m.group(1)), body["name"], body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/experiments/([^/]+)$", path)
+        if m and method == "PUT":
+            library = unquote(m.group(1))
+            api.save_experiment(library, body.get("name") or unquote(m.group(2)), body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines$", path)
+        if m and method == "POST":
+            api.save_pipeline(unquote(m.group(1)), body["name"], body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines/([^/]+)$", path)
+        if m and method == "PUT":
+            library = unquote(m.group(1))
+            api.save_pipeline(library, body.get("name") or unquote(m.group(2)), body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/overlays/validate$", path)
+        if m and method == "POST":
+            return self._send(200, {"status": "ok", "errors": []})
+        m = re.match(r"^/api/libraries/([^/]+)/overlays$", path)
+        if m and method == "POST":
+            api.save_overlay(unquote(m.group(1)), body["name"], body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/overlays/([^/]+)$", path)
+        if m and method == "PUT":
+            library = unquote(m.group(1))
+            api.save_overlay(library, body.get("name") or unquote(m.group(2)), body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/pipelines/validate$", path)
+        if m and method == "POST":
+            return self._send(200, {"status": "ok"})
+        m = re.match(r"^/api/libraries/([^/]+)/datasets$", path)
+        if m and method == "POST":
+            api.save_dataset(unquote(m.group(1)), body["name"], body["content"])
+            return self._send(200, {"ok": True})
+        m = re.match(r"^/api/libraries/([^/]+)/datasets/([^/]+)$", path)
+        if m and method == "PUT":
+            library = unquote(m.group(1))
+            api.save_dataset(library, body.get("name") or unquote(m.group(2)), body["content"])
+            return self._send(200, {"ok": True})
+
+        self._error(404, "not found")
+
+
+def start_http_server(*, api: ControllerAPI, host: str = "127.0.0.1", port: int = 9000) -> None:
+    class _Bound(ControllerHTTPHandler):
+        api = api
+
+    server = ThreadingHTTPServer((host, port), _Bound)
+    logger.info("Controller HTTP listening on http://%s:%s", host, port)
+    server.serve_forever()

mas/lab/controller/io_capture.py

@@ -0,0 +1,55 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+"""Capture stdout/stderr from worker threads into WorkerRecord buffers."""
+from __future__ import annotations
+
+import logging
+import sys
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mas.lab.controller.worker_model import WorkerRecord
+
+
+class _StreamCapture:
+    def __init__(self, record: "WorkerRecord", attr: str) -> None:
+        self._record = record
+        self._attr = attr
+
+    def write(self, text: str) -> int:
+        if text:
+            getattr(self._record, f"append_{self._attr}_chunk")(text)
+        return len(text)
+
+    def flush(self) -> None:
+        pass
+
+    def isatty(self) -> bool:
+        return False
+
+
+@contextmanager
+def capture_worker_io(record: "WorkerRecord"):
+    """Redirect sys.stdout/stderr and logging into the worker record.
+
+    Logging handlers are bound when the daemon starts (often to daemon.log),
+    so replacing sys.stderr alone does not move ``logger.error`` into the
+    stream the CLI polls.  A temporary StreamHandler writes those lines
+    into the worker record as well.
+    """
+    out = _StreamCapture(record, "stdout")
+    err = _StreamCapture(record, "stderr")
+    old_out, old_err = sys.stdout, sys.stderr
+    handler = logging.StreamHandler(err)
+    handler.setLevel(logging.NOTSET)
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    root = logging.getLogger()
+    sys.stdout, sys.stderr = out, err  # type: ignore[assignment]
+    root.addHandler(handler)
+    try:
+        yield
+    finally:
+        root.removeHandler(handler)
+        handler.close()
+        sys.stdout, sys.stderr = old_out, old_err