mlspace-plugin 0.2.1__py3-none-any.whl

mlspace_mcp/resource_refs.py

@@ -0,0 +1,188 @@
+"""Validated resource addresses and small jobs/notebooks operation adapters.
+
+A reference is an address, not permission to perform the operation. Output
+references are best-effort and only read recognized resource response shapes.
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator
+
+from .errors import MLSpaceError
+
+
+class Address(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+
+    @field_validator("*", check_fields=False)
+    @classmethod
+    def safe_segment(cls, value: Any) -> Any:
+        if isinstance(value, str) and (
+            not value.strip() or value in {".", ".."}
+            or re.search(r"[/\\?#\x00-\x1f\x7f]", value)
+        ):
+            raise ValueError("address fields must be non-empty opaque names or IDs")
+        return value
+
+
+class JobAddress(Address):
+    job_name: str
+    region: str | None = None
+
+
+class NotebookAddress(Address):
+    notebook_uuid: str | None = None
+    notebook_name: str | None = None
+    namespace: str | None = None
+    region: str | None = None
+
+    @model_validator(mode="after")
+    def has_identity(self) -> NotebookAddress:
+        if not self.notebook_uuid and not (self.notebook_name and self.namespace):
+            raise ValueError("notebook address requires UUID or name and namespace")
+        return self
+
+
+class ResourceRef(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+    version: Literal[1]
+    environment: str
+    target: str
+    kind: Literal["job", "notebook"]
+    address: JobAddress | NotebookAddress
+
+    @model_validator(mode="after")
+    def matching_kind(self) -> ResourceRef:
+        if (self.kind == "job") != isinstance(self.address, JobAddress):
+            raise ValueError("resource kind does not match address")
+        return self
+
+
+def parse_reference(value: Any, environment: str) -> ResourceRef | None:
+    if value is None:
+        return None
+    try:
+        ref = ResourceRef.model_validate(value)
+    except (ValidationError, TypeError, ValueError) as exc:
+        # Do not echo arbitrary input (it may contain credentials or huge payloads).
+        raise MLSpaceError("Invalid resource_ref: use the typed resource address schema.") from exc
+    if ref.environment != environment:
+        raise MLSpaceError("resource_ref belongs to a different configured environment.")
+    return ref
+
+
+def _supply(values: dict[str, Any], name: str, value: Any) -> None:
+    if value is None:
+        return
+    if name in values and values[name] != value:
+        raise MLSpaceError(f"resource_ref conflicts with explicit {name}.")
+    values[name] = value
+
+
+def apply_reference(ref: ResourceRef | None, domain: str, action: str,
+                    values: dict[str, Any]) -> None:
+    if ref is None:
+        return
+    if domain == "jobs" and ref.kind == "job" and isinstance(ref.address, JobAddress):
+        if action not in {"get", "logs", "list_nodes", "list_pods", "get_params",
+                          "get_preemptors", "delete", "restart"}:
+            raise MLSpaceError("resource_ref is not supported for this jobs action.")
+        _supply(values, "job_name", ref.address.job_name)
+        _supply(values, "region", ref.address.region)
+        if action == "restart":
+            body = values.get("body", {})
+            if not isinstance(body, dict):
+                raise MLSpaceError("resource_ref restart requires an object body.")
+            body = dict(body)
+            _supply(body, "job_name", ref.address.job_name)
+            values["body"] = body
+        return
+    if domain == "notebooks" and ref.kind == "notebook" and isinstance(ref.address, NotebookAddress):
+        address = ref.address
+        if action in {"users_list", "users_set", "users_revoke"}:
+            if not (address.notebook_name and address.namespace):
+                raise MLSpaceError("resource_ref for users actions requires notebook name and namespace.")
+        elif action in {"get", "delete", "pause", "resume", "modify"}:
+            if not address.notebook_uuid:
+                raise MLSpaceError("resource_ref for this notebook action requires notebook UUID.")
+        else:
+            raise MLSpaceError("resource_ref is not supported for this notebooks action.")
+        # Body region may be a legitimate resume/modify override, not an address.
+        for name, value in address.model_dump(exclude_none=True).items():
+            _supply(values, name, value)
+        return
+    raise MLSpaceError("resource_ref kind is not supported for this domain/action.")
+
+
+def response_references(domain: str, action: str, data: Any, *, environment: str,
+                        target: str, values: dict[str, Any], list_cap: int) -> list[dict[str, Any]]:
+    """No recursive id/name scan: only known jobs/notebooks resource operations."""
+    allowed = {"jobs": {"list", "get", "run", "restart"},
+               "notebooks": {"list", "get", "create", "resume", "modify"}}
+    if action not in allowed.get(domain, set()):
+        return []
+    if action == "list":
+        rows = data if isinstance(data, list) else None
+        if isinstance(data, dict):
+            for key in ("items", "jobs", "notebooks", "results"):
+                if isinstance(data.get(key), list):
+                    rows = data[key]
+                    break
+        rows = rows[:list_cap] if isinstance(rows, list) else []
+    else:
+        rows = [data]
+    refs = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        # These domain endpoints are workspace-scoped. An explicit owner wins;
+        # a foreign owner is not inferred from the context that observed it.
+        owner = row.get("workspace_id", target)
+        if not isinstance(owner, str) or not owner:
+            continue
+        body = values.get("body")
+        region = row.get("region") or values.get("region")
+        if region is None and isinstance(body, dict):
+            region = body.get("region")
+        if domain == "jobs":
+            identity = row.get("job_name")
+            # A restart must confirm a NEW identity, not repeat the old address.
+            original_name = body.get("job_name") if isinstance(body, dict) else None
+            if action == "restart" and identity == original_name:
+                continue
+            address = {"job_name": identity, "region": region}
+            kind = "job"
+        else:
+            address = {"notebook_uuid": row.get("uid") or row.get("uuid"),
+                       "notebook_name": row.get("name"),
+                       "namespace": row.get("namespace") or values.get("namespace"),
+                       "region": region}
+            kind = "notebook"
+        try:
+            reference = ResourceRef.model_validate({"version": 1, "environment": environment,
+                "target": owner, "kind": kind, "address": address})
+        except (ValidationError, TypeError, ValueError):
+            continue  # an applied write must remain success even if its new address is unknown
+        refs.append(reference.model_dump(exclude_none=True))
+    return refs
+
+
+def with_context(formatted: str, *, environment: str, target: str, workspace_name: str,
+                 refs: list[dict[str, Any]], response_format: str,
+                 kind: str, address_note: str | None = None) -> str:
+    data: Any = formatted
+    if response_format == "json" and kind != "log":
+        try:
+            data = json.loads(formatted)
+        except (ValueError, TypeError):
+            pass
+    result = {"request_context": {"environment": environment, "target": target,
+                                  "workspace_name": workspace_name},
+              "data": data, "resource_refs": refs}
+    if address_note:
+        result["address_note"] = address_note
+    text = json.dumps(result, ensure_ascii=False)
+    return f"```json\n{text}\n```" if response_format == "markdown" else text

mlspace_mcp/server.py

@@ -0,0 +1,190 @@
+"""Assemble the FastMCP server: shared async client via lifespan, transport
+security, and registration of the enabled domain umbrella tools."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+
+import httpx
+from mcp.server.fastmcp import FastMCP
+from mcp.server.transport_security import TransportSecuritySettings
+from mcp.types import ToolAnnotations
+
+from .auth import TokenManager
+from .config import Settings
+from .overview import register_overviews
+from .registry import DomainTool, register_domain
+from .tls import tls_context
+from .workspace_context import (
+    WorkspaceClients,
+    environment_id,
+    normalized_endpoint,
+    selected_workspaces,
+)
+
+logger = logging.getLogger(__name__)
+
+# Domain modules, each exposing a module-level ``DOMAIN: DomainTool``.
+DOMAIN_MODULES: tuple[str, ...] = (
+    "inference",
+    "async_inference",
+    "dalle",
+    "jobs",
+    "build_image",
+    "notebooks",
+    "tensorboards",
+    "workspaces",
+    "allocations",
+    "queues",
+    "docker_registry",
+    "data_transfer",
+    "resources",
+)
+
+# Shared rules every task needs. Step-by-step recipes ship as skills in the plugin
+# (plugin/skills/); only these invariants live in the server, so a client sees them
+# once whether or not a skill is active.
+CONTEXT_INSTRUCTIONS = """\
+Context and evidence rules (apply to every step):
+- mlspace_contexts is the fixed selected catalogue, for one, three or ten workspaces.
+  mlspace_workspaces list shows credential-visible workspaces and never expands it.
+  No global default/current workspace exists. A standalone overview (including
+  "now show jobs") covers the selected set even after inspecting an object in A.
+- Address an existing object with its resource_ref, or exact target and resource
+  address. Keep that address for diagnosis/restart; retain the new restart reference.
+  An explicit task in A scopes its steps; an unrelated read in B does not change it.
+  A singleton needs no placement question. Creation uses explicit placement, the
+  established task, the singleton, or a delegated criterion (e.g. any suitable GPU
+  in A/B). Evaluate that criterion and report the choice without asking again.
+  A comparison/example alone does not choose placement. Ask only for real ambiguity;
+  recover a lost object reference or clarify it rather than select the first match.
+- Broad jobs/notebooks reads use mlspace_jobs_overview/mlspace_notebooks_overview.
+  targets omitted means all selected contexts; explicit targets restrict observation.
+  Inspect requested_scope, checked_scope, failures, observed_at, fetch_complete,
+  output_truncated and complete. observed_count counts observations, not a complete
+  inventory total or unique shared capacity. Partial search proves neither absence
+  nor uniqueness. "All" after a shortened list needs a defined set (shown vs matching);
+  an explicitly chosen object remains actionable despite unrelated failed reads.
+- "My jobs" needs a confirmed author mapping or an explicitly supplied author filter.
+  Reuse a user-specified saved author filter if available, label it as such; it is
+  not verified identity/ownership. Membership, LLM login and a shared service account
+  do not identify an author. If no usable mapping/filter exists, explain and obtain
+  an author or offer the workspace overview; never silently label colleagues' jobs mine.
+- Keep paused notebooks in inventory with their status. Configured GPU/instance size
+  is not current usage; do not recommend pausing a paused notebook to free capacity.
+  Compute nodes and Jupyter servers are different objects. Pending demand, running
+  pods and node load are separate observations; do not sum repeated shared-resource
+  views. Missing or inaccessible workload means unknown, not idle or safe to move.
+- Use existing write/confirm rules. Context selection and resource_ref do not add a
+  confirmation requirement or replace a required confirm=true. Read-only diagnosis
+  never submits a paid canary merely to test quota.
+"""
+
+INSTRUCTIONS = (
+    "Tools for the Cloud.ru MLSpace platform (ML training, inference, notebooks, "
+    "data transfer, registry, resources). Each tool is an umbrella over one domain; "
+    "pass an `action` plus that action's parameters. Read the tool description for "
+    "the per-action parameter index. Destructive actions require confirm=true; all "
+    "write actions disappear when the server runs with MLSPACE_READONLY=true."
+) + "\n\n" + CONTEXT_INSTRUCTIONS
+
+
+@dataclass
+class AppContext:
+    """What lives in the FastMCP lifespan context (read by tool handlers)."""
+
+    clients: WorkspaceClients
+
+
+def _load_domain_tools(settings: Settings) -> list[DomainTool]:
+    enabled = settings.enabled_domain_set
+    if enabled is not None:
+        unknown = enabled - set(DOMAIN_MODULES)
+        if unknown:
+            logger.warning(
+                "MLSPACE_ENABLED_DOMAINS lists unknown domain(s) %s; known domains: %s",
+                ", ".join(sorted(unknown)),
+                ", ".join(DOMAIN_MODULES),
+            )
+    tools: list[DomainTool] = []
+    for mod_name in DOMAIN_MODULES:
+        if enabled is not None and mod_name not in enabled:
+            continue
+        full = f"{__package__}.tools.{mod_name}"
+        try:
+            module = importlib.import_module(full)
+        except ModuleNotFoundError as exc:
+            # Not-yet-implemented domain module: skip. A genuine missing dependency
+            # inside an existing module (different name) is re-raised.
+            if exc.name == full:
+                continue
+            raise
+        tools.append(module.DOMAIN)
+    return tools
+
+
+def _transport_security(settings: Settings) -> TransportSecuritySettings:
+    def forms(host: str) -> set[str]:
+        # IPv6 literals appear bracketed in the Host header ([::1], [::1]:8000)
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        return {host, f"{host}:{settings.port}"}
+
+    hosts = forms(settings.host) | forms("127.0.0.1") | forms("localhost")
+    origins = {f"http://{h}" for h in hosts} | {f"https://{h}" for h in hosts}
+    return TransportSecuritySettings(
+        enable_dns_rebinding_protection=True,
+        allowed_hosts=sorted(hosts),
+        allowed_origins=sorted(origins),
+    )
+
+
+def build_server(settings: Settings) -> FastMCP:
+    """Build (but do not run) the FastMCP server. Does not require credentials —
+    credentials are only needed to serve requests."""
+
+    @asynccontextmanager
+    async def lifespan(_server: FastMCP):
+        async with httpx.AsyncClient(timeout=settings.timeout, verify=tls_context(settings.ca_file)) as http:
+            tokens = TokenManager(
+                http,
+                settings.base_url,
+                settings.client_id,
+                settings.client_secret.get_secret_value(),
+            )
+            clients = WorkspaceClients(settings, http, tokens)
+            yield AppContext(clients=clients)
+
+    mcp = FastMCP(
+        "mlspace_mcp",
+        instructions=INSTRUCTIONS,
+        host=settings.host,
+        port=settings.port,
+        lifespan=lifespan,
+        transport_security=_transport_security(settings),
+    )
+
+    for dt in _load_domain_tools(settings):
+        register_domain(mcp, settings, dt)
+
+    catalogue = {
+        "environment": environment_id(settings.base_url),
+        "endpoint": normalized_endpoint(settings.base_url),
+        "workspaces": [{"id": item.id, "name": item.name, "project_name": item.project_name}
+                       for item in selected_workspaces(settings)],
+    }
+
+    async def mlspace_contexts() -> str:
+        return json.dumps(catalogue, ensure_ascii=False)
+
+    mcp.add_tool(mlspace_contexts, description="Saved connected workspaces available as target. "
+                 "This fixed catalogue differs from credential-visible mlspace_workspaces list.",
+                 annotations=ToolAnnotations(readOnlyHint=True, destructiveHint=False,
+                                             idempotentHint=True, openWorldHint=False))
+
+    register_overviews(mcp, settings)
+    return mcp

mlspace_mcp/setup_ui.py

@@ -0,0 +1,103 @@
+"""Bounded terminal setup UI, also usable over SSH without a browser."""
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from typing import Any
+
+from prompt_toolkit import Application, PromptSession
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.layout import HSplit, Layout, Window
+from prompt_toolkit.layout.controls import FormattedTextControl
+from prompt_toolkit.output.defaults import create_output
+from prompt_toolkit.widgets import TextArea
+
+
+def display(value: str) -> str:
+    """API-provided labels must not control the user's terminal."""
+    return ''.join(char if char.isprintable() else ' ' for char in value)
+
+
+def masked_prompt(label: str, **kwargs: Any) -> str:
+    return str(PromptSession(is_password=True,
+                      output=kwargs.pop('output', None) or create_output(stdout=sys.stderr),
+                      **kwargs).prompt(label))
+
+
+@dataclass
+class WorkspaceSelection:
+    rows: list[dict[str, str]]
+    selected: set[str] = field(default_factory=set)
+    query: str = ''
+    cursor: int = 0
+
+    def matches(self) -> list[dict[str, str]]:
+        terms = self.query.casefold().split()
+        return [row for row in self.rows if all(term in ' '.join(
+            row.get(k, '') for k in ('id', 'name', 'project_name')).casefold() for term in terms)]
+
+    def toggle(self) -> None:
+        rows = self.matches()
+        if rows:
+            key = rows[self.cursor % len(rows)]['id']
+            self.selected.symmetric_difference_update({key})
+
+    def lines(self) -> list[str]:
+        rows = self.matches()
+        if not rows:
+            return ['No matches. Change the search; previous selections are retained.']
+        self.cursor %= len(rows)
+        start = self.cursor // 12 * 12
+        return [display(f"{'>' if start+i == self.cursor else ' '} "
+                        f"[{'x' if r['id'] in self.selected else ' '}] {r['name']} "
+                        f"| {r.get('project_name', '')} | {r['id']}")
+                for i, r in enumerate(rows[start:start+12])]
+
+
+def choose_workspaces(rows: list[dict[str, str]], selected_ids: list[str] | None = None,
+                      **kwargs: Any) -> list[dict[str, str]]:
+    state = WorkspaceSelection(rows, set(selected_ids or []) & {r['id'] for r in rows})
+    search = TextArea(height=1, prompt='Search name / project / ID: ', multiline=False)
+    bindings = KeyBindings()
+
+    def changed(_buffer: Any) -> None:
+        state.query = search.text
+        state.cursor = 0
+
+    search.buffer.on_text_changed += changed
+
+    @bindings.add('up')
+    def up(event: Any) -> None:
+        state.cursor = (state.cursor - 1) % max(1, len(state.matches()))
+
+    @bindings.add('down')
+    def down(event: Any) -> None:
+        state.cursor = (state.cursor + 1) % max(1, len(state.matches()))
+
+    @bindings.add(' ')
+    def toggle(event: Any) -> None:
+        state.toggle()
+
+    @bindings.add('enter')
+    def done(event: Any) -> None:
+        if state.selected:
+            event.app.exit(result=[row for row in rows if row['id'] in state.selected])
+
+    @bindings.add('c-c')
+    @bindings.add('c-d')
+    def cancel(event: Any) -> None:
+        event.app.exit(exception=KeyboardInterrupt())
+
+    app: Application[list[dict[str, str]]] = Application(
+        layout=Layout(HSplit([
+            search,
+            Window(FormattedTextControl(lambda: '\n'.join(state.lines())), height=12),
+            Window(FormattedTextControl(lambda: (
+                f'{len(state.matches())} matches / {len(rows)} total; {len(state.selected)} selected. '
+                'Arrows: move | Space: select | Ctrl-U: clear search | Enter: finish | Ctrl-C: cancel'
+            )), height=2),
+        ]), focused_element=search),
+        key_bindings=bindings, full_screen=False, erase_when_done=True,
+        output=kwargs.pop('output', None) or create_output(stdout=sys.stderr), **kwargs,
+    )
+    return app.run()

mlspace_mcp/skills/mlspace-call-deployed-model-predict/SKILL.md

@@ -0,0 +1,36 @@
+---
+name: mlspace-call-deployed-model-predict
+description: "Call a deployed model (sync predict) on Cloud.ru MLSpace. Run a live prediction against a deployed inference service. Uses the mlspace_contexts, mlspace_inference tools."
+license: MIT
+compatibility: Requires the `mlspace` MCP server from this plugin (Cloud.ru MLSpace credentials needed).
+---
+
+# Call a deployed model (sync predict)
+
+Run a live prediction against a deployed inference service.
+
+This skill drives the `mlspace` MCP server shipped in the same plugin. If its
+tools are not connected, say so instead of guessing — none of these steps can be
+carried out any other way.
+
+## Arguments
+
+- `service_name` **(required)** — Deployed inference service name.
+- `input_payload` **(required)** — JSON input forwarded to the model.
+- `model_name` — Model name within the service (default = service).
+- `region` — Service region (default DGX2-INF-001).
+- `target` — Connected workspace ID or unambiguous name, if scoped.
+
+## Procedure
+
+Goal: run a synchronous prediction against a deployed inference service.
+NOTE: inference.predict is a side effect — it runs the deployed (billed) model.
+1. Resolve the service target from its address/task or mlspace_contexts selected catalogue.
+2. mlspace_inference list -> find the service whose manifest name matches service_name.
+3. Resolve model_name (often == service_name) and region (arg, else default
+   DGX2-INF-001). mlspace_inference get (inference_name, region) to confirm it's Ready.
+4. Echo back service/model/region and that this is a billed call; then
+   mlspace_inference predict (inference_name=service_name, model_name, body=input_payload
+   passed through unchanged).
+Output: the model's raw prediction JSON + a one-line note of what was called.
+Caveat: the predict response shape is model-defined (no fixed schema).

mlspace_mcp/skills/mlspace-deploy-inference-service/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: mlspace-deploy-inference-service
+description: "Deploy a model image as an inference service on Cloud.ru MLSpace. Deploy an already-built registry image as a new inference service. Write action. Uses the mlspace_docker_registry, mlspace_inference, mlspace_resources tools."
+license: MIT
+compatibility: Requires the `mlspace` MCP server from this plugin (Cloud.ru MLSpace credentials needed).
+---
+
+# Deploy a model image as an inference service
+
+Deploy an already-built registry image as a new inference service. Write action.
+
+This skill drives the `mlspace` MCP server shipped in the same plugin. If its
+tools are not connected, say so instead of guessing — none of these steps can be
+carried out any other way.
+
+## Arguments
+
+- `image` **(required)** — Registry image ref to deploy.
+- `region` — Target region (default CCE-INF).
+- `instance_type` — Inference instance type.
+- `replicas_min` — Min replicas.
+- `replicas_max` — Max replicas.
+- `alias` — Service alias/name.
+- `description` — Service description.
+- `target` — Connected workspace ID or unambiguous name, if scoped.
+
+## Procedure
+
+Goal: deploy an already-built registry image as an inference service (write
+action; refused under MLSPACE_READONLY=true).
+1. INTENT GATE: if the user actually wants to BUILD an image, STOP — image build is not
+   possible via this API (build-image is multipart-only; no create-repo). Hand off to
+   the console/CLI for the build, then come back to deploy the resulting image ref.
+2. (best-effort) verify the image: docker_registry current_registry -> list_repos /
+   tags to confirm the ref exists.
+3. mlspace_resources configs cluster_type=INF -> pick a valid inference instance_type.
+4. Build CreateInferenceServiceRequestV2 {image, region (default CCE-INF), instance_type,
+   replicas:{min,max}, optional alias/description}. Echo it.
+5. mlspace_inference create (body=...). Capture the service name/status.
+6. mlspace_inference get (inference_name, region) to confirm provisioning.
+Output: the created service alias, image, region, instance_type, replicas, and status.
+Caveat: you can deploy an existing image but cannot build one here.

mlspace_mcp/skills/mlspace-diagnose-data-transfer/SKILL.md

@@ -0,0 +1,41 @@
+---
+name: mlspace-diagnose-data-transfer
+description: "Diagnose a failed / partial data transfer on Cloud.ru MLSpace. Did my data transfer actually fail / land incompletely, and why. Uses the mlspace_data_transfer tools."
+license: MIT
+compatibility: Requires the `mlspace` MCP server from this plugin (Cloud.ru MLSpace credentials needed).
+---
+
+# Diagnose a failed / partial data transfer
+
+Did my data transfer actually fail / land incompletely, and why.
+
+This skill drives the `mlspace` MCP server shipped in the same plugin. If its
+tools are not connected, say so instead of guessing — none of these steps can be
+carried out any other way.
+
+## Arguments
+
+- `transfer_id` — Transfer id (or give a name).
+- `transfer_name` — Transfer name to look up.
+- `expected_rows` — Rows you expected, to judge completeness.
+- `act` — If true, may rerun a failed run (after confirm).
+- `target` — Connected workspace ID or unambiguous name, if scoped.
+
+## Procedure
+
+Goal: determine whether a data transfer truly failed / was partial, and why.
+1. Resolve the transfer: if transfer_id given use it, else data_transfer list_transfers
+   and match transfer_name.
+2. data_transfer list_history (transfer_id, page=1, page_size=20 — this endpoint uses
+   page/page_size). Take the latest run's history_id + metrics (size_bytes, rows_count,
+   progress, status).
+3. data_transfer get_history_status (ids=[history_id]) for the authoritative status
+   (in_progress|success|failed|cancelled).
+4. If failed/cancelled, data_transfer get_event_logs (history_id, offset=0, limit=50,
+   as_enum_resolved=true) for the decoded reason.
+5. Compare rows_count/size_bytes vs the user's expectation to judge "partial".
+Output: status + hard numbers (rows/bytes/progress) + completeness verdict + decoded
+event reasons; separate "genuinely failed" from "clean exit / fewer rows than expected".
+Only if act=true: rerun_history (body={history_id}) for a failed run, after confirming.
+Caveat: status is only a 4-value enum; "finished but data looks wrong" is inferred from
+the numbers, not a dedicated field.

mlspace_mcp/skills/mlspace-diagnose-failed-training-job/SKILL.md

@@ -0,0 +1,64 @@
+---
+name: mlspace-diagnose-failed-training-job
+description: "Diagnose a failed / stuck training job on Cloud.ru MLSpace. Why did my training job fail or why is it stuck Pending; or roll up recent failures. Uses the mlspace_jobs, mlspace_jobs_overview tools."
+license: MIT
+compatibility: Requires the `mlspace` MCP server from this plugin (Cloud.ru MLSpace credentials needed).
+---
+
+# Diagnose a failed / stuck training job
+
+Why did my training job fail or why is it stuck Pending; or roll up recent failures.
+
+This skill drives the `mlspace` MCP server shipped in the same plugin. If its
+tools are not connected, say so instead of guessing — none of these steps can be
+carried out any other way.
+
+## Arguments
+
+- `job_name` — Job to diagnose (omit for a recent-failures roll-up).
+- `region` — Region of the job, if known.
+- `days` — Look-back window in days for the roll-up.
+- `job_author` — Filter the roll-up by author email.
+- `queue_id` — Filter the roll-up by queue id.
+- `target` — Connected workspace ID or unambiguous name, if scoped.
+
+## Procedure
+
+Goal: explain why a training job FAILED or is STUCK PENDING, and/or roll up recent failures.
+Use mlspace_jobs and relevant read-only resource/queue helpers; never run/restart/delete.
+
+If a job name is given:
+1. jobs get (job_name only — no region arg). Read status (lowercase) + timeline
+   (created_at/pending_at/running_at/completed_at). NOTE error_code/error_message
+   here are deprecated constants — ignore them.
+2. jobs get_params (job_name) and record the authoritative script, region, image and
+   instance_type. `/home/jovyan` is region-scoped; the same absolute path in another
+   notebook region is a different NFS volume.
+3. Classify from the timeline BEFORE logs: running_at>0 & failed -> runtime crash
+   (expect traceback); pending with running_at=0 -> never scheduled (capacity/quota);
+   terminated -> likely preempted.
+4. jobs logs (job_name, region from get_params, tail=100, verbose=true; raise tail
+   to ~300 if a traceback is cut off). This is the actual error.
+   If the job failed before the first user-script line and the launcher cannot access
+   the entrypoint: for type=binary check that the command/executable exists in the
+   selected image and that its arguments are valid; for an NFS script, treat a wrong
+   regional NFS as the leading cause even when the path begins with /home/jovyan.
+5. jobs list_pods (job_name) -> per-pod status + `reason` (OOMKilled / Unschedulable /
+   ImagePull...). This is the key "why" for both crash and pending.
+6. If pending/terminated: jobs get_preemptors (job_name) -> who preempted; and check
+   capacity (see the discover_launchable_resources / capacity playbook): instance
+   availability + node load in the job's region.
+
+For a recent-failures roll-up: mlspace_jobs_overview with status=["Failed","Terminated"]
+and author=job_author when explicitly known, plus targets for an explicitly restricted
+workspace scope. Regions are discovered dynamically and pages are collected by the
+helper. Filter returned data by the requested days/queue/region, disclosing incomplete
+coverage. If exact server-side date/queue/region filtering is needed, use jobs list
+in each selected context and discovered MT region with start_date/end_date (RFC3339 Z),
+queue_id and pagination; never treat one page as the full set. Group observed failures
+by instance_type/author/queue, retaining each source/resource_ref.
+
+Output: a one-line verdict (status + inferred cause), the phase timeline, the key
+log excerpt, pod reason, and a concrete next step.
+Caveats: no per-pod logs beyond the job log stream; capacity is inferred from node
+load + preemptors (no quota/cost numbers exist in this API).