mlspace-plugin 0.2.1__py3-none-any.whl

mlspace_mcp/init_cli.py

@@ -0,0 +1,356 @@
+"""Terminal onboarding shared by humans and explicitly configured automation."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+
+from .client_setup import (
+    CLIENTS,
+    atomic_write,
+    detect_clients,
+    install_clients,
+    prepare_clients,
+    server_command,
+)
+from .config import DEFAULT_BASE_URL, LITERAL_DOTENV_MARKER, read_dotenv_file, user_config_file
+from .setup_ui import choose_workspaces, display, masked_prompt
+from .tls import safe_error as _safe_error
+from .tls import tls_context
+
+HEADER = f'# MLSpace credentials, written by mlspace-plugin setup.\n{LITERAL_DOTENV_MARKER}\n'
+
+
+def _ask(label: str, secret: bool = True, hidden: bool = True) -> str:
+    while True:
+        value = masked_prompt(f'  {label}: ') if hidden else input(f'  {label}: ')
+        if re.match(r'^\s*(?:export\s+)?MLSPACE_\w+\s*=', value):
+            print('Enter only the value, without MLSPACE_...= or export.', file=sys.stderr)
+        elif value.strip():
+            return value if secret else value.strip()
+        else:
+            print('Required. Paste the value; asterisks show that it was entered.', file=sys.stderr)
+
+
+def _resolve_base_url(target: Path, override: str | None) -> str:
+    saved = read_dotenv_file(target) if target.is_file() else {}
+    return ((override or '').strip() or os.environ.get('MLSPACE_BASE_URL', '').strip()
+            or str(saved.get('MLSPACE_BASE_URL') or DEFAULT_BASE_URL)).rstrip('/')
+
+
+def _fetch_workspaces(key_id: str, key_secret: str, *, base_url: str,
+                      ca_file: str = '') -> list[dict[str, str]]:
+    from .auth import TokenManager
+    from .client import MLSpaceClient
+
+    async def fetch() -> Any:
+        async with httpx.AsyncClient(timeout=30, verify=tls_context(ca_file)) as http:
+            tokens = TokenManager(http, base_url=base_url, client_id=key_id, client_secret=key_secret)
+            client = MLSpaceClient(http, tokens, base_url=base_url, api_key='', workspace_id='')
+            return await client.request('GET', '/public/v2/workspaces/v3/')
+
+    response = asyncio.run(fetch())
+    spaces = response.get('workspaces') if isinstance(response, dict) else None
+    if not isinstance(spaces, list) or not spaces:
+        raise ValueError('No workspaces visible to these access keys.')
+    catalogue = []
+    seen = set()
+    for row in spaces:
+        if not isinstance(row, dict) or not (key := row.get('id') or row.get('workspace_id')):
+            raise ValueError('Workspace API returned an entry without an ID.')
+        if str(key) in seen:
+            raise ValueError('Workspace API returned duplicate IDs.')
+        seen.add(str(key))
+        catalogue.append({'id': str(key), 'name': str(row.get('name') or key),
+                          'project_name': str(row.get('project_name') or '')})
+    return sorted(catalogue, key=lambda r: (r['name'].casefold(), r['project_name'], r['id']))
+def _write(path: Path, values: dict[str, str]) -> None:
+    """Atomically replace ``path`` with a 0600 dotenv file."""
+    if path.is_symlink():
+        raise ValueError("Credentials target must not be a symlink.")
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if path.parent == user_config_file().parent:
+        path.parent.chmod(0o700)
+    fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+    temporary = Path(temporary_name)
+    try:
+        os.fchmod(fd, stat.S_IRUSR | stat.S_IWUSR)
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            body = "".join(
+                f"MLSPACE_{name}={json.dumps(value, ensure_ascii=False)}\n"
+                for name, value in values.items()
+            )
+            fh.write(HEADER + body)
+            fh.flush()
+            os.fsync(fh.fileno())
+        os.replace(temporary, path)
+    except BaseException:
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def _verify(values: dict[str, str], *, base_url: str) -> str | None:
+    """Check selected workspace details and IDs. Returns a safe error, or None.
+
+    The caller preserves the previous file if any selected workspace fails.
+    """
+    import asyncio
+
+    import httpx
+    from pydantic import SecretStr
+
+    from .auth import TokenManager
+    from .config import Settings
+    from .workspace_context import WorkspaceClients
+
+    async def run() -> str | None:
+        try:
+            # Explicit values keep old environment selectors out of this check.
+            settings = Settings(  # type: ignore[call-arg]
+                _env_file=None,
+                base_url=base_url,
+                client_id=values["CLIENT_ID"],
+                client_secret=SecretStr(values["CLIENT_SECRET"]),
+                api_key=SecretStr(values.get("API_KEY", "")),
+                workspace_id=values.get("WORKSPACE_ID", ""),
+                workspaces=json.loads(values.get("WORKSPACES", "[]")),
+                namespace="",
+            )
+            async with httpx.AsyncClient(timeout=30, verify=tls_context(values.get("CA_FILE", ""))) as http:
+                tokens = TokenManager(
+                    http, base_url=base_url, client_id=values["CLIENT_ID"],
+                    client_secret=values["CLIENT_SECRET"],
+                )
+                clients = WorkspaceClients(settings, http, tokens)
+                verified = 0
+                errors = []
+                for selection in clients.selected:
+                    try:
+                        client = await clients.resolve(selection.id)
+                        data = await client.request(
+                            "GET", "/public/v2/workspaces/v3/{workspace_id}",
+                            path_params={"workspace_id": selection.id},
+                        )
+                        if not isinstance(data, dict) or data.get("id") != selection.id:
+                            errors.append(f"{selection.id}: workspace identity was not confirmed")
+                            continue
+                        verified += 1
+                    except Exception as exc:  # noqa: BLE001 — verify every selected workspace
+                        errors.append(f"{selection.id}: {_safe_error(exc)}")
+                print(
+                    f"  checked: {verified}/{len(clients.selected)} selected workspace(s) verified "
+                    "(workspace details and ID)",
+                    file=sys.stderr,
+                )
+                return "; ".join(errors) or None
+        except Exception as exc:  # noqa: BLE001 — report failures without exposing transport details
+            return _safe_error(exc)
+
+    return asyncio.run(run())
+
+
+
+def _clients(requested: list[str], paths: list[str], non_interactive: bool) -> dict[str, str]:
+    detected = detect_clients()
+    for item in paths:
+        name, separator, path = item.partition('=')
+        executable = Path(path).expanduser().absolute()
+        if not separator or name not in CLIENTS or not executable.is_file() or not os.access(executable, os.X_OK):
+            raise ValueError('Use --client-path CLIENT=/absolute/path/to/executable.')
+        detected[name] = str(executable)
+    if requested:
+        if any(name not in detected for name in requested):
+            raise ValueError('Requested client not found. Install it or use --client-path CLIENT=/path.')
+        return {name: detected[name] for name in requested}
+    if len(detected) == 1:
+        print('Client detected: ' + next(iter(detected)), file=sys.stderr)
+        return detected
+    if not detected:
+        if non_interactive:
+            raise ValueError('No coding client found; use --client-path or --config-only.')
+        print('No Claude Code, Codex or OpenCode found. Enter CLIENT=/absolute/path, '
+              'or press Enter to stop and install a client.', file=sys.stderr)
+        custom = input('Client executable: ').strip()
+        if custom:
+            return _clients([], [custom], True)
+        raise ValueError('Install Claude Code, Codex or OpenCode, then run setup again. '
+                         'For other MCP clients use --config-only.')
+    if non_interactive:
+        raise ValueError('Multiple clients found; select with --client (repeat for several).')
+    print('Choose clients: type to search, Space to select, Enter to continue.', file=sys.stderr)
+    selected = choose_workspaces([{'id': name, 'name': name, 'project_name': path}
+                                  for name, path in detected.items()])
+    return {row['id']: detected[row['id']] for row in selected}
+
+
+def run_init(path: Path | None = None, *, force: bool = False, verify: bool = True,
+             base_url: str | None = None, config_only: bool = False,
+             non_interactive: bool = False, from_env: bool = False,
+             credentials_file: Path | None = None, workspace_ids: list[str] | None = None,
+             clients: list[str] | None = None, client_paths: list[str] | None = None,
+             ca_file: Path | None = None, replace_credentials: bool = False,
+             list_workspaces: bool = False, search: str = '', offset: int = 0, limit: int = 20) -> int:
+    try:
+        return _run_init(path, force=force, verify=verify, base_url=base_url,
+                         config_only=config_only, non_interactive=non_interactive,
+                         from_env=from_env, credentials_file=credentials_file,
+                         workspace_ids=workspace_ids or [], clients=clients or [],
+                         client_paths=client_paths or [], ca_file=ca_file,
+                         replace_credentials=replace_credentials, list_workspaces=list_workspaces,
+                         search=search, offset=offset, limit=limit)
+    except KeyboardInterrupt:
+        print('\nSetup cancelled; previous credentials preserved unless saving already completed.', file=sys.stderr)
+        return 130
+    except EOFError:
+        print('\nNo interactive input. Run `mlspace-plugin setup` yourself in a terminal (SSH works). '
+              'Agents: use INSTALL_AGENT.md; never ask for keys in chat.', file=sys.stderr)
+        return 1
+    except (ValueError, OSError, httpx.HTTPError) as exc:
+        # Only our own ValueErrors carry displayable text. Third-party errors may echo keys.
+        message = str(exc) if type(exc) is ValueError else _safe_error(exc)
+        print('Setup failed: ' + display(message), file=sys.stderr)
+        return 1
+
+
+def _run_init(path: Path | None, **options: Any) -> int:
+    target = (path or user_config_file()).expanduser().absolute()
+    listing = options['list_workspaces']
+    non_interactive = options['non_interactive']
+    if not non_interactive and not sys.stdin.isatty():
+        raise EOFError
+    if options['from_env'] and options['credentials_file']:
+        raise ValueError('Choose either --from-env or --credentials-file.')
+    if target.exists() and not options['force'] and not listing:
+        raise ValueError('Configuration already exists. Use --force to reconfigure; saved keys will be reused.')
+    if not options['verify'] and (not options['config_only'] or not options['workspace_ids']):
+        raise ValueError('--no-verify requires --config-only and explicit --workspace IDs; result is unverified.')
+    if options['offset'] < 0 or not 1 <= options['limit'] <= 100:
+        raise ValueError('Use offset >= 0 and limit between 1 and 100.')
+    saved = read_dotenv_file(target) if target.is_file() else {}
+    endpoint = _resolve_base_url(target, options['base_url'])
+    parsed = urlsplit(endpoint)
+    if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ValueError('API endpoint must be an HTTPS URL without credentials, query or fragment.')
+    if os.environ.get('SSL_CERT_DIR'):
+        raise ValueError('SSL_CERT_DIR cannot be carried into client startup. Unset it for setup and use '
+                         '--ca-file /path/to/trusted-ca.pem or install the CA in the system trust store.')
+    selected_clients = {} if options['config_only'] or listing else _clients(
+        options['clients'], options['client_paths'], non_interactive)
+    command = server_command() if selected_clients else []
+    if selected_clients:
+        prepare_clients(selected_clients, target, command)
+    if not listing:
+        print('[1/4] Access keys. Paste values only; asterisks hide both fields. '
+              'Keys stay in your user config, not in chat.', file=sys.stderr)
+    if options['credentials_file']:
+        source = read_dotenv_file(options['credentials_file'].expanduser())
+    elif options['from_env']:
+        source = dict(os.environ)
+    elif not options['replace_credentials'] and str(saved.get('MLSPACE_BASE_URL') or DEFAULT_BASE_URL).rstrip('/') == endpoint:
+        source = saved
+    else:
+        source = {}
+    keys = {}
+    for name, label in [('CLIENT_ID', 'Cloud.ru Key ID'), ('CLIENT_SECRET', 'Cloud.ru Key Secret')]:
+        value = source.get('MLSPACE_' + name)
+        if not value or not value.strip():
+            if non_interactive or options['credentials_file'] or options['from_env']:
+                raise ValueError('Missing credentials for this endpoint. Run setup in your terminal, '
+                                 'or explicitly select --from-env / --credentials-file.')
+            value = _ask(label, secret=name == 'CLIENT_SECRET')
+        keys[name] = value
+    ca = str(options['ca_file'] or saved.get('MLSPACE_CA_FILE') or os.environ.get('SSL_CERT_FILE') or '')
+    ca = str(Path(ca).expanduser().absolute()) if ca else ''
+    if not listing:
+        print('[2/4] Checking TLS and loading workspaces…', file=sys.stderr)
+    if options['verify'] or listing:
+        while True:
+            try:
+                rows = _fetch_workspaces(keys['CLIENT_ID'], keys['CLIENT_SECRET'], base_url=endpoint, ca_file=ca)
+                break
+            except ValueError:
+                raise
+            except Exception as exc:
+                error = _safe_error(exc)
+                if not non_interactive and 'TLS certificate' in error:
+                    print(error, file=sys.stderr)
+                    ca = input('Trusted CA PEM path from your administrator (Enter to cancel): ').strip()
+                    if not ca:
+                        raise ValueError('TLS setup stopped; no credentials saved.') from None
+                    ca = str(Path(ca).expanduser().absolute())
+                    continue
+                raise ValueError(error) from None
+    else:
+        rows = [{'id': key, 'name': key, 'project_name': ''} for key in dict.fromkeys(options['workspace_ids'])]
+    if listing:
+        terms = options['search'].casefold().split()
+        matches = [row for row in rows if all(term in ' '.join(row.values()).casefold() for term in terms)]
+        offset, limit = options['offset'], options['limit']
+        print(json.dumps({'total': len(matches), 'offset': offset, 'limit': limit,
+                          'has_more': offset + limit < len(matches), 'workspaces': matches[offset:offset+limit]}, ensure_ascii=False))
+        return 0
+    requested = options['workspace_ids']
+    if requested:
+        if set(requested) - {row['id'] for row in rows}:
+            raise ValueError('Requested workspace IDs are not in the accessible workspace list.')
+        selected = [row for row in rows if row['id'] in requested]
+    elif non_interactive:
+        raise ValueError('Select workspace IDs with --workspace. Use --list-workspaces --search to find them.')
+    elif len(rows) == 1:
+        selected = rows
+    else:
+        print('Find your workspaces: type a name, project or ID. Space selects; Enter finishes.', file=sys.stderr)
+        old = json.loads(str(saved.get('MLSPACE_WORKSPACES') or '[]'))
+        selected = choose_workspaces(rows, [row['id'] for row in old if isinstance(row, dict) and 'id' in row])
+    values = {key.removeprefix('MLSPACE_'): value for key, value in saved.items()
+              if key.startswith('MLSPACE_') and value is not None
+              and key not in {'MLSPACE_API_KEY', 'MLSPACE_WORKSPACE_ID', 'MLSPACE_WORKSPACES', 'MLSPACE_NAMESPACE'}}
+    values.update(keys)
+    values.update(BASE_URL=endpoint, CA_FILE=ca,
+                  WORKSPACES=json.dumps(selected, ensure_ascii=False, separators=(',', ':')))
+    if len(selected) == 1:
+        values['WORKSPACE_ID'] = selected[0]['id']
+    print(f'[3/4] Verifying {len(selected)} selected workspace(s)…', file=sys.stderr)
+    if options['verify']:
+        problem = _verify(values, base_url=endpoint)
+        if problem:
+            raise ValueError(problem)
+    if ca:
+        # Persist the validated public CA independently of /tmp and shell environment.
+        content = Path(ca).read_text(encoding='ascii')
+        destination = target.parent / ('ca-' + hashlib.sha256(content.encode()).hexdigest()[:16] + '.pem')
+        tls_context(ca)
+        atomic_write(destination, content)
+        values['CA_FILE'] = str(destination)
+    installed = []
+    if selected_clients:
+        print('[4/4] Connecting MCP and installing skills…', file=sys.stderr)
+        installed = install_clients(selected_clients, target, command)
+    _write(target, values)
+    print(f'Credentials saved: {target} (0600).', file=sys.stderr)
+    if not options['verify']:
+        print('UNVERIFIED configuration saved by explicit request. No client was connected.', file=sys.stderr)
+        return 0
+    if selected_clients:
+        for client in installed:
+            print(f'  {client}: MCP registered and skills installed.', file=sys.stderr)
+        print('Done. Open a new client session (restart it if MCP is not visible). '
+              'The client starts the MCP server automatically.\n'
+              'Try: «Покажи выбранные воркспейсы и запущенные задачи, ничего не меняй».', file=sys.stderr)
+    else:
+        print('API/TLS verified. Config-only setup complete; client integration was not requested.', file=sys.stderr)
+    return 0

mlspace_mcp/job_hooks.py

@@ -0,0 +1,242 @@
+"""Job-specific preflight, deletion guard and response compatibility hooks.
+
+Leaf module: imports no dispatcher or domain declarations.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ._status import JOB_STATUS_CANON
+from .errors import MLSpaceError
+
+# Earlier API observations motivated these compatibility guards. They are not
+# current-tenant evidence. Binary commands are documented; non-binary entrypoints
+# retain regional NFS path checks. The API cannot stat a file or verify an image command.
+NFS_SCRIPT_PREFIX = "/home/jovyan"
+
+
+def _preflight_job_region(values: dict[str, Any]) -> None:
+    """Refuse a job `run` that leaves `region` to the API default.
+
+    Runs UNCONDITIONALLY — the region trap does not depend on how `script` looks,
+    so this must not sit inside the path branch (a missing or empty script would
+    otherwise skip it and let the default region through).
+
+    A top-level ``region=`` is accepted as well and copied into the body: `run`
+    carries no top-level region parameter, so it would otherwise be dropped
+    silently and the caller refused for something they did supply.
+    """
+    body = values.get("body")
+    in_body = body.get("region") if isinstance(body, dict) else None
+    if isinstance(in_body, str) and in_body.strip():
+        return
+    top = values.get("region")
+    if isinstance(top, str) and top.strip():
+        if isinstance(body, dict):
+            body["region"] = top
+        return
+    raise MLSpaceError(
+        "`region` must be explicit for a job submission. The API default (DGX2-MT) "
+        "can select a different /home/jovyan volume and fail only after allocating "
+        "a node.",
+        hint=(
+            "Set body.region. For an NFS entrypoint, verify it from a notebook or "
+            "transfer attached to that same region because the Public API cannot stat "
+            "the target region's NFS; for type=binary, verify that the command or "
+            "executable is available in the selected image."
+        ),
+    )
+
+
+def _preflight_job_script(values: dict[str, Any]) -> None:
+    """Apply only the script checks supported for the selected job type.
+
+    Empty strings are unsafe for every type. ``binary`` accepts a command or executable
+    path, so the wrapper must preserve a non-empty value without pretending it can
+    resolve the command in the selected image. Other types retain the established NFS
+    path-shape checks. Existence is never claimed: the Public API has no remote stat.
+    """
+    body = values.get("body")
+    script = body.get("script") if isinstance(body, dict) else values.get("script")
+    job_type = body.get("type") if isinstance(body, dict) else values.get("type")
+    if script is None:
+        return  # absent/None: the API's own 422 is exemplary, don't duplicate it
+    if not isinstance(script, str):
+        return  # the spec-driven body lint reports the type error in the real handler
+    if not script.strip():
+        raise MLSpaceError(
+            f"`script` is empty ({script!r}). The API would accept this and return a "
+            "job_name, then allocate a node and fail within seconds with an empty "
+            "error_message.",
+            hint=(
+                "Provide a non-empty entrypoint. For type=binary this may be a command "
+                "or executable path; for other job types use the documented NFS script "
+                f"path under {NFS_SCRIPT_PREFIX}/…."
+            ),
+        )
+    if job_type == "binary":
+        return
+    if any(seg == ".." for seg in script.split("/")):
+        # a '..' segment escapes the NFS home the prefix check exists to enforce:
+        # "/home/jovyan/../tmp/x.sh" resolves to /tmp/x.sh
+        problem = f"`script` contains a '..' path segment ({script!r})"
+    elif script == NFS_SCRIPT_PREFIX or script.endswith("/"):
+        problem = f"`script` points at a directory, not a file ({script!r})"
+    elif not script.startswith("/"):
+        problem = f"`script` is a relative path ({script!r})"
+    elif not script.startswith(NFS_SCRIPT_PREFIX + "/"):
+        problem = f"`script` points outside the region's NFS ({script!r})"
+    else:
+        return
+    raise MLSpaceError(
+        f"{problem}. The API would accept this and return a job_name, then the job "
+        f"would allocate a node and fail within seconds with an empty error_message.",
+        hint=(
+            f"The script must already exist on the chosen region's NFS home, under "
+            f"{NFS_SCRIPT_PREFIX}/… — upload it there first, then pass the absolute "
+            f"path to the file (e.g. {NFS_SCRIPT_PREFIX}/myproject/train.sh). The same "
+            "path in a different notebook region is a different volume."
+        ),
+    )
+
+
+def preflight_job_run(values: dict[str, Any]) -> None:
+    """Composed preflight for ``POST /jobs`` (one dict key, two independent concerns).
+
+    Region first — it applies to every submission; then the script shape.
+    """
+    _preflight_job_region(values)
+    _preflight_job_script(values)
+
+
+def _preflight_jobs_list_region(values: dict[str, Any]) -> None:
+    """Refuse a jobs `list` with no region and explain the silent count=0.
+
+    Live finding: ``GET /jobs`` without ``region`` returns 200 ``{"count":0,
+    "jobs":[]}`` for EVERY region, while a scoped ``GET /jobs?region=<r>`` returns the
+    real, non-zero count — so an empty result here proves the scope is missing, not
+    that the workspace has no jobs. (The exact per-region count is a fast-moving live
+    datum, so it is deliberately NOT quoted here.) A deliberate refusal (like the
+    /home/jovyan precondition) beats leaking a false emptiness to the model.
+    """
+    if not values.get("region"):
+        raise MLSpaceError(
+            'GET /jobs without `region` returns 200 {"count":0,"jobs":[]} for every '
+            "region; with a region it returns the real (non-zero) count — an empty "
+            "result here does NOT prove there are no jobs, only that the region scope "
+            "is missing.",
+            hint="Pass region=<e.g. DGX2-MT|A100-MT|SR003|SR005|SR006|SR008> (examples, "
+            "not a closed set). For existing jobs, use mlspace_jobs_overview, which "
+            "combines regions from configs and workspace allocations; configs alone "
+            "can omit allocated regions. Jobs live per region.",
+        )
+
+
+def _preflight_jobs_list_status(values: dict[str, Any]) -> None:
+    """Normalise a jobs `list` status filter to the enum casing the API accepts.
+
+    The list endpoint filters on the Capitalized ``JobStatus-Input`` enum, but a
+    model naturally passes get's lowercase status (e.g. 'failed'). This is a pure
+    translation to the value the API itself requires (precedent: the silent limit
+    clamp in the handler), NOT invention: unknown values are left untouched so the
+    API returns its own exemplary 422.
+    """
+    raw = values.get("status")
+    if raw is None:
+        return
+    if isinstance(raw, (list, tuple)):
+        values["status"] = [JOB_STATUS_CANON.get(str(s).casefold(), s) for s in raw]
+    else:
+        values["status"] = JOB_STATUS_CANON.get(str(raw).casefold(), raw)
+
+
+def preflight_jobs_list(values: dict[str, Any]) -> None:
+    """Composed preflight for ``GET /jobs`` (one dict key, two concerns).
+
+    Region enforcement first (may refuse), then status-casing normalisation — the
+    two live as separate, independently-testable helpers but share a single
+    PREFLIGHT registration to avoid a key collision.
+    """
+    _preflight_jobs_list_region(values)
+    _preflight_jobs_list_status(values)
+
+
+def _precheck_unconfirmed(job_name: Any, reason: str) -> MLSpaceError:
+    """The honest fail-CLOSED refusal shared by every 'could not confirm existence'
+    branch of the delete guard."""
+    return MLSpaceError(
+        f"Could not confirm that job {job_name} exists before deleting it ({reason}). "
+        "The delete was NOT sent. MLSpace replies status=\"deleted\" with a fresh "
+        "deleted_at even for a name that does not exist, so issuing the DELETE without a "
+        "confirmed precheck could report a FALSE 'deleted'.",
+        hint="Retry, or confirm the exact job_name via `list` (with region) first, then "
+        "re-run the delete.",
+    )
+
+
+async def guard_job_delete(client: Any, values: dict[str, Any]) -> None:
+    """Refuse a jobs delete unless a status precheck CONFIRMS the job exists.
+
+    Live finding: the delete endpoint replies 200 ``status="deleted"`` with a fresh
+    ``deleted_at`` even for a name that never existed — a FALSE confirmation. One
+    cheap status GET before the destructive call is the gate. This guard is
+    FAIL-CLOSED (a safety validator, not a cosmetic transform): the DELETE proceeds
+    ONLY when the precheck returns a concrete live status. A positive
+    'job not found', a precheck error (timeout/403/500), or an unexpected shape all
+    refuse — because none of them proves the job is really there to delete.
+
+    Edge case (05#11): a job created <~2s ago also answers 'job not found' and is
+    blocked here — fail-safe (better to refuse than to falsely confirm), covered by
+    the retry-after-propagation hint.
+    """
+    job_name = values.get("job_name")
+    try:
+        body = await client.request(
+            "GET", "/public/v2/jobs/{job_name}", path_params={"job_name": str(job_name)}
+        )
+    except MLSpaceError as exc:
+        # precheck could not reach a verdict (4xx/5xx/timeout/odd body already mapped
+        # to MLSpaceError) → fail-closed rather than let a blind DELETE claim success.
+        raise _precheck_unconfirmed(job_name, f"precheck failed: {exc.message}") from exc
+    except Exception as exc:  # defensive: any non-mapped failure is still unconfirmed
+        raise _precheck_unconfirmed(job_name, f"precheck error: {type(exc).__name__}") from exc
+
+    raw_status = body.get("status") if isinstance(body, dict) else None
+    if not isinstance(raw_status, str):
+        # null / object / missing: str() would turn these into a truthy "none"/"{...}"
+        raise _precheck_unconfirmed(job_name, "precheck returned an unexpected shape")
+    status = raw_status.strip().casefold()
+    if status == "job not found":
+        raise MLSpaceError(
+            f"Job {job_name} does not exist (its status endpoint returns 'job not found') "
+            "— nothing to delete. The delete endpoint would still reply status=\"deleted\" "
+            "with a fresh deleted_at for this non-existent job; that is a FALSE confirmation.",
+            hint="Verify the exact job_name via `list` (with region); if you created it "
+            "<~2s ago, wait for it to propagate and retry.",
+        )
+    if not status:
+        raise _precheck_unconfirmed(job_name, "precheck returned no status")
+    # else: a concrete live status (pending/running/failed/…) → existence confirmed.
+
+
+def postprocess_job_delete(data: Any, values: dict[str, Any]) -> Any:
+    """Annotate a jobs delete so the caller does not read 'deleted' as a purge.
+
+    Live finding: after a 200 ``status="deleted"`` the job KEEPS appearing in
+    `list`/`get` with a terminal status, and each repeated delete returns a fresh
+    ``deleted_at``. MLSpace delete is therefore LOGICAL (mark-deleted/stopped), not a
+    history purge. We add a leading-underscore note and leave the original fields
+    untouched. Fail-open on any other shape."""
+    if isinstance(data, dict) and str(data.get("status", "")).strip().casefold() == "deleted":
+        return {
+            **data,
+            "_wrapper_note": (
+                "MLSpace delete is LOGICAL: status='deleted' with a fresh deleted_at means "
+                "the job was marked deleted/stopped, NOT purged. It can still appear in "
+                "`list`/`get` with a terminal status (completed/failed/stopped) afterward, "
+                "and a repeated delete returns a fresh deleted_at again — do NOT treat this "
+                "as proof the job is gone from history."
+            ),
+        }
+    return data