inference-aiops 0.1.0__py3-none-any.whl

inference_aiops/__init__.py

@@ -0,0 +1,9 @@
+"""inference-aiops — governed Inference SCALE operations for AI agents.
+
+Standalone and self-contained: the governance harness (audit, token budget,
+undo-token recording, graduated risk tiers, prompt-injection sanitize) is
+bundled under ``inference_aiops.governance`` — this package has no external
+skill-family dependency. Preview: not yet full-coverage.
+"""
+
+__version__ = "0.1.0"

inference_aiops/cli/__init__.py

@@ -0,0 +1,9 @@
+"""CLI package for inference-aiops.
+
+Re-exports ``app`` so the pyproject entry point
+``inference-aiops = "inference_aiops.cli:app"`` works unchanged.
+"""
+
+from inference_aiops.cli._root import app
+
+__all__ = ["app"]

inference_aiops/cli/_common.py

@@ -0,0 +1,78 @@
+"""Shared helpers for inference-aiops CLI sub-modules."""
+
+from __future__ import annotations
+
+import functools
+from collections.abc import Callable
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich.console import Console
+
+console = Console()
+
+# ─── Shared Option types ───────────────────────────────────────────────────
+
+TargetOption = Annotated[
+    str | None, typer.Option("--target", "-t", help="Target name from config")
+]
+DryRunOption = Annotated[
+    bool, typer.Option("--dry-run", help="Print the API call without executing")
+]
+
+
+def _cli_error_types() -> tuple[type[BaseException], ...]:
+    """Exceptions translated to a one-line teaching error instead of a traceback."""
+    from inference_aiops.connection import InferenceApiError
+
+    return (InferenceApiError, KeyError, OSError, ValueError)
+
+
+def cli_errors(fn: Callable) -> Callable:
+    """Translate known exceptions into one red line + exit code 1."""
+
+    @functools.wraps(fn)
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return fn(*args, **kwargs)
+        except (typer.Exit, typer.Abort):
+            raise
+        except _cli_error_types() as e:
+            message = str(e)
+            if isinstance(e, KeyError):
+                message = f"Missing required key or environment variable: {message}"
+            console.print(f"[red]Error: {message}[/]")
+            raise typer.Exit(1) from e
+
+    return wrapper
+
+
+def get_connection(target: str | None, config_path: Path | None = None):
+    """Return a (conn, config) tuple for the given target."""
+    from inference_aiops.config import load_config
+    from inference_aiops.connection import ConnectionManager
+
+    cfg = load_config(config_path)
+    mgr = ConnectionManager(cfg)
+    return mgr.connect(target), cfg
+
+
+def dry_run_print(*, operation: str, api_call: str, parameters: dict | None = None) -> None:
+    """Print a dry-run preview of the API call that would be made."""
+    console.print("\n[bold magenta][DRY-RUN] No changes will be made.[/]")
+    console.print(f"[magenta]  Operation: {operation}[/]")
+    console.print(f"[magenta]  API Call:  {api_call}[/]")
+    for k, v in (parameters or {}).items():
+        console.print(f"[magenta]  Param:     {k} = {v}[/]")
+    console.print("[magenta]  Run without --dry-run to execute.[/]\n")
+
+
+def double_confirm(action: str, resource: str) -> None:
+    """Require two confirmations for a destructive operation."""
+    console.print(f"[bold yellow]⚠️  About to: {action} '{resource}'[/]")
+    typer.confirm(f"Confirm 1/2: {action} '{resource}'?", abort=True)
+    typer.confirm(
+        f"Confirm 2/2: really {action} '{resource}'? This may be irreversible.",
+        abort=True,
+    )

inference_aiops/cli/_root.py

@@ -0,0 +1,57 @@
+"""Top-level Typer app: assembles sub-apps and top-level commands."""
+
+from __future__ import annotations
+
+import typer
+
+from inference_aiops.cli._common import cli_errors
+from inference_aiops.cli.doctor import doctor_cmd
+from inference_aiops.cli.init import init_cmd
+from inference_aiops.cli.metrics import metrics_app
+from inference_aiops.cli.overview import overview_cmd
+from inference_aiops.cli.secret import secret_app
+from inference_aiops.cli.serve import serve_app
+
+app = typer.Typer(
+    name="inference-aiops",
+    help="Governed AI-ops for GPU inference (vLLM + Ray Serve): metrics/RCA, "
+    "scaling, drain, models, jobs.",
+    no_args_is_help=True,
+)
+
+app.add_typer(serve_app, name="serve")
+app.add_typer(metrics_app, name="metrics")
+app.add_typer(secret_app, name="secret")
+app.command("init")(init_cmd)
+app.command("overview")(overview_cmd)
+app.command("doctor")(doctor_cmd)
+
+
+@app.command("mcp")
+@cli_errors
+def mcp_cmd() -> None:
+    """Start the MCP server (stdio transport).
+
+    Single-command entry point for MCP clients (does not go through uvx/PyPI
+    resolution at launch):
+        inference-aiops mcp
+    """
+    import sys
+
+    if sys.version_info < (3, 11):
+        typer.echo(
+            f"ERROR: inference-aiops requires Python >= 3.11 "
+            f"(got {sys.version_info.major}.{sys.version_info.minor}).\n"
+            f"Fix: uv python install 3.12 && "
+            f"uv tool install --python 3.12 --force inference-aiops",
+            err=True,
+        )
+        raise typer.Exit(2)
+
+    from mcp_server.server import main as _mcp_main
+
+    _mcp_main()
+
+
+if __name__ == "__main__":
+    app()

inference_aiops/cli/doctor.py

@@ -0,0 +1,21 @@
+"""Doctor top-level command: environment and connectivity check."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import typer
+
+from inference_aiops.cli._common import cli_errors
+
+
+@cli_errors
+def doctor_cmd(
+    skip_auth: Annotated[
+        bool, typer.Option("--skip-auth", help="Skip connectivity check (faster)")
+    ] = False,
+) -> None:
+    """Check environment, config, secrets, and connectivity."""
+    from inference_aiops.doctor import run_doctor
+
+    raise typer.Exit(run_doctor(skip_auth=skip_auth))

inference_aiops/cli/init.py

@@ -0,0 +1,97 @@
+"""``inference-aiops init`` — a friendly, interactive onboarding wizard.
+
+Walks a new user through connecting their first Inference SCALE target: collects
+the non-secret connection details into ``config.yaml`` and the API key into the
+*encrypted* store (never plaintext on disk). Designed to be run on a terminal;
+everything it needs is prompted with sensible defaults.
+"""
+
+from __future__ import annotations
+
+import getpass
+
+import typer
+import yaml
+
+from inference_aiops.cli._common import cli_errors, console
+from inference_aiops.config import CONFIG_DIR, CONFIG_FILE, DEFAULT_RAY_PORT, DEFAULT_VLLM_PORT
+from inference_aiops.secretstore import SecretStore, resolve_master_password
+
+
+def _load_existing_targets() -> list[dict]:
+    if not CONFIG_FILE.exists():
+        return []
+    raw = yaml.safe_load(CONFIG_FILE.read_text("utf-8")) or {}
+    return list(raw.get("targets", []))
+
+
+def _write_targets(targets: list[dict]) -> None:
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    try:
+        CONFIG_DIR.chmod(0o700)
+    except OSError:
+        pass
+    CONFIG_FILE.write_text(yaml.safe_dump({"targets": targets}, sort_keys=False), "utf-8")
+
+
+@cli_errors
+def init_cmd() -> None:
+    """Interactively set up your first Inference connection."""
+    console.print("[bold cyan]Inference AIops — setup wizard[/]")
+    console.print(
+        "This collects Ray dashboard + vLLM connection details (saved to "
+        "config.yaml). A bearer token is optional (many inference stacks run "
+        "open); if given it is saved [bold]encrypted[/] to secrets.enc.\n"
+    )
+
+    console.print("[bold]Step 1 — master password[/]")
+    console.print(
+        "[dim]Encrypts secrets.enc. You'll set it via the "
+        "INFERENCE_AIOPS_MASTER_PASSWORD env var for non-interactive/MCP use.[/]"
+    )
+    password = resolve_master_password(confirm_if_new=True)
+    store = SecretStore.unlock(password)
+
+    targets = _load_existing_targets()
+    existing_names = {t.get("name") for t in targets}
+
+    while True:
+        console.print("\n[bold]Step 2 — add an inference target[/]")
+        name = typer.prompt("Target name (e.g. prod)").strip()
+        if name in existing_names:
+            if not typer.confirm(f"'{name}' already exists — overwrite?", default=False):
+                continue
+            targets = [t for t in targets if t.get("name") != name]
+
+        host = typer.prompt("Host (IP or FQDN, shared by Ray + vLLM)").strip()
+        ray_port = typer.prompt("Ray dashboard port", default=DEFAULT_RAY_PORT, type=int)
+        vllm_port = typer.prompt("vLLM port", default=DEFAULT_VLLM_PORT, type=int)
+
+        if typer.confirm("Does the API require a bearer token?", default=False):
+            secret = getpass.getpass(f"Token for '{name}' (hidden): ")
+            store = store.set(name, secret)
+
+        entry = {
+            "name": name,
+            "host": host,
+            "ray_port": ray_port,
+            "vllm_port": vllm_port,
+            "scheme": "http",
+        }
+        targets.append(entry)
+        existing_names.add(name)
+        _write_targets(targets)
+        console.print(f"[green]✓ Saved target '{name}'.[/]")
+
+        if not typer.confirm("\nAdd another target?", default=False):
+            break
+
+    console.print(f"\n[green]✓ Setup complete.[/] Config: {CONFIG_FILE}")
+    console.print(
+        "[dim]Tip: export INFERENCE_AIOPS_MASTER_PASSWORD=... in your shell profile "
+        "so the MCP server and CLI can unlock secrets non-interactively.[/]"
+    )
+    if typer.confirm("Run a connectivity check now (inference-aiops doctor)?", default=True):
+        from inference_aiops.doctor import run_doctor
+
+        raise typer.Exit(run_doctor())

inference_aiops/cli/metrics.py

@@ -0,0 +1,45 @@
+"""``inference-aiops metrics`` — vLLM metric reads + latency/util RCA."""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from inference_aiops.cli._common import TargetOption, cli_errors, console, get_connection
+
+metrics_app = typer.Typer(
+    name="metrics",
+    help="vLLM request metrics, queue depth, KV cache, and latency/util RCA.",
+    no_args_is_help=True,
+)
+
+
+@metrics_app.command("requests")
+@cli_errors
+def metrics_requests(target: TargetOption = None) -> None:
+    """TTFT / TPOT / e2e latency + generation-token totals."""
+    from inference_aiops.ops import metrics as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.get_request_metrics(conn)))
+
+
+@metrics_app.command("queue")
+@cli_errors
+def metrics_queue(target: TargetOption = None) -> None:
+    """Running vs waiting requests (backpressure signal)."""
+    from inference_aiops.ops import metrics as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.get_queue_depth(conn)))
+
+
+@metrics_app.command("diagnose")
+@cli_errors
+def metrics_diagnose(target: TargetOption = None) -> None:
+    """RCA: rank the probable cause of a latency spike + the knob to turn."""
+    from inference_aiops.ops import metrics as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.diagnose_latency_spike(conn)))

inference_aiops/cli/overview.py

@@ -0,0 +1,16 @@
+"""``inference-aiops overview`` — one-shot fleet health."""
+
+from __future__ import annotations
+
+import json
+
+from inference_aiops.cli._common import TargetOption, cli_errors, console, get_connection
+
+
+@cli_errors
+def overview_cmd(target: TargetOption = None) -> None:
+    """One-shot stack summary: deployments, total replicas, queue backpressure."""
+    from inference_aiops.ops import overview as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.fleet_overview(conn)))

inference_aiops/cli/secret.py

@@ -0,0 +1,105 @@
+"""``inference-aiops secret`` — manage the encrypted credential store.
+
+Secrets (Inference API keys) are stored in ``~/.inference-aiops/secrets.enc``
+(Fernet, key derived from a master password via scrypt). Nothing here ever
+prints a secret value.
+"""
+
+from __future__ import annotations
+
+import getpass
+from typing import Annotated
+
+import typer
+
+from inference_aiops.cli._common import cli_errors, console
+from inference_aiops.config import SECRET_ENV_PREFIX, SECRET_ENV_SUFFIX
+from inference_aiops.secretstore import (
+    SECRETS_FILE,
+    SecretStore,
+    check_permissions,
+    migrate_legacy_env,
+    resolve_master_password,
+)
+
+secret_app = typer.Typer(
+    name="secret",
+    help="Manage the encrypted credential store (secrets.enc).",
+    no_args_is_help=True,
+)
+
+NameArg = Annotated[str, typer.Argument(help="Target name the API key belongs to")]
+
+
+@secret_app.command("set")
+@cli_errors
+def secret_set(
+    name: NameArg,
+    value: Annotated[
+        str | None,
+        typer.Option("--value", help="API key value (omit to be prompted, hidden)"),
+    ] = None,
+) -> None:
+    """Store (or replace) the API key for a target — value is read hidden."""
+    password = resolve_master_password(confirm_if_new=True)
+    if value is None:
+        value = getpass.getpass(f"API key for '{name}' (hidden): ")
+    store = SecretStore.unlock(password)
+    store.set(name, value)
+    console.print(f"[green]✓ Stored encrypted API key for '{name}' in {SECRETS_FILE}[/]")
+
+
+@secret_app.command("list")
+@cli_errors
+def secret_list() -> None:
+    """List target names that have a stored API key (values never shown)."""
+    store = SecretStore.unlock()
+    names = store.names()
+    if not names:
+        console.print(
+            "[yellow]No secrets stored yet. Add one: inference-aiops secret set <name>[/]"
+        )
+        return
+    console.print("[bold]Stored secrets:[/]")
+    for n in names:
+        console.print(f"  • {n}")
+    warning = check_permissions()
+    if warning:
+        console.print(f"[yellow]! {warning}[/]")
+
+
+@secret_app.command("rm")
+@cli_errors
+def secret_rm(name: NameArg) -> None:
+    """Delete a stored API key."""
+    store = SecretStore.unlock()
+    store.delete(name)
+    console.print(f"[green]✓ Deleted API key for '{name}'[/]")
+
+
+@secret_app.command("migrate")
+@cli_errors
+def secret_migrate() -> None:
+    """Import API keys from a legacy plaintext .env into the encrypted store."""
+    password = resolve_master_password(confirm_if_new=True)
+    imported = migrate_legacy_env(SECRET_ENV_PREFIX, SECRET_ENV_SUFFIX, password)
+    if not imported:
+        console.print("[yellow]Nothing to migrate (no legacy .env secrets found).[/]")
+        return
+    console.print(f"[green]✓ Imported {len(imported)} secret(s): {', '.join(imported)}[/]")
+    console.print("[dim]The old .env was renamed to .env.migrated — delete it once verified.[/]")
+
+
+@secret_app.command("rotate-password")
+@cli_errors
+def secret_rotate_password() -> None:
+    """Re-encrypt the whole store under a new master password."""
+    console.print("[bold]Unlock with the current master password:[/]")
+    store = SecretStore.unlock()
+    new_pw = getpass.getpass("New master password: ")
+    confirm = getpass.getpass("Confirm new master password: ")
+    if new_pw != confirm:
+        console.print("[red]Passwords did not match. Aborted.[/]")
+        raise typer.Exit(1)
+    store.with_password(new_pw)
+    console.print("[green]✓ Master password rotated. Update INFERENCE_AIOPS_MASTER_PASSWORD.[/]")

inference_aiops/cli/serve.py

@@ -0,0 +1,90 @@
+"""``inference-aiops serve`` — Ray Serve reads + guarded scaling writes."""
+
+from __future__ import annotations
+
+import json
+from typing import Annotated
+
+import typer
+
+from inference_aiops.cli._common import (
+    DryRunOption,
+    TargetOption,
+    cli_errors,
+    console,
+    double_confirm,
+    dry_run_print,
+    get_connection,
+)
+
+serve_app = typer.Typer(
+    name="serve",
+    help="Ray Serve: list/status, scale, scale-to-zero, drain.",
+    no_args_is_help=True,
+)
+
+AppArg = Annotated[str, typer.Argument(help="Serve application name")]
+DepArg = Annotated[str, typer.Argument(help="Deployment name")]
+
+
+@serve_app.command("list")
+@cli_errors
+def serve_list(target: TargetOption = None) -> None:
+    """List Ray Serve deployments."""
+    from inference_aiops.ops import serve as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.list_serve_deployments(conn)))
+
+
+@serve_app.command("status")
+@cli_errors
+def serve_status(application: AppArg, deployment: DepArg, target: TargetOption = None) -> None:
+    """Show one deployment's status + replica count."""
+    from inference_aiops.ops import serve as ops
+
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.get_deployment_status(conn, application, deployment)))
+
+
+@serve_app.command("scale")
+@cli_errors
+def serve_scale(
+    application: AppArg,
+    deployment: DepArg,
+    num_replicas: Annotated[int, typer.Argument(help="Target replica count")],
+    target: TargetOption = None,
+    dry_run: DryRunOption = False,
+) -> None:
+    """Scale a deployment to a target replica count."""
+    from inference_aiops.ops import serve as ops
+
+    if dry_run:
+        dry_run_print(operation="scale_replicas",
+                      api_call=f"PUT /api/serve/applications/{application}/deployments/"
+                               f"{deployment}",
+                      parameters={"num_replicas": num_replicas})
+        return
+    conn, _ = get_connection(target)
+    result = ops.scale_replicas_up(conn, application, deployment, num_replicas)
+    console.print_json(json.dumps(result))
+
+
+@serve_app.command("scale-to-zero")
+@cli_errors
+def serve_scale_zero(
+    application: AppArg, deployment: DepArg,
+    target: TargetOption = None, dry_run: DryRunOption = False,
+) -> None:
+    """Park a deployment at 0 replicas (dry-run + double confirm)."""
+    from inference_aiops.ops import serve as ops
+
+    if dry_run:
+        dry_run_print(operation="scale_to_zero",
+                      api_call=f"PUT /api/serve/applications/{application}/deployments/"
+                               f"{deployment}",
+                      parameters={"num_replicas": 0})
+        return
+    double_confirm("scale to zero", f"{application}/{deployment}")
+    conn, _ = get_connection(target)
+    console.print_json(json.dumps(ops.scale_to_zero(conn, application, deployment)))

inference_aiops/config.py

@@ -0,0 +1,140 @@
+"""Configuration management for Inference AIops.
+
+Loads GPU inference-cluster connection targets from a YAML config file. A
+"target" is one inference stack reached over two HTTP endpoints:
+
+  * a **Ray dashboard** (Ray Serve + Jobs API, default port ``8265``), and
+  * a **vLLM** OpenAI-compatible + Prometheus ``/metrics`` server (default port
+    ``8000``).
+
+Inference endpoints frequently run with **no auth** on a trusted network, so the
+bearer **token is optional**: if none is stored, requests are sent unauthenticated.
+When a token *is* used it is NEVER stored in the config file or in plaintext on
+disk — it lives in the encrypted store ``~/.inference-aiops/secrets.enc`` (see
+:mod:`inference_aiops.secretstore`), with a legacy env var
+(``INFERENCE_<TARGET>_TOKEN``) honoured as a fallback.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from inference_aiops.secretstore import SecretStoreError, get_secret, has_store
+
+CONFIG_DIR = Path.home() / ".inference-aiops"
+CONFIG_FILE = CONFIG_DIR / "config.yaml"
+ENV_FILE = CONFIG_DIR / ".env"
+
+DEFAULT_RAY_PORT = 8265
+DEFAULT_VLLM_PORT = 8000
+
+# Legacy env-var prefix/suffix; also used by the migration helper.
+SECRET_ENV_PREFIX = "INFERENCE_"  # nosec B105 — env-var name, not a secret
+SECRET_ENV_SUFFIX = "_TOKEN"  # nosec B105 — env-var name, not a secret
+
+_log = logging.getLogger("inference-aiops.config")
+
+
+def _secret_env_key(name: str) -> str:
+    """Legacy per-target token env var name, e.g. INFERENCE_PROD_TOKEN."""
+    return f"{SECRET_ENV_PREFIX}{name.upper().replace('-', '_')}{SECRET_ENV_SUFFIX}"
+
+
+def _resolve_secret(name: str) -> str:
+    """Return a target's bearer token, or "" when none is configured (auth optional)."""
+    if has_store():
+        try:
+            return get_secret(name)
+        except SecretStoreError:
+            pass  # fall through to legacy env var / no-auth
+    legacy = os.environ.get(_secret_env_key(name))
+    if legacy:
+        _log.warning(
+            "Using plaintext env var %s. Migrate to the encrypted store with "
+            "'inference-aiops secret migrate'.",
+            _secret_env_key(name),
+        )
+        return legacy
+    return ""  # no token → unauthenticated (common for on-prem inference)
+
+
+@dataclass(frozen=True)
+class TargetConfig:
+    """A connection target for one GPU inference stack (Ray dashboard + vLLM).
+
+    ``host`` is shared by both services; ``ray_port`` (8265) reaches the Ray
+    Serve/Jobs dashboard API and ``vllm_port`` (8000) the vLLM OpenAI +
+    ``/metrics`` server. ``token`` is optional (empty = unauthenticated).
+    """
+
+    name: str
+    host: str
+    ray_port: int = DEFAULT_RAY_PORT
+    vllm_port: int = DEFAULT_VLLM_PORT
+    scheme: str = "http"
+    verify_ssl: bool = False
+
+    @property
+    def token(self) -> str:
+        return _resolve_secret(self.name)
+
+    @property
+    def ray_url(self) -> str:
+        return f"{self.scheme}://{self.host}:{self.ray_port}"
+
+    @property
+    def vllm_url(self) -> str:
+        return f"{self.scheme}://{self.host}:{self.vllm_port}"
+
+
+@dataclass(frozen=True)
+class AppConfig:
+    """Top-level application config."""
+
+    targets: tuple[TargetConfig, ...] = ()
+
+    def get_target(self, name: str) -> TargetConfig:
+        for t in self.targets:
+            if t.name == name:
+                return t
+        available = ", ".join(t.name for t in self.targets) or "(none)"
+        raise KeyError(f"Target '{name}' not found. Available: {available}")
+
+    @property
+    def default_target(self) -> TargetConfig:
+        if not self.targets:
+            raise ValueError("No targets configured. Check config.yaml")
+        return self.targets[0]
+
+
+def load_config(config_path: Path | None = None) -> AppConfig:
+    """Load config from YAML; any bearer token comes from the encrypted store."""
+    path = config_path or CONFIG_FILE
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Config file not found: {path}\n"
+            f"Run 'inference-aiops init' to set up an inference target, or create "
+            f"{CONFIG_FILE} with a 'targets' list."
+        )
+
+    with open(path) as f:
+        raw = yaml.safe_load(f) or {}
+
+    targets = tuple(
+        TargetConfig(
+            name=t["name"],
+            host=t["host"],
+            ray_port=t.get("ray_port", DEFAULT_RAY_PORT),
+            vllm_port=t.get("vllm_port", DEFAULT_VLLM_PORT),
+            scheme=t.get("scheme", "http"),
+            verify_ssl=t.get("verify_ssl", False),
+        )
+        for t in raw.get("targets", [])
+    )
+
+    return AppConfig(targets=targets)