chutes-miner-cli 0.2.0__tar.gz

chutes_miner_cli-0.2.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.3
+Name: chutes-miner-cli
+Version: 0.2.0
+Summary: Chutes miner CLI
+License: MIT
+Author: Jon Durbin
+Author-email: jon@jondurbin.com
+Requires-Python: >=3.12,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.10
+Requires-Dist: aiohttp[speedups] (>3.10,<4)
+Requires-Dist: click (>8.1.0,<8.2.0)
+Requires-Dist: loguru (>=0.7.3,<0.8.0)
+Requires-Dist: py-bip39-bindings (>=0.1.11,<0.2.0)
+Requires-Dist: pydantic (>2.9,<3)
+Requires-Dist: rich (>13.0.0)
+Requires-Dist: setuptools (>80.0)
+Requires-Dist: substrate-interface (>=1.7.11,<2.0.0)
+Requires-Dist: typer (>=0.14.0,<0.15.0)
+Project-URL: Homepage, https://github.com/rayonlabs/chutes-miner/tree/main/cli
+Description-Content-Type: text/markdown
+
+

chutes_miner_cli-0.2.0/chutes_miner_cli/cli.py

@@ -0,0 +1,487 @@
+import json
+import asyncio
+from typing import Optional
+import aiohttp
+import typer
+from rich.console import Console
+from rich.table import Table
+from rich import box
+import datetime
+from chutes_miner_cli.util import sign_request
+from loguru import logger
+
+app = typer.Typer(no_args_is_help=True)
+
+
+def format_memory(memory_bytes):
+    """
+    Convert memory from bytes to GB and format nicely.
+    """
+    return f"{memory_bytes / (1024**3):.1f}GB"
+
+
+def format_date(date_str):
+    """
+    Format datetime string to a more readable format.
+    """
+    dt = datetime.datetime.fromisoformat(date_str)
+    return dt.strftime("%Y-%m-%d %H:%M")
+
+
+def format_verification(error, verified_at):
+    """
+    Helper to format table cell for GPU verification.
+    """
+    if verified_at:
+        return f"[green]Verified: {format_date(verified_at)}[/green]"
+    elif error:
+        return f"[red]Error: {error}[/red]"
+    return "[yellow]Pending[/yellow]"
+
+
+async def delete_preflight(server, hotkey, miner_api):
+    """
+    Check if a node has any jobs running and confirm the miner wants
+    to crush their incentives if they terminate regardless.
+    """
+
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        headers, payload_string = sign_request(hotkey, purpose="management")
+        async with session.get(
+            f"{miner_api.rstrip('/')}/servers/{server}/delete_preflight",
+            headers=headers,
+        ) as resp:
+            data = await resp.json()
+            if data.get("jobs"):
+                logger.warning(
+                    f"There are {len(data['jobs'])} jobs running on server {server}, "
+                    "deleting this node will be highly detrimental to your score."
+                )
+                user_input = input("Continue (y/n)? ").strip().lower()
+                if user_input != "y":
+                    return False
+    return True
+
+
+def display_local_inventory(inventory):
+    """
+    Render inventory in fancy tables.
+    """
+    console = Console()
+    for server in inventory:
+        server_table = Table(title=f"Server: {server['name']}", box=box.ROUNDED)
+        server_table.add_column("Property", style="cyan")
+        server_table.add_column("Value")
+        server_table.add_row("Status", server["status"])
+        server_table.add_row("GPUs", str(server["gpu_count"]))
+        server_table.add_row("Memory/GPU", f"{server['memory_per_gpu']}GB")
+        server_table.add_row("CPU/GPU", str(server["cpu_per_gpu"]))
+        server_table.add_row("Hourly Cost", f"${server['hourly_cost']:.2f}")
+        server_table.add_row("IP Address", server["ip_address"])
+        server_table.add_row("Created", format_date(server["created_at"]))
+        console.print(server_table)
+        console.print()
+
+        # Deployments.
+        if server["deployments"]:
+            deploy_table = Table(title="Active Deployments", box=box.ROUNDED)
+            deploy_table.add_column("Model Name")
+            deploy_table.add_column("GPUs")
+            deploy_table.add_column("Port")
+            deploy_table.add_column("Created")
+            deploy_table.add_column("Status")
+            for deploy in server["deployments"]:
+                status_text = (
+                    "[green]Active[/green]"
+                    if deploy["active"] and not deploy["stub"]
+                    else "[red]Inactive[/red]"
+                )
+                deploy_table.add_row(
+                    deploy["chute"]["name"],
+                    str(len(deploy["gpus"])),
+                    str(deploy["port"]),
+                    format_date(deploy["created_at"]),
+                    status_text,
+                )
+            console.print(deploy_table)
+            console.print()
+
+        # GPU details.
+        gpu_table = Table(title="GPU Details", box=box.ROUNDED)
+        gpu_table.add_column("Name")
+        gpu_table.add_column("Memory")
+        gpu_table.add_column("Clock (MHz)")
+        gpu_table.add_column("Processors")
+        gpu_table.add_column("Status")
+        for gpu in server["gpus"]:
+            status_text = "[green]Verified[/green]" if gpu["verified"] else "[red]Unverified[/red]"
+            gpu_table.add_row(
+                gpu["device_info"]["name"],
+                format_memory(gpu["device_info"]["memory"]),
+                str(int(gpu["device_info"]["clock_rate"] / 1000)),
+                str(gpu["device_info"]["processors"]),
+                status_text,
+            )
+
+        console.print(gpu_table)
+        console.print("\n" + "=" * 80 + "\n")
+
+
+def display_remote_inventory(inventory):
+    """
+    Render remote/validator inventory.
+    """
+    console = Console()
+    table = Table(title="GPU Information")
+    table.add_column("Name", style="cyan")
+    table.add_column("Chute", style="cyan")
+    table.add_column("Memory (GB)", justify="right", style="green")
+    table.add_column("Clock (MHz)", justify="right", style="red")
+    table.add_column("Created At", style="blue")
+    table.add_column("GPU Verification", style="white")
+    table.add_column("Instance verification", style="white")
+    for gpu in inventory:
+        table.add_row(
+            gpu["name"],
+            f"{gpu['chute_id']} {gpu['chute']}",
+            format_memory(gpu["memory"]),
+            f"{gpu['clock_rate'] / 1000:.0f}",
+            format_date(gpu["created_at"]),
+            format_verification(gpu["verification_error"], gpu["verified_at"]),
+            format_verification(gpu["inst_verification_error"], gpu["inst_verified_at"]),
+        )
+    console.print(table)
+    console.print("\n" + "=" * 80 + "\n")
+
+
+def local_inventory(
+    raw_json: bool = typer.Option(False, help="Display raw JSON output"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Show local inventory.
+    """
+
+    async def _local_inventory():
+        nonlocal hotkey, miner_api, raw_json
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, _ = sign_request(hotkey, purpose="management")
+            async with session.get(
+                f"{miner_api.rstrip('/')}/servers/",
+                headers=headers,
+                timeout=30,
+            ) as resp:
+                inventory = await resp.json()
+                if raw_json:
+                    print(json.dumps(inventory, indent=2))
+                else:
+                    display_local_inventory(inventory)
+
+    asyncio.run(_local_inventory())
+
+
+def remote_inventory(
+    raw_json: bool = typer.Option(False, help="Display raw JSON output"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    validator_api: str = typer.Option("https://api.chutes.ai", help="Validator API base URL"),
+):
+    """
+    Show remote (i.e., what the validator has tracked) inventory.
+    """
+
+    async def _remote_inventory():
+        nonlocal hotkey, validator_api, raw_json
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, _ = sign_request(hotkey, purpose="miner", remote=True)
+            inventory = []
+            gpu_map = {}
+            async with session.get(f"{validator_api}/miner/nodes/", headers=headers) as resp:
+                async for content_enc in resp.content:
+                    content = content_enc.decode()
+                    if content.startswith("data: "):
+                        inventory.append(json.loads(content[6:]))
+                        gpu_map[inventory[-1]["uuid"]] = inventory[-1]
+                        gpu_map[inventory[-1]["uuid"]].update(
+                            {
+                                "chute": None,
+                                "chute_id": None,
+                                "inst_verification_error": None,
+                                "inst_verified_at": None,
+                            }
+                        )
+            async with session.get(f"{validator_api}/miner/inventory", headers=headers) as resp:
+                for item in await resp.json():
+                    if item["gpu_id"] in gpu_map:
+                        gpu_map[item["gpu_id"]].update(
+                            {
+                                "chute": item["chute_name"],
+                                "chute_id": item["chute_id"],
+                                "inst_verification_error": item["verification_error"],
+                                "inst_verified_at": item["last_verified_at"],
+                            }
+                        )
+            inventory = sorted(inventory, key=lambda o: o["created_at"])
+            if raw_json:
+                print(json.dumps(inventory, indent=2))
+            else:
+                display_remote_inventory(inventory)
+
+    asyncio.run(_remote_inventory())
+
+
+def add_node(
+    name: str = typer.Option(..., help="Name of the server/node"),
+    validator: str = typer.Option(..., help="Validator ss58 this node is allocated to"),
+    hourly_cost: float = typer.Option(..., help="Hourly cost, used in optimizing autoscaling"),
+    gpu_short_ref: str = typer.Option(..., help="GPU short reference"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    agent_api: Optional[str] = typer.Option(None, help="Agent API base URL (K3s Only)"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Entrypoint for adding a new kubernetes node.
+    """
+
+    async def _add_node():
+        nonlocal name, validator, hourly_cost, gpu_short_ref, hotkey, miner_api
+        async with aiohttp.ClientSession(raise_for_status=False) as session:
+            payload = {
+                "name": name,
+                "validator": validator,
+                "hourly_cost": hourly_cost,
+                "gpu_short_ref": gpu_short_ref,
+                "agent_api": agent_api,
+            }
+            headers, payload_string = sign_request(hotkey, payload=payload)
+            async with session.post(
+                f"{miner_api.rstrip('/')}/servers/",
+                headers=headers,
+                data=payload_string,
+                timeout=900,
+            ) as resp:
+                if resp.status != 200:
+                    print(f"\033[31mError adding node:\n{await resp.text()}\033[0m")
+                    resp.raise_for_status()
+                async for content in resp.content:
+                    if content.strip():
+                        payload = json.loads(content.decode()[6:])
+                        print(f"\033[34m{payload['timestamp']}\033[0m {payload['message']}")
+
+    asyncio.run(_add_node())
+
+
+def delete_node(
+    name: str = typer.Option(..., help="Name of the server/node"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Entrypoint for deleting a kubernetes node.
+    """
+
+    async def _delete_node():
+        nonlocal name, hotkey, miner_api
+
+        # Before we do anything, check if the server has any jobs running. Deleting
+        # a job is very, very bad.
+        if not await delete_preflight(name, hotkey, miner_api):
+            return
+
+        # Proceed with deletion.
+        async with aiohttp.ClientSession(raise_for_status=False) as session:
+            headers, payload_string = sign_request(hotkey, purpose="management")
+            async with session.delete(
+                f"{miner_api.rstrip('/')}/servers/{name}",
+                headers=headers,
+            ) as resp:
+                if resp.status == 404:
+                    logger.warning(f"Node not found: {name}")
+                    return
+                resp.raise_for_status()
+                print(json.dumps(await resp.json(), indent=2))
+
+    asyncio.run(_delete_node())
+
+
+def purge_deployments(
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Rebalance all chutes - this just deletes all current instances and let's gepetto re-scale for max $$$
+    This does not purge jobs!
+    """
+
+    async def _purge_deployments():
+        nonlocal hotkey, miner_api
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, payload_string = sign_request(hotkey, purpose="management")
+            async with session.delete(
+                f"{miner_api.rstrip('/')}/deployments/purge",
+                headers=headers,
+            ) as resp:
+                print(json.dumps(await resp.json(), indent=2))
+
+    asyncio.run(_purge_deployments())
+
+
+def purge_deployment(
+    deployment_id: str = typer.Option(
+        None, "--deployment-id", "-d", help="The ID of the deployment to purge."
+    ),
+    node_id: str = typer.Option(
+        None, "--node-id", "-n", help="The ID of the node to purge the deployment from."
+    ),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Purge the target deployment
+    """
+
+    if (deployment_id is None and node_id is None) or (
+        deployment_id is not None and node_id is not None
+    ):
+        typer.echo("Error: Either deployment_id or node_id must be provided, but not both.")
+        raise typer.Exit(1)
+
+    target_id = deployment_id or node_id
+    endpoint = f"deployments/{target_id}" if deployment_id else f"servers/{target_id}/deployments"
+
+    async def _purge_deployment():
+        nonlocal target_id, hotkey, miner_api, endpoint
+
+        # Make sure there aren't jobs and or miner wants to delete anyways.
+        if node_id and not await delete_preflight(node_id, hotkey, miner_api):
+            return
+
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, payload_string = sign_request(hotkey, purpose="management")
+            async with session.delete(
+                f"{miner_api.rstrip('/')}/{endpoint}",
+                headers=headers,
+            ) as resp:
+                print(json.dumps(await resp.json(), indent=2))
+
+    asyncio.run(_purge_deployment())
+
+
+def scorch_remote(
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    validator_api: str = typer.Option("https://api.chutes.ai", help="Validator API base URL"),
+):
+    """
+    Purge all inventory from the validator API.
+    """
+
+    async def _scorch_remote():
+        nonlocal hotkey, validator_api
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, _ = sign_request(hotkey, purpose="miner", remote=True)
+            async with session.get(
+                f"{validator_api.rstrip('/')}/miner/nodes/",
+                headers=headers,
+            ) as resp:
+                async for line in resp.content:
+                    if not line or not line.startswith(b"data:"):
+                        continue
+                    gpu = json.loads(line.decode()[6:])
+                    print(f"Deleting {gpu['name']} with uuid {gpu['uuid']}")
+                    headers, _ = sign_request(hotkey, purpose="nodes", remote=True)
+                    async with session.delete(
+                        f"{validator_api.rstrip('/')}/nodes/{gpu['uuid']}",
+                        headers=headers,
+                    ) as resp:
+                        print(f"  successfully deleted {gpu['name']} with uuid {gpu['uuid']}")
+
+    asyncio.run(_scorch_remote())
+
+
+def delete_remote(
+    gpu_id: str = typer.Option(help="GPU UUID to delete, aka node_node on validator side"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    validator_api: str = typer.Option("https://api.chutes.ai", help="Validator API base URL"),
+):
+    """
+    Delete a single GPU from validator.
+    """
+
+    async def _delete_remote():
+        nonlocal hotkey, validator_api, gpu_id
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            headers, _ = sign_request(hotkey, purpose="nodes", remote=True)
+            async with session.delete(
+                f"{validator_api.rstrip('/')}/nodes/{gpu_id}",
+                headers=headers,
+            ) as _:
+                print(f"  successfully deleted {gpu_id}")
+
+    asyncio.run(_delete_remote())
+
+
+async def _lock_or_unlock_server(lock: bool, name: str, hotkey: str, miner_api: str):
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        headers, _ = sign_request(hotkey, purpose="management")
+        path = "/lock" if lock else "/unlock"
+        async with session.get(
+            f"{miner_api.rstrip('/')}/servers/{name}{path}",
+            headers=headers,
+        ) as resp:
+            server = await resp.json()
+            print(f"Server {server['name']} lock status is now: {server['locked']}")
+
+
+def lock_server(
+    name: str = typer.Option(..., help="Name of the server/node"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Lock a server's deployments.
+    """
+
+    async def _lock_server():
+        nonlocal name, hotkey, miner_api
+        await _lock_or_unlock_server(lock=True, name=name, hotkey=hotkey, miner_api=miner_api)
+
+    asyncio.run(_lock_server())
+
+
+def unlock_server(
+    name: str = typer.Option(..., help="Name of the server/node"),
+    hotkey: str = typer.Option(..., help="Path to the hotkey file for your miner"),
+    miner_api: str = typer.Option("http://127.0.0.1:32000", help="Miner API base URL"),
+):
+    """
+    Unlock a server's deployments.
+    """
+
+    async def _unlock_server():
+        nonlocal name, hotkey, miner_api
+        await _lock_or_unlock_server(lock=False, name=name, hotkey=hotkey, miner_api=miner_api)
+
+    asyncio.run(_unlock_server())
+
+
+app.command(name="add-node", help="Add a new kubernetes node to your cluster")(add_node)
+app.command(name="delete-node", help="Delete a kubernetes node from your cluster")(delete_node)
+app.command(
+    name="purge-deployments",
+    help="Purge all deployments, allowing autoscale from scratch",
+)(purge_deployments)
+app.command(name="purge-deployment", help="Purge the target deployment")(purge_deployment)
+app.command(name="local-inventory", help="Show local inventory")(local_inventory)
+app.command(name="remote-inventory", help="Show remote inventory")(remote_inventory)
+app.command(name="scorch-remote", help="Purge all GPUs/instances/etc. from validator")(
+    scorch_remote
+)
+app.command(name="delete-remote", help="Remove a single GPU from validator inventory")(
+    delete_remote
+)
+app.command(name="lock", help="Lock a server's deployments")(lock_server)
+app.command(name="unlock", help="Unlock a server's deployments")(unlock_server)
+
+
+if __name__ == "__main__":
+    app()

chutes_miner_cli-0.2.0/chutes_miner_cli/constants.py

@@ -0,0 +1,5 @@
+VALIDATOR_HEADER = "X-Chutes-Validator"
+MINER_HEADER = "X-Chutes-Miner"
+NONCE_HEADER = "X-Chutes-Nonce"
+SIGNATURE_HEADER = "X-Chutes-Signature"
+HOTKEY_HEADER = "X-Chutes-Hotkey"

chutes_miner_cli-0.2.0/chutes_miner_cli/util.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import json
+import hashlib
+import time
+from substrateinterface import Keypair
+from typing import Dict, Any
+from chutes_miner_cli.constants import (
+    VALIDATOR_HEADER,
+    HOTKEY_HEADER,
+    MINER_HEADER,
+    NONCE_HEADER,
+    SIGNATURE_HEADER,
+)
+
+
+def get_signing_message(
+    hotkey: str,
+    nonce: str,
+    payload_str: str | bytes | None,
+    purpose: str | None = None,
+    payload_hash: str | None = None,
+) -> str:
+    """
+    Get the signing message for a given hotkey, nonce, and payload.
+    """
+    if payload_str:
+        if isinstance(payload_str, str):
+            payload_str = payload_str.encode()
+        return f"{hotkey}:{nonce}:{hashlib.sha256(payload_str).hexdigest()}"
+    elif purpose:
+        return f"{hotkey}:{nonce}:{purpose}"
+    elif payload_hash:
+        return f"{hotkey}:{nonce}:{payload_hash}"
+    else:
+        raise ValueError("Either payload_str or purpose must be provided")
+
+
+def sign_request(
+    hotkey: str,
+    payload: Dict[str, Any] | str | None = None,
+    purpose: str = None,
+    remote: bool = False,
+):
+    """
+    Generate a signed request (for miner requests to validators).
+    """
+    hotkey_data = json.loads(open(hotkey).read())
+    nonce = str(int(time.time()))
+    headers = {
+        MINER_HEADER: hotkey_data["ss58Address"],
+        NONCE_HEADER: nonce,
+    }
+    if remote:
+        headers[HOTKEY_HEADER] = headers.pop(MINER_HEADER)
+    signature_string = None
+    payload_string = None
+    if payload is not None:
+        if isinstance(payload, (list, dict)):
+            headers["Content-Type"] = "application/json"
+            payload_string = json.dumps(payload)
+        else:
+            payload_string = payload
+        signature_string = get_signing_message(
+            hotkey_data["ss58Address"],
+            nonce,
+            payload_str=payload_string,
+            purpose=None,
+        )
+    else:
+        signature_string = get_signing_message(
+            hotkey_data["ss58Address"], nonce, payload_str=None, purpose=purpose
+        )
+    if not remote:
+        signature_string = hotkey_data["ss58Address"] + ":" + signature_string
+        headers[VALIDATOR_HEADER] = headers[MINER_HEADER]
+    keypair = Keypair.create_from_seed(hotkey_data["secretSeed"])
+    headers[SIGNATURE_HEADER] = keypair.sign(signature_string.encode()).hex()
+    return headers, payload_string

chutes_miner_cli-0.2.0/pyproject.toml

@@ -0,0 +1,39 @@
+[tool.poetry]
+name = "chutes-miner-cli"
+version = "0.2.0"
+description = "Chutes miner CLI"
+readme = "README.md"
+license = "MIT"
+authors = [ "Jon Durbin <jon@jondurbin.com>" ]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3.10",
+]
+
+[tool.poetry.dependencies]
+python = "^3.12"
+loguru = "^0.7.3"
+pydantic = ">2.9,<3"
+setuptools = ">80.0"
+substrate-interface = "^1.7.11"
+py-bip39-bindings = "^0.1.11"
+rich = ">13.0.0"
+typer = "^0.14.0"
+click = ">8.1.0,<8.2.0"
+aiohttp = {extras = ["speedups"], version = ">3.10,<4"}
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.12.2"
+wheel = "^0.45.1"
+
+[tool.poetry.urls]
+Homepage = "https://github.com/rayonlabs/chutes-miner/tree/main/cli"
+
+[tool.poetry.scripts]
+chutes-miner = "chutes_miner_cli.cli:app"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"