gipper 0.3.0__py3-none-any.whl

gipper/providers/runpod.py

@@ -0,0 +1,225 @@
+"""Runpod: GraphQL for prices, stock and the account, REST for pods and network
+volumes. Containers with ssh as root; TCP and HTTP ports only ("Pods do not
+support UDP connections", docs.runpod.io/pods/configuration/expose-ports).
+
+Regions: stock is asked per country (`lowestPrice(countryCode:)`), and the pod is
+created with `countryCodes`, so a 4090 that only exists in Iceland is not rented
+for a job that must stay in the US.
+"""
+import re
+import time
+from datetime import datetime, timezone
+
+from gipper.config import settings
+from gipper.http import ProviderError, request
+from gipper.market import Offer
+from gipper.providers import BadHost, Provider, Unavailable
+from gipper.ssh import Remote
+
+GRAPHQL = "https://api.runpod.io/graphql"
+REST = "https://rest.runpod.io/v1"
+RunpodError = ProviderError
+# What POST /pods accepts for allowedCudaVersions (REST v1 schema, 2026-10)
+CUDA_VERSIONS = ("13.0", "12.9", "12.8", "12.7", "12.6", "12.5", "12.4", "12.3", "12.2", "12.1", "12.0", "11.8")
+
+
+class Api:
+    def __init__(self, key):
+        self._key = key
+
+    def _req(self, method, url, body=None):
+        return request(method, url, body, headers={"Authorization": f"Bearer {self._key}"})
+
+    def gql(self, query):
+        d = self._req("POST", GRAPHQL, {"query": query})
+        if d.get("errors"):
+            raise ProviderError(str(d["errors"])[:500])
+        return d["data"]
+
+    def account(self):
+        return self.gql("query { myself { clientBalance currentSpendPerHr spendLimit } }")["myself"]
+
+    def gpu_types(self, countries=()):
+        """Every GPU type with prices; stock overall (`lowestPrice`) and, per country
+        and cloud, as `stock_<CC>_<community|secure>`."""
+        extra = "".join(
+            f' stock_{cc}_{c}: lowestPrice(input:{{gpuCount:1, countryCode:"{cc}", secureCloud:{str(c == "secure").lower()}}}){{ stockStatus }}'
+            for cc in countries for c in ("community", "secure"))
+        q = ("query { gpuTypes { id displayName memoryInGb secureCloud communityCloud securePrice communityPrice "
+             f"lowestPrice(input:{{gpuCount:1}}){{ uninterruptablePrice stockStatus }}{extra} }} }}")
+        return self.gql(q)["gpuTypes"]
+
+    def create_pod(self, spec):
+        return self._req("POST", f"{REST}/pods", spec)
+
+    def get_pod(self, pod_id):
+        return self._req("GET", f"{REST}/pods/{pod_id}")
+
+    def list_pods(self):
+        d = self._req("GET", f"{REST}/pods")
+        return d if isinstance(d, list) else d.get("pods", d.get("data", []))
+
+    def terminate_pod(self, pod_id):
+        return self._req("DELETE", f"{REST}/pods/{pod_id}")
+
+    def volumes(self):
+        d = self._req("GET", f"{REST}/networkvolumes")
+        return d if isinstance(d, list) else d.get("networkVolumes", d.get("data", []))
+
+    def create_volume(self, name, size_gb, datacenter):
+        return self._req("POST", f"{REST}/networkvolumes", {"name": name, "size": size_gb, "dataCenterId": datacenter})
+
+    def delete_volume(self, volume_id):
+        return self._req("DELETE", f"{REST}/networkvolumes/{volume_id}")
+
+    _gql = gql   # the name scripts written before 2026-10-26 use
+
+
+def _price(g, cloud):
+    return g.get("communityPrice" if cloud == "COMMUNITY" else "securePrice") or None
+
+
+def offers_from(gpu_types, clouds=("COMMUNITY", "SECURE"), countries=()):
+    """One offer per GPU type and cloud that Runpod prices. With countries, stock is
+    the best stock in any of them and the offer is placed in the first stocked one;
+    without stock anywhere it still counts (Runpod's stock is a hint), ranked last."""
+    out = []
+    for cloud in clouds:
+        c = cloud.lower()
+        for g in gpu_types:
+            if not g.get("communityCloud" if cloud == "COMMUNITY" else "secureCloud"):
+                continue
+            p = _price(g, cloud)
+            if not p:
+                continue
+            if countries:
+                stocked = [(cc, (g.get(f"stock_{cc}_{c}") or {}).get("stockStatus")) for cc in countries]
+                stocked = [(cc, s) for cc, s in stocked if s]
+                country, stock = stocked[0] if stocked else (countries[0], None)
+            else:
+                country, stock = None, (g.get("lowestPrice") or {}).get("stockStatus")
+            out.append(Offer("runpod", g["id"], float(g.get("memoryInGb") or 0), p, where=f"{c} {country or ''}".strip(),
+                             country=country,
+                             stock=stock, udp=False, ref={"gpu_id": g["id"], "cloud": cloud}))
+    return out
+
+
+def _tcp_ports(ports):
+    """Job ports ("8080/tcp", "8888/http") as Runpod wants them; 22 always."""
+    out = ["22/tcp"]
+    for p in ports:
+        num, _, proto = p.partition("/")
+        proto = (proto or "tcp").lower()
+        if proto == "udp":
+            raise Unavailable("Runpod pods cannot expose UDP")
+        if f"{num}/{proto}" not in out:
+            out.append(f"{num}/{proto}")
+    return out
+
+
+class Runpod(Provider):
+    name = "runpod"
+    key_name = "RUNPOD_API_KEY"
+    udp = False
+
+    def api(self):
+        return Api(self.key())
+
+    def offers(self, job):
+        clouds = [job.cloud] + (["SECURE"] if job.cloud == "COMMUNITY" else [])
+        regions = [r.upper() for r in job.regions]
+        if any(not re.fullmatch(r"[A-Z]{2}", r) for r in regions):
+            raise ProviderError(f"regions must be ISO country codes, got {regions}")
+        return offers_from(self.api().gpu_types(regions), clouds, regions)
+
+    def create(self, offer, job, pubkey, name):
+        api = self.api()
+        s = settings()
+        cloud = offer.ref["cloud"]
+        spec = {
+            "name": name, "imageName": job.image or s["image"], "gpuTypeIds": [offer.ref["gpu_id"]], "gpuCount": 1,
+            "cloudType": cloud, "computeType": "GPU", "containerDiskInGb": job.container_disk_gb, "volumeInGb": 0,
+            # direct ssh (tar streams) needs a public IP: community hosts must offer one; secure always does
+            "ports": _tcp_ports(job.ports), **({"supportPublicIp": True} if cloud == "COMMUNITY" else {}),
+            **_cuda(job.placement.min_cuda, s["cuda_versions"]),
+            "env": {"PUBLIC_KEY": pubkey, **job.env},
+        }
+        if job.volume:  # the volume pins the data centre, which already fixes the country
+            vol = next((v for v in api.volumes() if v.get("name") == job.volume), None)
+            if not vol:
+                raise SystemExit(f"network volume {job.volume!r} not found")
+            spec.update(networkVolumeId=vol["id"], volumeMountPath="/workspace", dataCenterIds=[vol["dataCenterId"]])
+        elif job.regions:
+            spec["countryCodes"] = [r.upper() for r in job.regions]
+        try:
+            pod = api.create_pod(spec)
+        except ProviderError as e:
+            msg = str(e).lower()
+            if "no instances" in msg or "not available" in msg:
+                raise Unavailable(str(e)) from None
+            if "http 400" in msg and "gputypeids" in msg:
+                # GraphQL lists types (MIG slices, new SKUs) that POST /pods doesn't accept yet
+                raise Unavailable(f"Runpod won't create {offer.gpu} through its REST API") from None
+            raise
+        return {"id": pod["id"], "usd_h": pod.get("costPerHr") or offer.usd_h,
+                "gpu": (pod.get("machine") or {}).get("gpuTypeId") or offer.gpu}
+
+    def wait_ready(self, pod_id, job, timeout_s=900):
+        api = self.api()
+        t0 = time.time()
+        while time.time() - t0 < timeout_s:
+            p = api.get_pod(pod_id)
+            mapped = p.get("portMappings") or {}
+            ip = p.get("publicIp")
+            if ip and mapped.get("22"):
+                ports = {k: {"host": ip, "port": v} for k, v in mapped.items() if k != "22"}
+                return Remote(ip, mapped["22"]), {"machine": p.get("machineId"),
+                                                  "gpu": (p.get("machine") or {}).get("gpuTypeId"), "ports": ports}
+            time.sleep(5)
+        raise BadHost(f"pod {pod_id} had no ssh endpoint after {timeout_s}s")
+
+    def terminate(self, pod_id):
+        self.api().terminate_pod(pod_id)
+
+    def list_ours(self):
+        out = []
+        for p in self.api().list_pods():
+            if (p.get("name") or "").startswith("gipper-"):
+                started = p.get("lastStartedAt")
+                try:
+                    started = datetime.fromisoformat(started.replace("Z", "+00:00")).timestamp() if started else None
+                except ValueError:
+                    started = None
+                out.append({"id": p["id"], "name": p["name"], "started": started,
+                            "status": p.get("desiredStatus"), "usd_h": p.get("costPerHr")})
+        return out
+
+    def billed(self, since):
+        """{pod id: (usd, detail)} Runpod billed since `since` (epoch s), per pod per day."""
+        start = datetime.fromtimestamp(since, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        rows = self.api()._req("GET", f"{REST}/billing/pods?bucketSize=day&grouping=podId&startTime={start}")
+        out = {}
+        for r in rows if isinstance(rows, list) else []:
+            usd, detail = out.get(r["podId"], (0.0, {}))
+            detail["billed_s"] = round(detail.get("billed_s", 0) + (r.get("timeBilledMs") or 0) / 1000)
+            detail["disk_gb"] = r.get("diskSpaceBilledGB")
+            out[r["podId"]] = (usd + (r.get("amount") or 0), detail)
+        return out
+
+    def account(self):
+        a = self.api().account()
+        return f"balance ${a['clientBalance']:.2f}; spending ${a['currentSpendPerHr']:.2f}/h now; spend limit {a['spendLimit']}"
+
+
+PROVIDER = Runpod()
+
+
+def _cuda(min_cuda, configured):
+    """allowedCudaVersions: the configured list, or every version Runpod knows at or
+    above the job's min_cuda; nothing (any host) if neither is set."""
+    versions = list(configured or CUDA_VERSIONS)
+    if min_cuda:
+        versions = [v for v in versions if tuple(map(int, v.split("."))) >= tuple(map(int, f"{min_cuda}".split(".")[:2]))]
+    if (configured or min_cuda) and not versions:   # an empty list would mean "any" to Runpod
+        raise Unavailable(f"no Runpod CUDA version satisfies min_cuda {min_cuda}")
+    return {"allowedCudaVersions": versions} if (configured or min_cuda) else {}

gipper/providers/shadeform.py

@@ -0,0 +1,143 @@
+"""Shadeform: one account and one bill across 30+ clouds (Lambda, Hyperstack,
+Massed Compute, Crusoe, ...). Instances are whole VMs with their own public IP,
+reached by ssh as `shadeform` (not root); drivers come with the "shade_os" images.
+Ports, UDP included, are opened with UFW rules at boot. The underlying cloud may
+have its own firewall too, so test a UDP job's ports before relying on them.
+
+Each instance is also created with Shadeform's own auto_delete (the job's deadline
+and a spend ceiling), a second guard behind gipper's teardown and reaper.
+
+API: https://docs.shadeform.ai/api-reference (the catalogue is public).
+"""
+import time
+from datetime import datetime, timedelta, timezone
+
+from gipper.http import ProviderError, request
+from gipper.market import Offer
+from gipper.providers import BadHost, Provider, Unavailable
+from gipper.ssh import Remote
+
+BASE = "https://api.shadeform.ai/v1"
+ERROR = ("error", "failed", "deleted", "deleting")
+
+
+def _country(display_name):
+    """Availability display names lead with the country: "US, Des Moines" -> "US"."""
+    head = (display_name or "").split(",", 1)[0].strip().upper()
+    return head if len(head) == 2 else None
+
+
+def offers_from(types):
+    """One offer per GPU, cloud, region and price: a cloud often lists the same card
+    under several instance types ("A6000", "A6000_plus", "A6000_base")."""
+    out, seen = [], set()
+    for t in types or []:
+        conf = t.get("configuration") or {}
+        if (t.get("num_gpus") or conf.get("num_gpus")) != 1:
+            continue
+        for a in t.get("availability") or []:
+            if not a.get("available") or a.get("rental_type", "on_demand") != "on_demand":
+                continue
+            key = (t["cloud"], a["region"], t.get("gpu_type"), t["hourly_price"])
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(Offer("shadeform", t.get("gpu_type") or t["shade_instance_type"],
+                             float(conf.get("vram_per_gpu_in_gb") or 0), t["hourly_price"] / 100,
+                             where=f"{t['cloud']} {a['region']}", country=_country(a.get("display_name")),
+                             stock="yes", udp=True,
+                             ref={"cloud": t["cloud"], "region": a["region"], "type": t["shade_instance_type"],
+                                  "os": next((o for o in conf.get("os_options") or [] if "cuda" in o), None),
+                                  "boot_s": (t.get("boot_time") or {}).get("max_boot_in_sec")}))
+    return out
+
+
+def _ufw(ports):
+    rules = [{"port": "22", "proto": "tcp"}]
+    for p in ports:
+        num, _, proto = p.partition("/")
+        rules.append({"port": num.replace("-", ":"), "proto": (proto or "tcp").lower()})
+    return rules
+
+
+class Shadeform(Provider):
+    name = "shadeform"
+    key_name = "SHADEFORM_API_KEY"
+    udp = True
+
+    def _req(self, method, path, body=None, params=None, auth=True):
+        headers = {"X-API-KEY": self.key()} if auth else {}
+        return request(method, f"{BASE}{path}", body, headers=headers, params=params)
+
+    def offers(self, job):
+        d = self._req("GET", "/instances/types", params={"num_gpus": 1, "available": "true", "sort": "price"}, auth=False)
+        return offers_from(d.get("instance_types"))
+
+    def _key_id(self, pubkey):
+        body = pubkey.split()[1]
+        for k in self._req("GET", "/sshkeys").get("ssh_keys") or []:
+            if body in (k.get("public_key") or ""):
+                return k["id"]
+        return self._req("POST", "/sshkeys/add", {"name": "gipper", "public_key": pubkey})["id"]
+
+    def create(self, offer, job, pubkey, name):
+        deadline = datetime.now(timezone.utc) + timedelta(hours=job.max_hours, minutes=30)
+        spec = {"cloud": offer.ref["cloud"], "region": offer.ref["region"], "shade_instance_type": offer.ref["type"],
+                "shade_cloud": True, "name": name[:50], "ssh_key_id": self._key_id(pubkey), "tags": ["gipper"],
+                "auto_delete": {"date_threshold": deadline.isoformat(timespec="seconds").replace("+00:00", "Z"),
+                                "spend_threshold": f"{max(job.max_usd * 1.5, 1):.2f}"}}
+        if offer.ref.get("os"):
+            spec["os"] = offer.ref["os"]
+        if job.ports:
+            spec["networking"] = {"ufw_rules": _ufw(job.ports)}
+        try:
+            d = self._req("POST", "/instances/create", spec)
+        except ProviderError as e:
+            if any(w in str(e).lower() for w in ("capacity", "not available", "unavailable", "out of stock")):
+                raise Unavailable(str(e)) from None
+            raise
+        return {"id": d["id"], "usd_h": offer.usd_h, "gpu": offer.gpu}
+
+    def wait_ready(self, iid, job, timeout_s=1200):   # VMs boot slower than containers (5-8 min listed)
+        t0 = time.time()
+        while time.time() - t0 < timeout_s:
+            i = self._req("GET", f"/instances/{iid}/info")
+            status = (i.get("status") or "").lower()
+            if status in ERROR or "error" in status:
+                raise BadHost(f"instance {iid} is {status}: {i.get('status_details') or ''}".strip())
+            if status == "active" and i.get("ip"):
+                # a whole VM: the job's ports are its own, unless the cloud forwards them
+                fwd = {f["internal_port"]: f["external_port"] for f in i.get("port_forwards") or []}
+                ports = {}
+                for p in job.ports:
+                    num, _, proto = p.partition("/")
+                    if num.isdigit():
+                        ports[f"{num}/{(proto or 'tcp').lower()}"] = {"host": i["ip"], "port": fwd.get(int(num), int(num))}
+                return Remote(i["ip"], i.get("ssh_port") or fwd.get(22, 22), user=i.get("ssh_user") or "shadeform"), {
+                    "machine": i.get("cloud_assigned_id"), "gpu": i.get("shade_instance_type"), "ports": ports,
+                    "where": f"{i.get('cloud')} {i.get('region')}"}
+            time.sleep(10)
+        raise BadHost(f"instance {iid} not active after {timeout_s}s")
+
+    def terminate(self, iid):
+        self._req("POST", f"/instances/{iid}/delete")
+
+    def list_ours(self):
+        out = []
+        for i in self._req("GET", "/instances").get("instances") or []:
+            if (i.get("name") or "").startswith("gipper-") and i.get("status") not in ("deleting", "deleted"):
+                started = i.get("created_at")
+                try:
+                    started = datetime.fromisoformat(started.replace("Z", "+00:00")).timestamp() if started else None
+                except ValueError:
+                    started = None
+                out.append({"id": i["id"], "name": i["name"], "started": started, "status": i.get("status"),
+                            "usd_h": (i.get("hourly_price") or 0) / 100})
+        return out
+
+    def account(self):
+        n = len(self.list_ours())
+        return f"key works; {n} gipper instance(s) running (the API reports no balance)"
+
+
+PROVIDER = Shadeform()

gipper/providers/vast.py

@@ -0,0 +1,170 @@
+"""Vast.ai: a marketplace of individual machines (many consumer GPUs). Each offer
+is one machine ("ask"); renting it makes a Docker container with direct ssh as
+root. Ports, UDP included, are published with docker `-p` options and land on a
+random external port of the host's (usually shared) public IP; gipper hands the
+mapping to the job as GIPPER_PORT_<n>_<PROTO>.
+
+API: https://docs.vast.ai/api-reference (search is public; renting needs the key).
+"""
+import time
+
+from gipper.config import settings
+from gipper.http import ProviderError, request
+from gipper.market import Offer
+from gipper.providers import BadHost, Provider, Unavailable
+from gipper.ssh import Remote
+
+BASE = "https://console.vast.ai/api"
+MIN_RELIABILITY = 0.97   # Vast's own uptime score for the machine
+MIN_INET_DOWN = 100      # Mb/s: inputs and model weights have to get there
+PER_GPU = 8              # machines per GPU model: enough variety in speed and track record
+DEAD = ("exited", "unknown", "offline")   # never reaches running from these (Vast's docs)
+
+
+def _port_opts(ports):
+    """["8082/udp", "8080"] -> {"-p 8082:8082/udp": "1", "-p 8080:8080": "1"}"""
+    out = {}
+    for p in ports:
+        num, _, proto = p.partition("/")
+        proto = (proto or "tcp").lower()
+        out[f"-p {num}:{num}" + ("/udp" if proto == "udp" else "")] = "1"
+    return out
+
+
+def offers_from(rows, per_gpu=PER_GPU):
+    """Offers from Vast search rows. Memory is decimal GB from Vast's MiB (an A40's
+    46068 MiB is 48.3 GB, a 4090's 24564 MiB is 25.8 GB), so nominal sizes qualify.
+    Speeds are Mb/s and fees $/GB, as Vast advertises them."""
+    out, per = [], {}
+    for o in rows:
+        name = o.get("gpu_name") or "?"
+        if per.get(name, 0) >= per_gpu:
+            continue
+        per[name] = per.get(name, 0) + 1
+        geo = o.get("geolocation") or ""
+        cap = o.get("compute_cap")
+        out.append(Offer("vast", name, (o.get("gpu_ram") or 0) * 1024**2 / 1e9, float(o["dph_total"]), where=geo,
+                         country=geo.rsplit(",", 1)[-1].strip().upper() or None, stock="yes", udp=True,
+                         ref={"ask": o["id"], "machine": str(o["machine_id"]) if o.get("machine_id") else None},
+                         metrics={"host_download_mbps": o.get("inet_down"), "host_upload_mbps": o.get("inet_up"),
+                                  "host_download_usd_per_gb": o.get("inet_down_cost"),
+                                  "host_upload_usd_per_gb": o.get("inet_up_cost"),
+                                  "cuda_max_good": o.get("cuda_max_good"),
+                                  "compute_capability": cap / 100 if cap and cap >= 100 else cap,
+                                  "dlperf": o.get("dlperf"),
+                                  "public_ip": o.get("public_ipaddr"),   # pingable before renting
+                                  "reliability": o.get("reliability")}))
+    return out
+
+
+class Vast(Provider):
+    name = "vast"
+    key_name = "VAST_API_KEY"
+    udp = True
+
+    def _req(self, method, path, body=None, params=None, auth=True):
+        headers = {"Authorization": f"Bearer {self.key()}"} if auth else {}
+        return request(method, f"{BASE}{path}", body, headers=headers, params=params)
+
+    def offers(self, job):
+        p = job.placement
+        q = {"verified": {"eq": True}, "external": {"eq": False}, "rentable": {"eq": True}, "rented": {"eq": False},
+             "num_gpus": {"eq": 1}, "gpu_ram": {"gte": job.min_mem_gb * 1e9 / 1024**2},   # MiB in the API
+             "reliability": {"gte": MIN_RELIABILITY},
+             "inet_down": {"gte": max(MIN_INET_DOWN, p.min_host_download_mbps or 0)},
+             "direct_port_count": {"gte": 1 + len(job.ports)},   # ssh plus the job's own
+             "type": "ondemand", "order": [["dph_total", "asc"]], "limit": 200,
+             "allocated_storage": job.container_disk_gb}   # so dph_total includes the disk
+        for key, floor in (("inet_up", p.min_host_upload_mbps), ("cuda_max_good", p.min_cuda), ("dlperf", p.min_dlperf),
+                           ("compute_cap", p.min_compute_capability * 100 if p.min_compute_capability else None)):
+            if floor:
+                q[key] = {"gte": floor}
+        if job.regions:
+            q["geolocation"] = {"in": [r.upper() for r in job.regions]}
+        return offers_from(self._req("POST", "/v0/bundles/", q, auth=self.has_key()).get("offers", []))
+
+    def _ensure_key(self, pubkey):
+        """Vast adds account keys to new instances; register gipper's once."""
+        d = self._req("GET", "/v0/ssh/")
+        keys = d if isinstance(d, list) else d.get("ssh_keys", d.get("keys", []))
+        body = pubkey.split()[1]
+        if not any(body in (k.get("public_key") or k.get("ssh_key") or "") for k in keys if isinstance(k, dict)):
+            self._req("POST", "/v0/ssh/", {"ssh_key": pubkey})
+
+    def create(self, offer, job, pubkey, name):
+        self._ensure_key(pubkey)
+        spec = {"client_id": "me", "image": job.image or settings()["vast_image"], "disk": job.container_disk_gb,
+                "runtype": "ssh_direct", "label": name, "env": _port_opts(job.ports), "cancel_unavail": True}
+        try:
+            d = self._req("PUT", f"/v0/asks/{offer.ref['ask']}/", spec)
+        except ProviderError as e:
+            if any(w in str(e).lower() for w in ("no_such_ask", "not available", "unavailable", "already rented", "http 404")):
+                raise Unavailable(str(e)) from None
+            raise
+        if not d.get("success") or not d.get("new_contract"):
+            raise Unavailable(f"vast would not rent ask {offer.ref['ask']}: {d.get('msg') or d.get('error') or d}")
+        return {"id": str(d["new_contract"]), "usd_h": offer.usd_h, "gpu": offer.gpu}
+
+    def get(self, iid):
+        d = self._req("GET", f"/v0/instances/{iid}/")
+        inst = d.get("instances", d)
+        return inst[0] if isinstance(inst, list) else (inst or {})
+
+    def wait_ready(self, iid, job, timeout_s=900):
+        t0 = time.time()
+        while time.time() - t0 < timeout_s:
+            i = self.get(iid)
+            status = i.get("actual_status")
+            if status in DEAD:
+                raise BadHost(f"instance {iid} is {status}: {i.get('status_msg') or ''}".strip())
+            mapped = i.get("ports") or {}
+            ip = i.get("public_ipaddr")
+            ssh = mapped.get("22/tcp") if isinstance(mapped, dict) else None
+            if status == "running" and ip and ssh:
+                ports = {k: {"host": ip, "port": int(v[0]["HostPort"])}
+                         for k, v in mapped.items() if k != "22/tcp" and v}
+                return Remote(ip.strip(), int(ssh[0]["HostPort"])), {
+                    "machine": i.get("machine_id"), "gpu": i.get("gpu_name"), "ports": ports,
+                    "where": i.get("geolocation")}
+            time.sleep(5)
+        raise BadHost(f"instance {iid} had no ssh endpoint after {timeout_s}s")
+
+    def terminate(self, iid):
+        self._req("DELETE", f"/v0/instances/{iid}/", {})
+
+    def list_ours(self):
+        out, params = [], {"select_filters": {}, "order_by": [{"col": "id", "dir": "asc"}], "limit": 25}
+        while True:
+            d = self._req("GET", "/v1/instances/", params=params)
+            for i in d.get("instances") or []:
+                if (i.get("label") or "").startswith("gipper-"):
+                    out.append({"id": str(i["id"]), "name": i["label"], "started": i.get("start_date"),
+                                "status": i.get("actual_status"), "usd_h": i.get("dph_total")})
+            if not d.get("next_token"):
+                return out
+            params["after_token"] = d["next_token"]
+
+    def billed(self, since):
+        """{instance id: (usd, detail)} Vast charged since `since` (epoch s), from its
+        itemised charges: GPU while running, disk from creation, bandwidth by the byte."""
+        out, params = {}, {"select_filters": {"day": {"gte": int(since), "lte": int(time.time())}}, "limit": 20}
+        while True:
+            d = self._req("GET", "/v0/charges/", params=params)
+            for r in d.get("results") or []:
+                if r.get("type") != "instance" or not str(r.get("source", "")).startswith("instance-"):
+                    continue
+                iid = r["source"].split("-", 1)[1]
+                usd, detail = out.get(iid, (0.0, {}))
+                for item in r.get("items") or []:
+                    detail[item["type"]] = round(detail.get(item["type"], 0) + item.get("amount", 0), 6)
+                out[iid] = (usd + r.get("amount", 0), detail)
+            if not d.get("next_token"):
+                return out
+            params["after_token"] = d["next_token"]
+
+    def account(self):
+        u = self._req("GET", "/v0/users/current/")
+        return f"credit ${float(u.get('credit') or 0):.2f}; balance ${float(u.get('balance') or 0):.2f}"
+
+
+PROVIDER = Vast()

gipper/reap.py

@@ -0,0 +1,95 @@
+"""Find and end rented GPUs gipper started and forgot, on every provider with a key.
+
+A machine is reaped when its name starts with `gipper-` and either the ledger has
+no live run for it (created but never terminated by a running client) past a grace
+period, or it is past the deadline recorded at creation. Machines not named
+`gipper-` are never touched.
+"""
+import time
+from datetime import datetime
+
+from gipper import ledger
+from gipper.config import settings
+from gipper.providers import load
+
+GRACE_S = 20 * 60  # a client may still be uploading or waiting for ssh
+
+
+def state():
+    """(provider, id) -> {job, created, deadline, terminated}; receipts from before
+    there was more than one provider are Runpod's."""
+    known = {}
+    for r in ledger.read(kinds={"pod.create", "pod.terminate", "pod.closure_verified", "pod.termination_pending", "reap"}):
+        pid = r["payload"].get("pod")
+        if not pid:
+            continue
+        k = (r["payload"].get("provider") or "runpod", str(pid))
+        p = known.setdefault(k, {"job": r["job"], "terminated": False})
+        if r["kind"] == "pod.create":
+            p["created"] = r["ts"]
+            p["deadline"] = r["payload"].get("deadline")
+            p["usd_h"] = r["payload"].get("usd_per_h")
+            try:
+                p["started"] = datetime.fromisoformat(r["ts"]).timestamp()
+            except (ValueError, TypeError):
+                p["started"] = None
+        elif r["kind"] == "pod.termination_pending":
+            p["pending"] = True
+            if "delete_accepted" in r["payload"]:
+                p["delete_accepted"] = r["payload"]["delete_accepted"]
+            p["terminated"] = False
+        else:
+            p["terminated"] = True
+            p["pending"] = False
+    return known
+
+
+def verdict(rec, machine, now):
+    if rec is None:
+        started = machine.get("started")
+        return None if started and now - started < GRACE_S else "not in the ledger"
+    if rec.get("pending"):
+        return "termination pending; closure unverified"
+    if rec["terminated"]:
+        return "ledger says terminated but it is still listed"
+    if rec.get("deadline") and now > rec["deadline"]:
+        return "past its deadline"
+    return None
+
+
+def reap(dry_run=False, providers=None):
+    """[(provider, id, name, why)] acted on (or that would be, with dry_run)."""
+    known = state()
+    acted = []
+    now = time.time()
+    for p in providers or [load(n) for n in settings()["providers"]]:
+        if not p.has_key():
+            continue
+        try:
+            machines = p.list_ours()
+        except Exception as e:  # one provider being down must not stop the others' sweep
+            print(f"{p.name}: could not list machines: {e}")
+            continue
+        # An accepted but asynchronous DELETE can disappear between sweeps.
+        # Keep its ledger claim until a new positive provider read confirms it.
+        listed = {str(m["id"]) for m in machines}
+        machines = list(machines) + [{"id": iid, "name": "gipper-pending-recovery"}
+                   for (provider, iid), rec in known.items()
+                   if provider == p.name and rec.get("pending") and rec.get("delete_accepted") and iid not in listed]
+        for m in machines:
+            why = verdict(known.get((p.name, str(m["id"]))), m, now)
+            if not why:
+                continue
+            acted.append((p.name, m["id"], m["name"], why))
+            if not dry_run:
+                from gipper.lifecycle import terminate
+                rec = known.get((p.name, str(m["id"]))) or {}
+                try:
+                    closed = terminate(p, m["id"], job_id=rec.get("job"), started=rec.get("started"),
+                                       usd_h=rec.get("usd_h"), delete_accepted=bool(rec.get("delete_accepted")))
+                except Exception as error:
+                    print(f"{p.name}: cleanup pending for {m['id']}: {error}")
+                    continue
+                if closed:
+                    ledger.write("reap", {"provider": p.name, "pod": m["id"], "name": m["name"], "why": why}, job=rec.get("job"))
+    return acted