gipper 0.3.0__py3-none-any.whl

gipper/history.py

@@ -0,0 +1,232 @@
+"""What gipper has learned from its own rentals, read from the ledger (nothing else
+is stored): how often a provider's or a machine's hosts turn out bad, how long they
+take to be ready, how fast transfers really go from here, how long a workload's
+compute takes per GPU, and how a provider's bill compares with the rate x lifetime
+gipper computes.
+
+A rental (one machine) is *good* once its job started on it, and *bad* if the
+host was broken: a busy GPU, CUDA that won't initialise, ssh that never came up,
+a failed probe, or a connection lost before or during the job. A host rejected
+because it doesn't suit this job (`run.Unsuitable`: compute capability, a
+client's runtime check) is neither. Nor are errors on this side (a local file
+that won't upload), errors before any machine existed (no stock), or a job that
+fails on a working host.
+
+Compute time is learnt only from jobs that succeeded; transfer speed only from
+real transfers of at least MIN_TRANSFER_BYTES. Small probes mostly measure ssh
+start-up, so they're never used for speed.
+
+Rates are smoothed so one bad night doesn't condemn a provider and one good run
+doesn't make a stranger's machine trusted: a machine starts from its provider's
+rate, a provider from PRIOR_FAIL.
+"""
+import re
+import statistics
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from gipper import ledger
+from gipper.config import settings
+
+PRIOR_FAIL = 0.10      # a provider gipper has never used
+PROVIDER_WEIGHT = 4    # rentals' worth of belief in the prior
+MACHINE_WEIGHT = 2     # rentals' worth of belief in the provider's rate, for one machine
+MIN_TRANSFER_BYTES = 20e6   # smaller transfers say more about ssh start-up than bandwidth
+MIN_BILLED_S = 600          # shorter rentals mostly measure a provider's rounding, not its rate
+LOST = re.compile(r"ssh|connection|timed out|timeout|broken pipe|reset by peer|no route|unreachable", re.I)
+# host.reject reasons written before rejects carried a "kind": the host's fault or not
+BROKEN = re.compile(r"busy|nvidia-smi|cuinit|cuda initiali|device enumeration|libcuda|ssh|never came up|no ssh endpoint|"
+                    r"not active|is (exited|unknown|offline)|probe|below configured minimum|timeout|connection", re.I)
+
+
+@dataclass
+class Rental:
+    provider: str
+    pod: str
+    job: str | None
+    ts: float
+    machine: str | None = None
+    gpu: str | None = None
+    country: str | None = None
+    ready_s: float | None = None
+    started: float | None = None        # run.start time
+    ended: float | None = None          # job exit (or collect, for older receipts)
+    compute_s: float | None = None      # only from a job that succeeded
+    succeeded: bool = False
+    outcome: str | None = None          # good | bad
+    why: str | None = None
+    up_mbps: list = field(default_factory=list)       # real transfers (inputs up, outputs down)
+    down_mbps: list = field(default_factory=list)
+    computed_usd: float = 0.0
+    computed_s: float = 0.0
+    billed_usd: float | None = None
+    cleanup_pending: bool = False
+
+
+def _ts(r):
+    return datetime.fromisoformat(r["ts"]).timestamp()
+
+
+def _median(xs):
+    xs = [x for x in xs if x is not None]
+    return statistics.median(xs) if xs else None
+
+
+class History:
+    def __init__(self, rentals, workloads, rtts=()):
+        self.rentals = rentals            # [Rental]
+        self.workloads = workloads        # job id -> workload
+        self.rtts = list(rtts)            # [(provider, machine, country, origin, ms)]
+        self._by = defaultdict(list)
+        for r in rentals:
+            self._by[r.provider].append(r)
+            if r.machine:
+                self._by[(r.provider, r.machine)].append(r)
+
+    @classmethod
+    def load(cls, days=None, records=None):
+        days = settings()["history_days"] if days is None else days
+        recs = records if records is not None else ledger.read(since=time.time() - days * 86400)
+        rentals, workloads, current, billed, rtts = {}, {}, {}, {}, []
+        for r in recs:
+            p, kind, job = r["payload"], r["kind"], r.get("job")
+            if (r.get("refs") or {}).get("workload") and job:
+                workloads[job] = r["refs"]["workload"]
+            key = (p.get("provider") or "runpod", str(p.get("pod"))) if p.get("pod") is not None else None
+            if kind == "pod.create" and key:
+                rentals[key] = Rental(key[0], key[1], job, _ts(r), machine=_s(p.get("machine")), gpu=p.get("gpu"),
+                                      country=p.get("country"))
+                current[job] = key
+                continue
+            if kind == "market.rtt":
+                rtts += [(x.get("provider"), _s(x.get("machine")), x.get("country"), p.get("origin"), x["ms"])
+                         for x in p.get("samples") or [] if x.get("ms") is not None]
+                continue
+            if kind == "error" and job in current:
+                rent, err = rentals.get(current[job]), p.get("error", "")
+                if rent and not rent.ended and LOST.search(err) and (rent.outcome is None or rent.started):
+                    rent.outcome, rent.why = "bad", ("lost: " if rent.started else "") + err[:200]
+                continue
+            if kind == "run.end" and job in current:
+                rent = rentals.get(current[job])
+                if rent and rent.started:
+                    rent.ended = rent.ended or _ts(r)
+                    rent.succeeded = p.get("outcome") == "succeeded"
+                    rent.compute_s = p.get("compute_s") or rent.compute_s
+                continue
+            if kind == "spend.billed" and key:
+                billed[key] = p.get("usd", 0.0)   # the latest bill for the machine counts
+            rent = rentals.get(key) if key else None
+            if rent is None:
+                continue
+            if kind == "pod.ready":
+                rent.ready_s = p.get("after_s")
+                rent.machine = rent.machine or _s(p.get("machine"))
+                rent.gpu = p.get("gpu") or rent.gpu
+            elif kind == "run.start":
+                rent.started, rent.outcome = _ts(r), "good"
+            elif kind == "host.reject":
+                why = p.get("why", "")
+                broken = p.get("kind") == "broken" or (p.get("kind") is None and BROKEN.search(why))
+                rent.outcome, rent.why = ("bad" if broken else None), why[:200]
+                rent.machine = rent.machine or _s(p.get("machine"))
+            elif kind == "transfer" and p.get("seconds") and p.get("bytes", 0) >= MIN_TRANSFER_BYTES:
+                mbps = p["bytes"] * 8 / 1e6 / p["seconds"]
+                (rent.up_mbps if p.get("direction") == "up" else rent.down_mbps).append(mbps)
+            elif kind == "collect" and rent.started and not rent.ended:
+                rent.ended = _ts(r)    # older receipts: compute ~ run.start .. collect
+            elif kind == "spend":
+                rent.computed_usd += p.get("usd", 0.0)
+                rent.computed_s += p.get("seconds", 0)
+            elif kind == "pod.termination_pending":
+                rent.cleanup_pending = True
+            elif kind == "pod.closure_verified":
+                rent.cleanup_pending = False
+            elif kind == "host.rtt" and p.get("ms") is not None:
+                rtts.append((rent.provider, rent.machine or _s(p.get("machine")), rent.country or p.get("country"),
+                             p.get("origin"), p["ms"]))
+        for key, usd in billed.items():
+            if key in rentals:
+                rentals[key].billed_usd = usd
+        for rent in rentals.values():
+            if not rent.succeeded:
+                rent.compute_s = None   # a capped, failed or interrupted run says nothing about how long it takes
+            elif rent.compute_s is None and rent.started and rent.ended:
+                rent.compute_s = rent.ended - rent.started
+        return cls(list(rentals.values()), workloads, rtts)
+
+    # ---------------------------------------------------------------- what it knows
+    def rentals_of(self, provider, machine=None):
+        return self._by[(provider, str(machine))] if machine is not None else self._by[provider]
+
+    def failure(self, provider, machine=None):
+        """Chance a fresh rental turns out bad: the machine's record if it has one,
+        pulled towards the provider's, pulled towards PRIOR_FAIL."""
+        rs = [r for r in self.rentals_of(provider) if r.outcome]
+        bad = sum(r.outcome == "bad" for r in rs)
+        p = (bad + PRIOR_FAIL * PROVIDER_WEIGHT) / (len(rs) + PROVIDER_WEIGHT)
+        if machine is None:
+            return p
+        ms = [r for r in self.rentals_of(provider, machine) if r.outcome]
+        return (sum(r.outcome == "bad" for r in ms) + p * MACHINE_WEIGHT) / (len(ms) + MACHINE_WEIGHT)
+
+    def ready_s(self, provider, machine=None):
+        return (_median([r.ready_s for r in self.rentals_of(provider, machine)]) if machine is not None else None) \
+            or _median([r.ready_s for r in self.rentals_of(provider)])
+
+    def mbps(self, provider, machine, direction, country=None):
+        """Measured transfer speed from here (real transfers): this machine, then the
+        provider's machines in the same country (far hosts upload slower), then the
+        provider; None if gipper hasn't timed one."""
+        pick = (lambda r: r.up_mbps) if direction == "up" else (lambda r: r.down_mbps)
+        scopes = ((self.rentals_of(provider, machine),) if machine is not None else ()) + \
+                 (([r for r in self.rentals_of(provider) if r.country == country],) if country else ()) + \
+                 (self.rentals_of(provider),)
+        for rs in scopes:
+            v = _median([x for r in rs for x in pick(r)])
+            if v:
+                return v
+        return None
+
+    def rtt(self, provider, machine, origin):
+        """Median round trip from `origin` to this machine, from pings and rentals."""
+        if machine is None:
+            return None
+        return _median([ms for p, m, c, o, ms in self.rtts if p == provider and m == str(machine) and o == origin])
+
+    def country_rtt(self, country, origin):
+        """Median round trip from `origin` to any host in `country`."""
+        if not country:
+            return None
+        return _median([ms for p, m, c, o, ms in self.rtts if c == country and o == origin])
+
+    def compute_s(self, workload, gpu):
+        """Median compute seconds this workload took on this GPU before."""
+        from gipper.market import matches
+        return _median([r.compute_s for r in self.rentals if r.compute_s and r.gpu and matches(gpu, r.gpu)
+                        and self.workloads.get(r.job) == workload])
+
+    def billing_factor(self, provider):
+        """Billed / computed over rentals of MIN_BILLED_S or more, as one ratio of
+        totals: how much a provider's bill exceeds rate x lifetime for real work.
+        (Short rentals mostly measure rounding: a 47 s Vast run billed 5x.)"""
+        rs = [r for r in self.rentals_of(provider)
+              if r.billed_usd is not None and r.computed_usd > 0 and r.computed_s >= MIN_BILLED_S]
+        computed = sum(r.computed_usd for r in rs)
+        return sum(r.billed_usd for r in rs) / computed if computed > 0.05 else None
+
+    def summary(self, provider):
+        rs = self.rentals_of(provider)
+        judged = [r for r in rs if r.outcome]
+        return {"rentals": len(rs), "good": sum(r.outcome == "good" for r in judged),
+                "bad": sum(r.outcome == "bad" for r in judged), "failure": self.failure(provider),
+                "ready_s": self.ready_s(provider), "up_mbps": self.mbps(provider, None, "up"),
+                "down_mbps": self.mbps(provider, None, "down"), "billing_factor": self.billing_factor(provider),
+                "cleanup_pending": sum(r.cleanup_pending for r in rs),
+                "reasons": [r.why for r in judged if r.outcome == "bad"][-5:]}
+
+
+def _s(v):
+    return str(v) if v is not None else None

gipper/http.py

@@ -0,0 +1,40 @@
+"""One small JSON-over-HTTPS client for every provider: stdlib only, retries on
+throttling and 5xx, and the key only ever in a header and scrubbed from errors."""
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from gipper.config import redact
+
+# Cloudflare in front of Runpod's API rejects Python's default agent (error 1010)
+AGENT = "gipper/0.1 (+https://github.com/lavallee/gipper)"
+
+
+class ProviderError(RuntimeError):
+    pass
+
+
+def request(method, url, body=None, headers=None, params=None, retries=3, timeout=60):
+    if params:
+        url += "?" + urllib.parse.urlencode({k: json.dumps(v) if isinstance(v, (dict, list)) else v for k, v in params.items()})
+    data = json.dumps(body).encode() if body is not None else None
+    hdrs = {"Content-Type": "application/json", "Accept": "application/json", "User-Agent": AGENT, **(headers or {})}
+    for attempt in range(retries):
+        req = urllib.request.Request(url, data=data, method=method, headers=hdrs)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                raw = r.read()
+                return json.loads(raw) if raw.strip() else {}
+        except urllib.error.HTTPError as e:
+            msg = redact(e.read().decode(errors="replace")[:500])
+            if e.code in (429, 500, 502, 503, 504) and attempt < retries - 1:
+                time.sleep(2 * (attempt + 1))
+                continue
+            raise ProviderError(f"{method} {url.split('?')[0]}: HTTP {e.code}: {msg}") from None
+        except (urllib.error.URLError, TimeoutError) as e:
+            if attempt < retries - 1:
+                time.sleep(2 * (attempt + 1))
+                continue
+            raise ProviderError(f"{method} {url.split('?')[0]}: {getattr(e, 'reason', e)}") from None

gipper/job.py

@@ -0,0 +1,153 @@
+"""The job spec: one TOML file, shared by every executor (rented GPUs from Runpod,
+Vast and Shadeform now; owned machines planned).
+
+    name = "batch-inference"
+    workload = "example-workload"          # what this is, for throughput history
+    units = { name = "items", count = 200 }
+    sensitivity = "cloud-ok"              # "owned-only" refuses rented GPUs
+
+    [gpu]
+    min_mem_gb = 16
+    prefer = ["RTX A5000", "RTX 4090"]     # optional order; names match across providers
+    regions = ["US", "CA"]                 # ISO country codes it may run in; default anywhere
+    providers = ["vast", "runpod"]         # markets to shop; default all with a key
+    cloud = "COMMUNITY"                    # Runpod: COMMUNITY (then SECURE) or SECURE
+    max_usd_per_h = 0.6                    # optional hourly ceiling; no ceiling by default
+
+    [pod]                                  # optional overrides
+    image = "runpod/base:1.4.1-cuda1300-ubuntu2404"   # default: each provider's own
+    container_disk_gb = 40
+    volume = "gipper-models"               # Runpod network volume name (pins the data centre)
+    ports = ["5004/udp", "8080/tcp"]       # published; the job sees GIPPER_PUBLIC_IP and
+                                           # GIPPER_PORT_5004_UDP (the external port). UDP
+                                           # rules out Runpod.
+
+    [env]                                  # non-secret environment for the job
+    PYTHONUNBUFFERED = "1"
+
+    [[inputs]]                             # local -> pod, under /root/job/<dst>
+    src = "~/gpu-project"
+    dst = "project"
+    exclude = ["runs", ".git", "__pycache__"]
+
+    [[outputs]]                            # pod /root/job/<src> -> local <dst>
+    src = "out"
+    dst = "~/gpu-project/runs/x/out"
+
+    [run]
+    setup = "bash project/setup.sh"
+    command = "python project/main.py"
+    checkpoint = { src = "out", every_s = 300 }   # synced down while running
+
+    [budget]
+    max_usd = 1.0
+    max_hours = 2.0
+
+    [placement]                            # optional: how to judge offers on the whole run
+    compute_s = 900                        # (all fields in gipper/placement.py)
+    setup_download_gb = 5
+"""
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from gipper.config import settings
+from gipper.placement import Placement, load_placement
+
+
+class JobError(ValueError):
+    pass
+
+
+@dataclass
+class Transfer:
+    src: str
+    dst: str
+    exclude: list = field(default_factory=list)
+
+
+@dataclass
+class Job:
+    name: str
+    workload: str
+    command: str
+    setup: str = ""
+    units_name: str = "units"
+    units_count: float | None = None
+    sensitivity: str = "cloud-ok"
+    min_mem_gb: float = 8
+    prefer: list = field(default_factory=list)
+    regions: list = field(default_factory=list)
+    providers: list = field(default_factory=list)
+    cloud: str = "COMMUNITY"
+    max_usd_per_h: float | None = None
+    image: str = ""
+    container_disk_gb: int = 40
+    volume: str | None = None
+    ports: list = field(default_factory=list)
+    env: dict = field(default_factory=dict)
+    inputs: list = field(default_factory=list)
+    outputs: list = field(default_factory=list)
+    checkpoint_src: str | None = None
+    checkpoint_every_s: int = 300
+    max_usd: float = 2.0
+    max_hours: float = 3.0
+    path: Path | None = None
+    placement: Placement = field(default_factory=Placement)
+
+
+def load(path):
+    path = Path(path).expanduser().resolve()
+    d = tomllib.loads(path.read_text())
+    s = settings()
+    for k in ("name", "workload"):
+        if not d.get(k):
+            raise JobError(f"{path.name}: `{k}` is required")
+    run = d.get("run", {})
+    if not run.get("command"):
+        raise JobError(f"{path.name}: [run] command is required")
+    gpu, pod, budget = d.get("gpu", {}), d.get("pod", {}), d.get("budget", {})
+    units = d.get("units", {})
+    ck = run.get("checkpoint") or {}
+    sens = d.get("sensitivity", "cloud-ok")
+    if sens not in ("cloud-ok", "owned-only"):
+        raise JobError(f"{path.name}: sensitivity must be cloud-ok or owned-only")
+    base = path.parent
+
+    def tr(items, local_key):
+        out = []
+        for it in items:
+            if not it.get("src") or not it.get("dst"):
+                raise JobError(f"{path.name}: inputs/outputs need src and dst")
+            t = Transfer(it["src"], it["dst"], list(it.get("exclude", [])))
+            loc = Path(getattr(t, local_key)).expanduser()
+            setattr(t, local_key, str(loc if loc.is_absolute() else (base / loc).resolve()))
+            out.append(t)
+        return out
+
+    job = Job(
+        name=d["name"], workload=d["workload"], command=run["command"], setup=run.get("setup", ""),
+        units_name=units.get("name", "units"), units_count=units.get("count"), sensitivity=sens,
+        min_mem_gb=float(gpu.get("min_mem_gb", 8)), prefer=list(gpu.get("prefer", [])),
+        regions=[r.upper() for r in gpu.get("regions", s["regions"])], providers=list(gpu.get("providers", [])),
+        cloud=gpu.get("cloud", s["cloud"]), max_usd_per_h=gpu.get("max_usd_per_h"), image=pod.get("image", ""),
+        container_disk_gb=int(pod.get("container_disk_gb", s["container_disk_gb"])), volume=pod.get("volume"),
+        ports=[str(p) for p in pod.get("ports", [])],
+        env={k: str(v) for k, v in d.get("env", {}).items()},
+        inputs=tr(d.get("inputs", []), "src"), outputs=tr(d.get("outputs", []), "dst"),
+        checkpoint_src=ck.get("src"), checkpoint_every_s=int(ck.get("every_s", s["checkpoint_every_s"])),
+        max_usd=float(budget.get("max_usd", s["default_job_cap_usd"])),
+        max_hours=float(budget.get("max_hours", s["default_max_hours"])), path=path,
+        placement=load_placement(d.get("placement"), error=JobError))
+    for p in job.ports:
+        num, _, proto = p.partition("/")
+        if not num.isdigit() or proto.lower() not in ("", "tcp", "udp", "http"):
+            raise JobError(f"{path.name}: ports look like \"5004/udp\" or \"8080/tcp\", got {p!r}")
+    if job.volume and job.providers and job.providers != ["runpod"]:
+        raise JobError(f"{path.name}: a network volume is a Runpod thing; set providers = [\"runpod\"]")
+    for t in job.inputs:
+        if not Path(t.src).exists():
+            raise JobError(f"{path.name}: input {t.src} does not exist")
+    if any(k.upper().endswith(("KEY", "TOKEN", "SECRET")) for k in job.env):
+        raise JobError(f"{path.name}: secrets do not belong in [env]")
+    return job

gipper/latency.py

@@ -0,0 +1,50 @@
+"""Round trips from this machine: to a host's public IP before renting (system
+`ping`; about 60% of Vast hosts answer), and to a rented machine's ssh port after
+(TCP connect, no login). Both measure from *here*; for a job whose user is
+elsewhere (`latency_target` names where its own measurements came from), they're
+a proxy, used to rank but never to pass `max_rtt_ms`, unless the target is "here".
+
+Vast's location labels can be wrong (on 2026-10-28 a "China" host answered in
+80 ms from the IP an "United States" host had used), so a measured round trip
+outranks the country it claims.
+"""
+import re
+import socket
+import subprocess
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+HERE = socket.gethostname()
+_RTT = re.compile(r"= [\d.]+/([\d.]+)/")   # ping summary: min/avg/max/mdev -> avg
+
+
+def ping_ms(ip, count=3, timeout_s=2):
+    """Average ICMP round trip in ms, or None if the host doesn't answer."""
+    try:
+        r = subprocess.run(["ping", "-n", "-q", "-c", str(count), "-W", str(timeout_s), ip],
+                           capture_output=True, text=True, timeout=count * timeout_s + 3)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    m = _RTT.search(r.stdout)
+    return float(m.group(1)) if r.returncode == 0 and m else None
+
+
+def ping_all(ips, workers=16):
+    """{ip: ms or None}, pinged in parallel (about 6 s for any number up to `workers`)."""
+    ips = sorted({ip for ip in ips if ip})
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        return dict(zip(ips, pool.map(ping_ms, ips)))
+
+
+def connect_ms(host, port, tries=5, timeout_s=3):
+    """Fastest TCP connect to host:port in ms (a round trip, without logging in)."""
+    best = None
+    for _ in range(tries):
+        t = time.perf_counter()
+        try:
+            with socket.create_connection((host, int(port)), timeout=timeout_s):
+                ms = (time.perf_counter() - t) * 1000
+        except OSError:
+            continue
+        best = ms if best is None else min(best, ms)
+    return round(best, 1) if best is not None else None

gipper/ledger.py

@@ -0,0 +1,68 @@
+"""The receipt ledger: append-only JSON lines, one file per month.
+
+Every record is `gipper.receipt/v1`: {schema, id, ts, kind, job, refs, payload}.
+Kinds so far: estimate, pod.create, pod.ready, run.start, run.end, spend,
+checkpoint, collect, pod.terminate, cap, reap, error. External readers and replay
+read these; nothing rewrites them.
+"""
+import json
+import os
+import time
+import uuid
+from datetime import datetime, timezone
+
+from gipper.config import DATA, redact
+
+SCHEMA = "gipper.receipt/v1"
+
+
+def _dir():
+    d = DATA / "receipts"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def new_id(prefix="r"):
+    return f"{prefix}_{uuid.uuid4().hex[:12]}"
+
+
+def write(kind, payload=None, *, job=None, refs=None):
+    now = datetime.now(timezone.utc)
+    rec = {"schema": SCHEMA, "id": new_id(), "ts": now.isoformat(timespec="seconds"), "kind": kind,
+           "job": job, "refs": refs or {}, "payload": payload or {}}
+    line = redact(json.dumps(rec, default=str))
+    path = _dir() / f"{now:%Y-%m}.jsonl"
+    with open(path, "a") as f:
+        f.write(line + "\n")
+        f.flush()
+        os.fsync(f.fileno())
+    return rec
+
+
+def read(since=None, kinds=None, job=None):
+    """Yield receipts (oldest first); since = epoch seconds or ISO date."""
+    if isinstance(since, str):
+        since = datetime.fromisoformat(since).replace(tzinfo=timezone.utc).timestamp()
+    for path in sorted(_dir().glob("*.jsonl")):
+        for line in path.read_text().splitlines():
+            if not line.strip():
+                continue
+            rec = json.loads(line)
+            if since and datetime.fromisoformat(rec["ts"]).timestamp() < since:
+                continue
+            if kinds and rec["kind"] not in kinds:
+                continue
+            if job and rec.get("job") != job:
+                continue
+            yield rec
+
+
+def spend_since(since):
+    """Dollars spent since `since`: providers' bills where reconciled, else gipper's own."""
+    from gipper.billing import spend_since as billed_or_computed
+    return billed_or_computed(since)
+
+
+def today_start():
+    t = time.localtime()
+    return time.mktime((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0, 0, 0, -1))

gipper/lifecycle.py

@@ -0,0 +1,60 @@
+"""Ending a rental, and knowing it ended: a provider accepting the delete is not
+the machine being gone. `terminate()` asks for the delete (retrying), then reads
+the provider's list until the machine is absent. Only then does it write
+`pod.closure_verified`, `pod.terminate` and the computed `spend`; until then the
+machine stays `pod.termination_pending`, which the reaper picks up first, and
+gipper rents no replacement while a previous machine may still be billing.
+"""
+import time
+
+from gipper import ledger
+
+DELETE_TRIES, LIST_TRIES, PAUSE_S = 5, 4, 2
+
+
+def terminate(provider, iid, *, job_id=None, started=None, usd_h=None, refs=None, delete_accepted=False):
+    """True once the provider no longer lists the machine."""
+    iid = str(iid)
+    base = {"provider": provider.name, "pod": iid}
+    ledger.write("pod.termination_pending", {**base, "delete_accepted": delete_accepted,
+                                             "reason": "termination requested; not yet confirmed gone"}, job=job_id)
+    accepted, error = delete_accepted, None
+    for attempt in range(0 if accepted else DELETE_TRIES):
+        try:
+            provider.terminate(iid)
+            accepted = True
+            break
+        except Exception as e:
+            error = f"{type(e).__name__}: {e}"
+            if attempt < DELETE_TRIES - 1:
+                time.sleep(PAUSE_S)
+    gone = False
+    if accepted:
+        for attempt in range(LIST_TRIES):
+            try:
+                listed = provider.list_ours()
+                if not isinstance(listed, list) or not all(isinstance(m, dict) and "id" in m for m in listed):
+                    raise ValueError("provider returned an unreadable machine list")
+                gone = not any(str(m["id"]) == iid for m in listed)
+                error = None if gone else "provider still lists the machine"
+                if gone:
+                    break
+            except Exception as e:   # an unreadable list is not proof of absence
+                error = f"{type(e).__name__}: {e}"
+            if attempt < LIST_TRIES - 1:
+                time.sleep(PAUSE_S)
+    if not gone:
+        ledger.write("pod.termination_pending", {**base, "delete_accepted": accepted,
+                                                 "reason": error or "not confirmed gone"}, job=job_id)
+        print(f"{provider.name} {iid}: termination pending ({error or 'not confirmed gone'})", flush=True)
+        return False
+    ledger.write("pod.closure_verified", {**base, "basis": "delete accepted, then absent from the provider's list"},
+                 job=job_id)
+    ledger.write("pod.terminate", base, job=job_id)
+    if started is not None and usd_h is not None:
+        secs = max(0, time.time() - started)
+        ledger.write("spend", {**base, "seconds": round(secs), "usd_per_h": usd_h, "usd": round(secs * usd_h / 3600, 4),
+                               "basis": "rate x lifetime to confirmed closure", "accuracy": "estimated"},
+                     job=job_id, refs=refs)
+        print(f"{provider.name} {iid} terminated after {secs / 60:.1f} min: ~${secs * usd_h / 3600:.3f}", flush=True)
+    return True