gipper 0.3.0__py3-none-any.whl

gipper/__init__.py

@@ -0,0 +1,2 @@
+"""gipper: GPU jobs across cloud markets."""
+__version__ = "0.3.0"

gipper/billing.py

@@ -0,0 +1,85 @@
+"""What providers actually charged, beside what gipper computed.
+
+gipper's own `spend` receipt is rate x lifetime, written at teardown. Bills differ:
+Vast charges disk from creation and GPU only while running, rounds up short
+rentals (a 47 s smoke run was billed 0.07 h) and charges bandwidth by the byte;
+Runpod bills container disk alongside the GPU. `reconcile()` fetches each
+provider's itemised charges for gipper's machines and records them as
+`spend.billed`, which `spend_since()` (the daily cap) and gipper.history (the
+billing factor used in estimates) then prefer.
+"""
+import time
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from gipper import ledger
+from gipper.config import settings
+from gipper.providers import load
+
+
+def _ours(since):
+    """(provider, id) -> job for every machine gipper created since `since`."""
+    return {((r["payload"].get("provider") or "runpod"), str(r["payload"]["pod"])): r["job"]
+            for r in ledger.read(since=since, kinds={"pod.create"}) if r["payload"].get("pod")}
+
+
+def reconcile(days=7, providers=None):
+    """Write `spend.billed` for gipper's machines whose bill has grown. Returns
+    [(provider, id, usd)] written. Providers bill by the day, and a day's charges
+    grow until it closes, so the query starts at a UTC midnight with a day to
+    spare, and a bill is only recorded if it exceeds the last one: a window that
+    clipped a day can't replace a fuller bill. One provider's billing API failing
+    doesn't stop the others."""
+    since = (int(time.time() // 86400) - days - 1) * 86400
+    ours = _ours(since - 86400)
+    last = {}
+    for r in ledger.read(since=since - 31 * 86400, kinds={"spend.billed"}):
+        key = (r["payload"]["provider"], str(r["payload"]["pod"]))
+        last[key] = max(last.get(key, 0.0), r["payload"]["usd"])
+    written = []
+    for p in providers or [load(n) for n in settings()["providers"]]:
+        if not p.has_key():
+            continue
+        try:
+            bills = p.billed(since)
+        except Exception as e:
+            print(f"{p.name}: could not read bills: {e}")
+            continue
+        for iid, (usd, detail) in (bills or {}).items():
+            key = (p.name, str(iid))
+            if key not in ours or round(usd, 4) <= round(last.get(key, -1), 4):
+                continue
+            ledger.write("spend.billed", {"provider": p.name, "pod": str(iid), "usd": round(usd, 6), "detail": detail,
+                                          "basis": "provider charges"}, job=ours[key])
+            written.append((p.name, str(iid), usd))
+    return written
+
+
+def spend_by_machine(since):
+    """(provider, id) -> usd for machines whose spend falls since `since`: the
+    largest bill where there is one, else gipper's computed spend. A machine's
+    spend is dated by its teardown (`spend` receipt), as the daily cap always
+    has been, or by its creation if it has no `spend` yet; a bill recorded days
+    later inherits that date."""
+    lookback = since - 31 * 86400 if since else None
+    created, ended, computed, billed = {}, {}, defaultdict(float), {}
+    for r in ledger.read(since=lookback, kinds={"pod.create", "spend", "spend.billed"}):
+        key = (r["payload"].get("provider") or "runpod", str(r["payload"].get("pod")))
+        if r["kind"] == "pod.create":
+            created[key] = r["ts"]
+        elif r["kind"] == "spend.billed":
+            billed[key] = max(billed.get(key, 0.0), r["payload"]["usd"])
+        else:
+            computed[key] += r["payload"].get("usd", 0.0)
+            ended[key] = r["ts"]
+    since_iso = datetime.fromtimestamp(since, timezone.utc).isoformat(timespec="seconds") if since else ""
+    out = {}
+    for key in set(computed) | set(billed):
+        when = ended.get(key) or created.get(key)
+        if when is not None and when >= since_iso:
+            out[key] = billed.get(key, computed.get(key, 0.0))
+    return out
+
+
+def spend_since(since):
+    return sum(spend_by_machine(since).values())

gipper/bootstrap.sh

@@ -0,0 +1,25 @@
+# Sourced at the start of every gipper job on a pod: tools jobs can rely on.
+# Idempotent and quiet; anything heavier belongs in the job's own setup step.
+export PATH="$HOME/.local/bin:$PATH"
+export PIP_DISABLE_PIP_VERSION_CHECK=1
+if ! command -v uv >/dev/null 2>&1; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1
+fi
+# Weights and environments persist on a network volume when one is attached.
+if [ -d /workspace ] && [ -w /workspace ]; then
+  export HF_HOME="${HF_HOME:-/workspace/hf}" UV_CACHE_DIR="${UV_CACHE_DIR:-/workspace/uv-cache}"
+fi
+# The PyTorch wheel index that matches this host's *driver* (not what nvidia-smi
+# reports as "CUDA Version": images with CUDA forward-compat libraries inflate
+# that). Jobs install torch with: uv pip install torch --index-url "$GIPPER_TORCH_INDEX"
+_drv=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | cut -d. -f1)
+_drv=${_drv:-0}
+if   [ "$_drv" -ge 580 ]; then _cuda=130
+elif [ "$_drv" -ge 570 ]; then _cuda=128
+elif [ "$_drv" -ge 560 ]; then _cuda=126
+elif [ "$_drv" -ge 550 ]; then _cuda=124
+else _cuda=121; fi
+GIPPER_TORCH_INDEX=https://download.pytorch.org/whl/cu$_cuda
+export GIPPER_TORCH_INDEX
+echo "gipper: driver $_drv -> torch cu$_cuda ($GIPPER_TORCH_INDEX)"
+nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>/dev/null | sed 's/^/gipper: gpu /'

gipper/cli.py

@@ -0,0 +1,216 @@
+"""gipper: GPU jobs across cloud markets.
+
+    gipper pod gpus [--min-mem GB] [--region US,CA] [--udp] [--provider P]   offers across markets
+    gipper pod estimate JOB.toml [--throughput U/S] [--provider P]           what a run would take and cost
+    gipper pod run JOB.toml [--dry-run] [--gpu NAME] [--provider P]          rent, run, collect, terminate
+    gipper pod hosts [--provider P]               what gipper has learned about providers and machines
+    gipper pod ls                                 gipper's machines now running, every provider
+    gipper pod reap [--dry-run]                   end machines gipper forgot
+    gipper pod reconcile [--days N]               record what providers actually billed
+    gipper pod cost [--since YYYY-MM-DD]          spend per job: billed where known, else computed
+    gipper pod account                            balance per provider
+    gipper ledger [--since YYYY-MM-DD] [--job ID] raw receipts
+
+Providers: runpod, vast, shadeform (keys RUNPOD_API_KEY, VAST_API_KEY,
+SHADEFORM_API_KEY in the environment or ~/.config/gipper/.env). Vast and Shadeform list offers without a key.
+"""
+import argparse
+import json
+import signal
+import sys
+from collections import defaultdict
+
+PROVIDERS = ["runpod", "vast", "shadeform"]
+
+
+def _providers(name=None):
+    from gipper.config import settings
+    from gipper.providers import load
+    return [load(n) for n in ([name] if name else settings()["providers"])]
+
+
+def _offer_line(o):
+    e = o.estimate
+    return (f"{o.provider:9s} {o.gpu[:28]:28s} {o.mem_gb:>4.0f} {o.usd_h:>6.2f} {e.get('wall_s', 0) / 60:>5.0f} "
+            f"{e.get('usd', 0):>7.3f} {e.get('p_fail', 0):>5.0%} {e.get('score', 0):>7.3f}  {o.where}")
+
+
+HEADER = f"{'provider':9s} {'gpu':28s} {'GB':>4s} {'$/h':>6s} {'min':>5s} {'$run':>7s} {'risk':>5s} {'score':>7s}  where"
+
+
+def _pod(args):
+    if args.cmd == "gpus":
+        from gipper import market
+        from gipper.job import Job
+        job = Job(name="gpus", workload="-", command="-", min_mem_gb=args.min_mem, cloud=args.cloud,
+                  regions=[r.strip().upper() for r in (args.region or "").split(",") if r.strip()],
+                  ports=["9/udp"] if args.udp else [])
+        offers, notes = market.shop(job, names=[args.provider] if args.provider else None, keyed_only=False)
+        keyed = {p.name for p in _providers() if p.has_key()}
+        print(HEADER + "   (a 30-minute job)")
+        for o in offers[:args.limit]:
+            print(_offer_line(o) + ("" if o.provider in keyed else "  (no key)"))
+        for n in notes:
+            print(f"note: {n}")
+        return
+    if args.cmd in ("estimate", "run"):
+        from gipper import run as runmod
+        from gipper.job import load
+        job = load(args.job)
+        if args.cmd == "run":
+            _, outcome = runmod.run(job, dry_run=args.dry_run, gpu_override=args.gpu, provider=args.provider,
+                                    throughput=args.throughput)
+            sys.exit(0 if outcome in ("succeeded", None) or args.dry_run else 1)
+        offers, _, notes = runmod.plan(job, provider=args.provider, throughput=args.throughput)
+        print(HEADER)
+        for o in offers[:8]:
+            print(_offer_line(o))
+        best = offers[0].estimate
+        print(f"\nbest: {offers[0].label}, objective {best['objective']}")
+        print("  seconds: " + ", ".join(f"{k} {v}" for k, v in best["phases_s"].items()))
+        print(f"  dollars: rental {best['rental_usd']:.4f} + bandwidth {best['bandwidth_usd']:.4f}"
+              + ("" if best["bandwidth_known"] else " (fees not advertised)")
+              + (f", x{best['billing_factor']:.2f} billed vs computed" if best["billing_factor"] else ""))
+        print(f"  host risk {best['p_fail']:.0%} -> ${best['risk_usd']:.3f}; waiting valued at ${best['hour_value_usd']}/h")
+        print("  from: " + ", ".join(f"{k} {v}" for k, v in best["basis"].items()))
+        for n in notes:
+            print(f"note: {n}")
+        return
+    if args.cmd == "hosts":
+        from gipper.history import History
+        h = History.load()
+        for name in [args.provider] if args.provider else PROVIDERS:
+            s = h.summary(name)
+            if not s["rentals"]:
+                continue
+            fmt = lambda v, f: f.format(v) if v is not None else "-"
+            print(f"{name}: {s['rentals']} rentals, {s['good']} good, {s['bad']} bad -> {s['failure']:.0%} risk for a new one; "
+                  f"ready {fmt(s['ready_s'], '{:.0f}')} s; measured up {fmt(s['up_mbps'], '{:.0f}')} / "
+                  f"down {fmt(s['down_mbps'], '{:.0f}')} Mb/s; billed {fmt(s['billing_factor'], 'x{:.2f}')} computed"
+                  + (f"; {s['cleanup_pending']} cleanup(s) unconfirmed" if s["cleanup_pending"] else ""))
+            for why in s["reasons"]:
+                print(f"  bad: {why[:110]}")
+            machines = defaultdict(list)
+            for r in h.rentals_of(name):
+                if r.machine:
+                    machines[r.machine].append(r)
+            for m, rs in sorted(machines.items(), key=lambda kv: -len(kv[1])):
+                bad = sum(r.outcome == "bad" for r in rs)
+                if len(rs) > 1 or bad:
+                    print(f"  machine {m:>10s}  {len(rs)} rentals, {bad} bad -> {h.failure(name, m):.0%}  "
+                          f"{rs[-1].gpu or ''}")
+        return
+    if args.cmd == "account":
+        for p in _providers():
+            if not p.has_key():
+                print(f"{p.name:10s} no {p.key_name}")
+                continue
+            try:
+                print(f"{p.name:10s} {p.account()}")
+            except Exception as e:
+                print(f"{p.name:10s} error: {e}")
+        return
+    if args.cmd == "ls":
+        for p in _providers():
+            if not p.has_key():
+                continue
+            try:
+                for m in p.list_ours():
+                    print(f"{p.name:10s} {m['id']}  {m['name']}  {m.get('status')}  ${m.get('usd_h')}/h")
+            except Exception as e:
+                print(f"{p.name:10s} error: {e}")
+        return
+    if args.cmd == "reap":
+        from gipper.reap import reap
+        acted = reap(dry_run=args.dry_run)
+        for prov, pid, name, why in acted:
+            print(f"{'would end' if args.dry_run else 'ended or retried'} {prov} {pid} {name}: {why}")
+        if not acted:
+            print("nothing to reap")
+        return
+    if args.cmd == "reconcile":
+        from gipper.billing import reconcile
+        written = reconcile(days=args.days)
+        for prov, iid, usd in written:
+            print(f"{prov} {iid}: billed ${usd:.4f}")
+        print(f"{len(written)} bill(s) recorded" if written else "bills already up to date")
+        return
+    if args.cmd == "cost":
+        from gipper import ledger
+        # a job belongs to the day it started; its bill may be recorded days later
+        per = defaultdict(lambda: {"computed": 0.0, "billed": {}, "s": 0, "name": "", "outcome": "", "first": None})
+        for r in ledger.read(kinds={"pod.create", "spend", "spend.billed", "run.end"}):
+            # some spend (e.g. a lease run outside `pod run`) is written without a job id
+            j = per[r["job"] or f"(no job) {r['refs'].get('name', '')}".strip()]
+            j["first"] = j["first"] or r["ts"]
+            if r["kind"] == "pod.create":
+                continue
+            j["name"] = r["refs"].get("name", j["name"])
+            if r["kind"] == "spend":
+                j["computed"] += r["payload"]["usd"]
+                j["s"] += r["payload"]["seconds"]
+            elif r["kind"] == "spend.billed":
+                j["billed"][(r["payload"]["provider"], r["payload"]["pod"])] = r["payload"]["usd"]
+            else:
+                j["outcome"] = r["payload"]["outcome"]
+        computed = billed_total = 0.0
+        print(f"{'job':16s}  {'name':30s} {'min':>6s} {'computed':>9s} {'billed':>8s}  outcome")
+        for jid, j in per.items():
+            if (not j["s"] and not j["billed"]) or (args.since and (j["first"] or "") < args.since):
+                continue
+            billed = sum(j["billed"].values()) if j["billed"] else None
+            shown = f"{billed:8.3f}" if billed is not None else f"{'-':>8s}"
+            print(f"{jid:16s}  {j['name'][:30]:30s} {j['s'] / 60:6.1f} {j['computed']:9.3f} {shown}  {j['outcome']}")
+            computed += j["computed"]
+            billed_total += billed if billed is not None else j["computed"]
+        print(f"total computed ${computed:.3f}; billed where known, else computed: ${billed_total:.3f}")
+
+
+def main(argv=None):
+    sys.stdout.reconfigure(line_buffering=True)  # live output when redirected to a file
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)  # `gipper pod hosts | head` ends quietly
+    ap = argparse.ArgumentParser(prog="gipper", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = ap.add_subparsers(dest="group")
+    pod = sub.add_parser("pod", help="rented GPUs (Runpod, Vast, Shadeform)")
+    ps = pod.add_subparsers(dest="cmd", required=True)
+    g = ps.add_parser("gpus")
+    g.add_argument("--min-mem", type=float, default=16)
+    g.add_argument("--region", help="ISO country codes, comma-separated (US,CA)")
+    g.add_argument("--udp", action="store_true", help="only providers that can expose UDP ports")
+    g.add_argument("--provider", choices=PROVIDERS)
+    g.add_argument("--cloud", default="COMMUNITY", choices=["COMMUNITY", "SECURE"], help="Runpod cloud to start from")
+    g.add_argument("--limit", type=int, default=30)
+    for name in ("estimate", "run"):
+        p = ps.add_parser(name)
+        p.add_argument("job")
+        p.add_argument("--throughput", type=float, help="units per second, if known")
+        p.add_argument("--provider", choices=PROVIDERS, help="shop only this provider")
+        if name == "run":
+            p.add_argument("--dry-run", action="store_true")
+            p.add_argument("--gpu", help="only this GPU (e.g. \"RTX 4090\")")
+    h = ps.add_parser("hosts")
+    h.add_argument("--provider", choices=PROVIDERS)
+    ps.add_parser("ls")
+    ps.add_parser("account")
+    r = ps.add_parser("reap")
+    r.add_argument("--dry-run", action="store_true")
+    rc = ps.add_parser("reconcile")
+    rc.add_argument("--days", type=int, default=7)
+    c = ps.add_parser("cost")
+    c.add_argument("--since")
+    lg = sub.add_parser("ledger", help="raw receipts")
+    lg.add_argument("--since")
+    lg.add_argument("--job")
+    a = ap.parse_args(argv)
+    if a.group == "pod":
+        return _pod(a)
+    if a.group == "ledger":
+        from gipper import ledger
+        for rec in ledger.read(since=a.since, job=a.job):
+            print(json.dumps(rec))
+        return
+    ap.print_help()
+
+
+if __name__ == "__main__":
+    main()

gipper/config.py

@@ -0,0 +1,91 @@
+"""Paths, settings and the provider keys.
+
+Each provider key (RUNPOD_API_KEY, VAST_API_KEY, SHADEFORM_API_KEY) comes from the
+environment or a configured .env (mode 600, git-ignored) and only ever travels in a
+request header: never logged, printed, written elsewhere or passed on a command
+line. `redact()` scrubs every one of them from any text that might be shown or
+recorded.
+"""
+import os
+import tomllib
+from pathlib import Path
+
+HOME = Path.home()
+ROOT = Path(__file__).resolve().parents[1]
+DATA = Path(os.environ.get("GIPPER_DATA", HOME / ".local/share/gipper"))
+CONFIG_FILE = Path(os.environ.get("GIPPER_CONFIG", HOME / ".config/gipper/config.toml"))
+KEY_DIR = Path(os.environ.get("GIPPER_KEYS", HOME / ".config/gipper"))
+
+DEFAULTS = {
+    "daily_cap_usd": 5.0,           # refuse a new pod if today's spend + estimate exceeds this
+    "default_job_cap_usd": 2.0,     # a job without its own budget gets this
+    "default_max_hours": 3.0,
+    "image": "runpod/base:1.4.1-ubuntu2404",   # Runpod's default; no CUDA in the image: torch wheels bring their own
+    "vast_image": "vastai/base-image:@vastai-automatic-tag",   # Vast picks the tag that fits the host
+    "providers": ["runpod", "vast", "shadeform"],   # markets to shop; those without a key are skipped
+    "regions": [],                  # ISO country codes a job may run in by default; empty = anywhere
+    "cuda_versions": [],            # no host filter: it empties the market; bootstrap picks a matching torch build
+    "cloud": "COMMUNITY",           # COMMUNITY or SECURE
+    "container_disk_gb": 40,
+    "checkpoint_every_s": 300,
+    # How offers are weighed (see docs/placement.md). An hour of waiting is worth
+    # this much; a failed host (time lost, attention, a re-run) costs this much on
+    # top of the rental it wasted. Short jobs end up chosen for reliability and
+    # speed, long ones for price.
+    "hour_value_usd": 3.0,
+    "failure_cost_usd": 1.0,
+    "history_days": 90,             # how far back the ledger counts as memory
+}
+
+
+def settings():
+    s = dict(DEFAULTS)
+    if CONFIG_FILE.exists():
+        s.update(tomllib.loads(CONFIG_FILE.read_text()))
+    return s
+
+
+def _dotenv(path):
+    out = {}
+    if path.exists():
+        for line in path.read_text().splitlines():
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                k, v = line.split("=", 1)
+                out[k.strip()] = v.strip().strip('"').strip("'")
+    return out
+
+
+KEYS = ("RUNPOD_API_KEY", "VAST_API_KEY", "SHADEFORM_API_KEY")
+
+
+def secret(name, required=True):
+    key = os.environ.get(name)
+    if key:
+        return key
+    # Explicit GIPPER_ENV is exclusive (including a missing file). Otherwise
+    # prefer user configuration while retaining editable installs' legacy .env.
+    if "GIPPER_ENV" in os.environ:
+        values = _dotenv(Path(os.environ["GIPPER_ENV"]))
+    else:
+        values = _dotenv(ROOT / ".env")
+        values.update(_dotenv(CONFIG_FILE.parent / ".env"))
+    key = values.get(name)
+    if not key and required:
+        raise SystemExit(f"{name} is not set (environment or configured .env; see GIPPER_ENV)")
+    return key or None
+
+
+def runpod_key():
+    return secret("RUNPOD_API_KEY")
+
+
+def redact(text):
+    """Remove every provider key from text before it is shown or recorded."""
+    if not isinstance(text, str):
+        return text
+    for name in KEYS:
+        key = secret(name, required=False)
+        if key:
+            text = text.replace(key, "<redacted>")
+    return text

gipper/costs.py

@@ -0,0 +1,125 @@
+"""The whole-run view of an offer: how long the job would take there, what it would
+cost, how likely the host is to fail, and one number to rank offers by.
+
+    wall     = ready + setup + upload + fetch + compute + download
+    expected = (rental for wall + bandwidth fees) x the provider's billing factor
+    score    = expected + p_fail x (failure_cost + rental and time lost to a bad host)
+               + hour_value x wall
+
+Every figure comes from the best source available, and the estimate says which:
+the job's own [placement] hints, the ledger's memory (gipper.history), what the
+provider advertises, or a stated default.
+
+The score is what makes short and long jobs come out differently without a
+switch: on a 15-minute job, $0.40/h against $0.80/h is ten cents, smaller than a
+26% chance of a bad host at $1 a failure, so reliability and speed win; on a
+ten-hour job the same prices are four dollars apart, and price wins.
+
+Objectives (placement.objective): "auto" values waiting at settings.hour_value_usd;
+"cost" doesn't value waiting at all (failures still cost); "time" values it at
+TIME_HOUR_VALUE; "latency" (interactive profile) is ranked by nearness first in
+gipper.market, and scored like "auto".
+
+The estimate also carries 0.1's field names (estimated_total_usd, setup_download_s,
+assumptions, ...), which clients' reports and the estimate receipt still read.
+"""
+from gipper.config import settings
+from gipper.placement import input_gb
+
+READY_S = {"runpod": 90, "vast": 120, "shadeform": 420}  # until ssh, before memory has a median
+SETUP_S = 120               # bootstrap and the job's own setup, if the job doesn't say
+COMPUTE_S = 1800            # if neither the job nor memory says
+PREFLIGHT_S = 60            # time a bad host costs beyond becoming ready
+TIME_HOUR_VALUE = 50.0      # $/h of waiting under objective = "time"
+
+
+def _mbps(measured, advertised, p, client_cap=None):
+    """Link speed and where it came from; a client-side cap limits whichever applies."""
+    if measured:
+        v, src = measured, "measured"
+    elif advertised and advertised > 0:
+        v, src = advertised * p.transfer_efficiency, "advertised"
+    else:
+        v, src = p.fallback_mbps * p.transfer_efficiency, "default"
+    if client_cap and client_cap * p.transfer_efficiency < v:
+        return client_cap * p.transfer_efficiency, "client cap"
+    return v, src
+
+
+def estimate(job, offer, history, *, throughput=None):
+    """Phases (seconds), dollars, risk and score for running `job` on `offer`."""
+    from gipper.market import matches
+    s, p, m = settings(), job.placement, offer.metrics
+    machine = offer.ref.get("machine")
+    basis = {}
+
+    ready = history.ready_s(offer.provider, machine)
+    basis["ready"] = "memory" if ready else "default"
+    ready = ready or READY_S.get(offer.provider, 180)
+
+    setup = p.setup_s if p.setup_s is not None else SETUP_S
+    basis["setup"] = "job" if p.setup_s is not None else "default"
+
+    up_gb = p.upload_gb if p.upload_gb is not None else input_gb(job)
+    up_mbps, basis["upload"] = _mbps(history.mbps(offer.provider, machine, "up", offer.country),
+                                     m.get("host_download_mbps"), p, p.client_upload_mbps)
+    down_mbps, basis["download"] = _mbps(history.mbps(offer.provider, machine, "down", offer.country),
+                                         m.get("host_upload_mbps"), p, p.client_download_mbps)
+    fetch_mbps, basis["fetch"] = _mbps(None, m.get("host_download_mbps"), p)  # host <- internet: no client cap
+    upload, download = up_gb * 8000 / up_mbps, p.download_gb * 8000 / down_mbps
+    fetch = p.setup_download_gb * 8000 / fetch_mbps
+
+    by_gpu = next((v for g, v in p.compute_s_by_gpu.items() if matches(g, offer.gpu)), None)
+    remembered = history.compute_s(job.workload, offer.gpu)
+    if by_gpu:
+        compute, basis["compute"] = by_gpu, "job (this GPU)"
+    elif throughput and job.units_count:
+        compute, basis["compute"] = job.units_count / throughput, "throughput"
+    elif p.compute_s is not None:
+        compute, basis["compute"] = p.compute_s, "job"
+    elif remembered:
+        compute, basis["compute"] = remembered, "memory"
+    else:
+        compute, basis["compute"] = min(COMPUTE_S, job.max_hours * 3600), "default"
+
+    wall = ready + setup + upload + fetch + compute + download
+    rental = wall / 3600 * offer.usd_h
+    fee_in, fee_out = m.get("host_download_usd_per_gb"), m.get("host_upload_usd_per_gb")
+    fees_known = offer.provider != "vast" or (fee_in is not None and fee_out is not None)
+    bandwidth = (up_gb + p.setup_download_gb) * (fee_in or 0) + p.download_gb * (fee_out or 0)
+    factor = history.billing_factor(offer.provider)
+    basis["billing"] = "memory" if factor else "rate"
+    expected = rental * (factor or 1.0) + bandwidth   # bills already itemise bandwidth: the factor is the rest
+
+    p_fail = history.failure(offer.provider, machine)
+    hour_value = {"auto": s["hour_value_usd"], "latency": s["hour_value_usd"], "cost": 0.0,
+                  "time": TIME_HOUR_VALUE}[p.objective]
+    lost_h = (ready + PREFLIGHT_S) / 3600
+    risk = p_fail * (s["failure_cost_usd"] + lost_h * (offer.usd_h + hour_value))
+    score = expected + risk + hour_value * wall / 3600
+    from gipper.placement import best_rtt, distance_km
+    rtt, rtt_basis = best_rtt(offer, p, history)
+    phases = {"ready": round(ready), "setup": round(setup), "upload": round(upload), "fetch": round(fetch),
+              "compute": round(compute), "download": round(download)}
+    legacy = {   # 0.1's names, read by clients' reports
+        "profile": p.profile, "estimated_total_usd": round(expected, 4),
+        "estimated_bandwidth_usd": round(bandwidth, 4) if fees_known else None, "known_bandwidth_usd": round(bandwidth, 4),
+        "bandwidth_fees_known": fees_known, "cost_is_lower_bound": not fees_known,
+        "setup_s": phases["setup"], "setup_download_s": phases["fetch"], "upload_s": phases["upload"],
+        "download_s": phases["download"], "compute_s": phases["compute"],
+        "upload_gb": up_gb, "download_gb": p.download_gb, "setup_download_gb": p.setup_download_gb,
+        "host_download_mbps": m.get("host_download_mbps"), "host_upload_mbps": m.get("host_upload_mbps"),
+        "host_download_usd_per_gb": fee_in, "host_upload_usd_per_gb": fee_out,
+        "effective_upload_mbps": round(up_mbps, 1), "effective_download_mbps": round(down_mbps, 1),
+        "dlperf": m.get("dlperf"), "measured_rtt_ms": rtt if rtt_basis == "measured" else None,
+        "latency_target": p.latency_target or None,
+        "distance_km": round(distance_km(offer, p), 2) if distance_km(offer, p) is not None else None,
+        "assumptions": [f"{k} from {v}" for k, v in basis.items()]
+                       + ([] if fees_known else ["bandwidth fees not advertised: total is a lower bound"]),
+    }
+    return {**legacy, "wall_s": round(wall), "usd": round(expected, 4), "score": round(score, 4),
+            "rtt_ms": rtt, "rtt_basis": rtt_basis,
+            "phases_s": phases,
+            "rental_usd": round(rental, 4), "bandwidth_usd": round(bandwidth, 4), "bandwidth_known": fees_known,
+            "billing_factor": factor, "p_fail": round(p_fail, 3), "risk_usd": round(risk, 4),
+            "objective": p.objective, "hour_value_usd": hour_value, "basis": basis}

gipper/cuda_probe.py

@@ -0,0 +1,78 @@
+"""Initialize the actual CUDA driver before any model/input upload.
+
+nvidia-smi uses NVML and cannot prove cuInit succeeds. Only an 804 may trigger a
+fresh-process retry excluding explicit CUDA forward-compatibility path entries.
+No driver paths are guessed, libraries installed, or host configuration changed.
+"""
+import json
+import shlex
+from pathlib import PurePosixPath
+from gipper.providers import BadHost
+
+SCRIPT = r'''
+import ctypes,json,os
+result={"ld_library_path":os.environ.get("LD_LIBRARY_PATH",""),"loaded_library_paths":[],"device_count":0}
+try:
+    cuda=ctypes.CDLL("libcuda.so.1")
+    cuda.cuInit.argtypes=[ctypes.c_uint];cuda.cuInit.restype=ctypes.c_int
+    code=cuda.cuInit(0);result["cu_init"]=code
+    if code==0:
+        count=ctypes.c_int()
+        cuda.cuDeviceGetCount.argtypes=[ctypes.POINTER(ctypes.c_int)];cuda.cuDeviceGetCount.restype=ctypes.c_int
+        result["cu_device_get_count"]=cuda.cuDeviceGetCount(ctypes.byref(count));result["device_count"]=count.value
+    try:
+        result["loaded_library_paths"]=sorted({line.split()[-1] for line in open("/proc/self/maps") if "/libcuda.so" in line})
+    except OSError: pass
+except Exception as error:
+    result["error"]=type(error).__name__+": "+str(error)
+print(json.dumps(result))
+'''
+
+
+class CudaProbeError(BadHost):
+    def __init__(self, message, receipt):
+        super().__init__(message)
+        self.receipt=receipt
+
+
+def compat_path(value):
+    parts=PurePosixPath(value).parts
+    return (len(parts)>1 and parts[-1]=='compat' and parts[-2].startswith('cuda')) or any(p.startswith('cuda-compat-') for p in parts)
+
+
+def initialized(result):
+    return result.get('cu_init')==0 and result.get('cu_device_get_count')==0 and isinstance(result.get('device_count'),int) and result['device_count']>0
+
+
+def check(remote, env=None):
+    receipt={'attempts':[], 'environment_override':{}, 'status':'rejected',
+             'basis':'stdlib ctypes libcuda.so.1 cuInit(0) and cuDeviceGetCount; fresh Python per attempt'}
+    def attempt(ld=None):
+        prefix=('LD_LIBRARY_PATH='+shlex.quote(ld)+' ') if ld is not None else ''
+        response=remote.run(prefix+'python3 -c '+shlex.quote(SCRIPT),check=False,timeout=30)
+        if response.returncode:
+            raise RuntimeError('CUDA Python preflight exited '+str(response.returncode))
+        data=json.loads(response.stdout)
+        if not isinstance(data,dict):raise ValueError('invalid CUDA preflight response')
+        receipt['attempts'].append(data)
+        return data
+    try:
+        first=attempt((env or {}).get('LD_LIBRARY_PATH'))
+        if initialized(first):
+            receipt['status']='ok';return receipt
+        if first.get('cu_init')==804:
+            path=first.get('ld_library_path','')
+            if not isinstance(path,str): raise ValueError('invalid CUDA library path response')
+            entries=path.split(':')
+            removed=[p for p in entries if compat_path(p)]
+            if removed:
+                cleaned=':'.join(p for p in entries if not compat_path(p))
+                receipt['removed_compat_entries']=removed
+                second=attempt(cleaned)  # new SSH/Python; never retry CDLL in loaded process
+                if initialized(second) and second.get("ld_library_path") == cleaned:
+                    receipt.update(status='ok',environment_override={'LD_LIBRARY_PATH':cleaned})
+                    return receipt
+        receipt['error']='CUDA initialization/device enumeration failed: '+json.dumps(receipt['attempts'][-1])
+    except Exception as error:
+        receipt['error']=f'{type(error).__name__}: {error}'
+    raise CudaProbeError(receipt['error'],receipt)