cotop 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotop/__init__.py +0 -0
- cotop/app.py +572 -0
- cotop/colab.py +158 -0
- cotop/model.py +492 -0
- cotop/widgets.py +132 -0
- cotop-0.1.0.dist-info/METADATA +100 -0
- cotop-0.1.0.dist-info/RECORD +10 -0
- cotop-0.1.0.dist-info/WHEEL +4 -0
- cotop-0.1.0.dist-info/entry_points.txt +2 -0
- cotop-0.1.0.dist-info/licenses/LICENSE +202 -0
cotop/colab.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Access to Colab, signed in with the colab CLI's own credentials.
|
|
2
|
+
|
|
3
|
+
Reads never write the CLI's session file: the CLI's `sessions` and `status`
|
|
4
|
+
commands prune and rewrite local state, so this reads the store directly and
|
|
5
|
+
joins it with the server's assignments instead. Actions on a named runtime go
|
|
6
|
+
through the colab CLI itself, so its session file and history stay its own.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import requests
|
|
18
|
+
from colab_cli.auth import get_credentials
|
|
19
|
+
from colab_cli.client import Client, Prod
|
|
20
|
+
from colab_cli.state import StateStore
|
|
21
|
+
|
|
22
|
+
from cotop.model import Resources, Runtime, Status, parse_json, utc_now
|
|
23
|
+
|
|
24
|
+
COLAB = "https://colab.research.google.com"
|
|
25
|
+
USERINFO = "https://openidconnect.googleapis.com/v1/userinfo"
|
|
26
|
+
HEADERS = {"Accept": "application/json", "X-Colab-Client-Agent": "colab-cli"}
|
|
27
|
+
TIMEOUT = 15
|
|
28
|
+
RESOURCES_TIMEOUT = 3 # readings come every 2 s; one later than that is a gap
|
|
29
|
+
CLI_HOME = Path("~/.config/colab-cli").expanduser()
|
|
30
|
+
|
|
31
|
+
# The CLI's client logs request headers and bodies at DEBUG; keep tokens out.
|
|
32
|
+
logging.getLogger("colab_cli").setLevel(logging.WARNING)
|
|
33
|
+
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Colab:
|
|
37
|
+
def __init__(self, session, ssh_identity: str | None = None):
|
|
38
|
+
self.session = session
|
|
39
|
+
self.ssh_identity = ssh_identity
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def sign_in(cls, ssh_identity: str | None = None) -> Colab:
|
|
43
|
+
"""Use the token the colab CLI stored; prompts for sign-in if there is none."""
|
|
44
|
+
return cls(get_credentials(), ssh_identity)
|
|
45
|
+
|
|
46
|
+
def _get(self, url: str, **params):
|
|
47
|
+
r = self.session.get(url, params=params, headers=HEADERS, timeout=TIMEOUT)
|
|
48
|
+
r.raise_for_status()
|
|
49
|
+
return parse_json(r.text)
|
|
50
|
+
|
|
51
|
+
def email(self) -> str:
|
|
52
|
+
return self._get(USERINFO)["email"]
|
|
53
|
+
|
|
54
|
+
def ccu_info(self) -> dict:
|
|
55
|
+
return self._get(f"{COLAB}/tun/m/ccu-info", authuser="0")
|
|
56
|
+
|
|
57
|
+
def assignments(self) -> list[dict]:
|
|
58
|
+
return self._get(f"{COLAB}/tun/m/assignments", authuser="0").get("assignments", [])
|
|
59
|
+
|
|
60
|
+
def stop(self, runtime: Runtime) -> str:
|
|
61
|
+
if runtime.name is None: # opened in a browser or on another host: no CLI name to stop
|
|
62
|
+
Client(Prod(), self.session).unassign(runtime.endpoint)
|
|
63
|
+
return f"Stopped {runtime.endpoint}."
|
|
64
|
+
self.confirm_still_assigned(runtime)
|
|
65
|
+
return _cli("stop", "-s", runtime.name)
|
|
66
|
+
|
|
67
|
+
def restart_kernel(self, runtime: Runtime) -> str:
|
|
68
|
+
self.confirm_still_assigned(runtime)
|
|
69
|
+
return _cli("restart-kernel", "-s", runtime.name)
|
|
70
|
+
|
|
71
|
+
def ssh_command(self, runtime: Runtime) -> list[str]:
|
|
72
|
+
self.confirm_still_assigned(runtime)
|
|
73
|
+
identity = ["-i", self.ssh_identity] if self.ssh_identity else []
|
|
74
|
+
return [_colab(), "ssh", "-s", runtime.name, *identity]
|
|
75
|
+
|
|
76
|
+
def confirm_still_assigned(self, runtime: Runtime) -> None:
|
|
77
|
+
"""Act on a name only while it still means this VM.
|
|
78
|
+
|
|
79
|
+
The CLI resolves names, not endpoints: a name stopped and reused since
|
|
80
|
+
the last poll would point the action at another VM, and `colab ssh`
|
|
81
|
+
creates a new runtime for a name it does not know.
|
|
82
|
+
"""
|
|
83
|
+
if not still_assigned(runtime, self.assignments(), local_sessions()):
|
|
84
|
+
raise RuntimeGone(f"{runtime.name or runtime.endpoint} is no longer this runtime; refresh and retry")
|
|
85
|
+
|
|
86
|
+
def status(self, assignments: list[dict]) -> dict[str, Status | Exception]:
|
|
87
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
88
|
+
results = pool.map(_runtime_status, assignments)
|
|
89
|
+
return {a["endpoint"]: s for a, s in zip(assignments, results)}
|
|
90
|
+
|
|
91
|
+
def resources(self, runtimes: list[Runtime]) -> dict[str, Resources | None]:
|
|
92
|
+
"""One reading per runtime; a failed read is None, a gap in the graph."""
|
|
93
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
94
|
+
results = pool.map(_runtime_resources, runtimes)
|
|
95
|
+
return {r.endpoint: res for r, res in zip(runtimes, results)}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class RuntimeGone(Exception):
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def still_assigned(runtime: Runtime, assignments: list[dict], local: dict[str, dict]) -> bool:
|
|
103
|
+
if runtime.name is None:
|
|
104
|
+
return False
|
|
105
|
+
live = {a["endpoint"] for a in assignments}
|
|
106
|
+
return runtime.endpoint in live and local.get(runtime.name, {}).get("endpoint") == runtime.endpoint
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _colab() -> str:
|
|
110
|
+
"""The colab CLI installed with cotop, so it matches the library cotop reads with."""
|
|
111
|
+
path = Path(sys.executable).with_name("colab")
|
|
112
|
+
if not path.exists():
|
|
113
|
+
raise FileNotFoundError(f"the colab CLI is missing beside {sys.executable}")
|
|
114
|
+
return str(path)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _cli(*args: str) -> str:
|
|
118
|
+
"""Run a colab CLI command; its last output line is the result to show."""
|
|
119
|
+
done = subprocess.run([_colab(), *args], capture_output=True, text=True, timeout=180)
|
|
120
|
+
lines = (done.stdout + done.stderr).strip().splitlines()
|
|
121
|
+
if done.returncode != 0:
|
|
122
|
+
raise RuntimeError(lines[-1] if lines else f"colab {args[0]} exited {done.returncode}")
|
|
123
|
+
return lines[-1] if lines else f"colab {args[0]} done"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _proxy_get(url: str, token: str, path: str, timeout: float):
|
|
127
|
+
headers = {**HEADERS, "X-Colab-Runtime-Proxy-Token": token}
|
|
128
|
+
# Jupyter Server skips activity tracking for this parameter; whether Colab's
|
|
129
|
+
# idle shutdown looks at it is not documented.
|
|
130
|
+
r = requests.get(url.rstrip("/") + path, headers=headers, params={"no_track_activity": "1"}, timeout=timeout)
|
|
131
|
+
r.raise_for_status()
|
|
132
|
+
return parse_json(r.text)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _runtime_status(assignment: dict) -> Status | Exception:
|
|
136
|
+
proxy = assignment["runtimeProxyInfo"]
|
|
137
|
+
try:
|
|
138
|
+
return Status.parse(
|
|
139
|
+
_proxy_get(proxy["url"], proxy["token"], "/api/status", TIMEOUT),
|
|
140
|
+
_proxy_get(proxy["url"], proxy["token"], "/api/kernels", TIMEOUT),
|
|
141
|
+
)
|
|
142
|
+
except Exception as e: # shown in the runtime's row, never swallowed
|
|
143
|
+
return e
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _runtime_resources(runtime: Runtime) -> Resources | None:
|
|
147
|
+
try:
|
|
148
|
+
return Resources.parse(utc_now(), _proxy_get(runtime.url, runtime.token, "/api/colab/resources", RESOURCES_TIMEOUT))
|
|
149
|
+
except Exception: # a gap in the graph; the slow loop reports an unreachable runtime
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def local_sessions() -> dict[str, dict]:
|
|
154
|
+
"""The CLI's named sessions, without their tokens."""
|
|
155
|
+
return {
|
|
156
|
+
name: {"name": s.name, "endpoint": s.endpoint, "running": s.running}
|
|
157
|
+
for name, s in StateStore().list().items()
|
|
158
|
+
}
|
cotop/model.py
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
"""What a Colab account looks like at one moment: compute units and runtimes.
|
|
2
|
+
|
|
3
|
+
Every value here is parsed from a raw Colab response, so the parsing is the
|
|
4
|
+
single place that knows the wire format (XSSI prefix, camelCase, fractions).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from collections import deque
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime, timedelta, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
XSSI_PREFIX = ")]}'\n"
|
|
16
|
+
SHAPES = {0: "STANDARD", 1: "HIGH_RAM"}
|
|
17
|
+
VARIANTS = {0: "DEFAULT", 1: "GPU", 2: "TPU"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def parse_json(body: str):
|
|
21
|
+
return json.loads(body.removeprefix(XSSI_PREFIX))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def parse_time(value: str) -> datetime:
|
|
25
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Account:
|
|
30
|
+
email: str
|
|
31
|
+
balance: float
|
|
32
|
+
rate_hourly: float
|
|
33
|
+
assignments_count: int
|
|
34
|
+
eligible_gpus: tuple[str, ...]
|
|
35
|
+
eligible_tpus: tuple[str, ...]
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def parse(cls, email: str, ccu: dict) -> Account:
|
|
39
|
+
return cls(
|
|
40
|
+
email=email,
|
|
41
|
+
balance=ccu["currentBalance"],
|
|
42
|
+
rate_hourly=ccu["consumptionRateHourly"],
|
|
43
|
+
assignments_count=ccu["assignmentsCount"],
|
|
44
|
+
eligible_gpus=tuple(ccu.get("eligibleGpus", ())),
|
|
45
|
+
eligible_tpus=tuple(ccu.get("eligibleTpus", ())),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def runway(self) -> timedelta | None:
|
|
50
|
+
if self.rate_hourly <= 0:
|
|
51
|
+
return None
|
|
52
|
+
return timedelta(hours=self.balance / self.rate_hourly)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class Kernel:
|
|
57
|
+
state: str
|
|
58
|
+
last_activity: datetime
|
|
59
|
+
connections: int
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class Gpu:
|
|
64
|
+
name: str
|
|
65
|
+
utilization: float | None # None when Colab left it out: a gap, not an idle GPU
|
|
66
|
+
memory_used: int
|
|
67
|
+
memory_total: int
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class Resources:
|
|
72
|
+
"""One reading of a runtime's memory, disk and GPUs; cotop takes one every two seconds."""
|
|
73
|
+
|
|
74
|
+
at: datetime
|
|
75
|
+
memory_total: int
|
|
76
|
+
memory_free: int
|
|
77
|
+
disk_total: int
|
|
78
|
+
disk_used: int
|
|
79
|
+
gpus: tuple[Gpu, ...]
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def parse(cls, at: datetime, resources: dict) -> Resources:
|
|
83
|
+
"""Sizes are optional and default to 0, as in Colab's own client (colab-vscode);
|
|
84
|
+
a missing utilization stays None, so it draws as a gap rather than an idle GPU."""
|
|
85
|
+
memory = resources.get("memory", {})
|
|
86
|
+
disks = [d.get("filesystem", {}) for d in resources.get("disks", [])]
|
|
87
|
+
return cls(
|
|
88
|
+
at=at,
|
|
89
|
+
memory_total=memory.get("totalBytes", 0),
|
|
90
|
+
memory_free=memory.get("freeBytes", 0),
|
|
91
|
+
disk_total=sum(d.get("totalBytes", 0) for d in disks),
|
|
92
|
+
disk_used=sum(d.get("usedBytes", 0) for d in disks),
|
|
93
|
+
gpus=tuple(
|
|
94
|
+
Gpu(g.get("name", "GPU"), g.get("gpuUtilization"), g.get("memoryUsedBytes", 0), g.get("memoryTotalBytes", 0))
|
|
95
|
+
for g in resources.get("gpus", [])
|
|
96
|
+
),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def gpu_utilization(self) -> float | None:
|
|
101
|
+
"""Mean over the GPUs that reported one; None on a CPU or TPU runtime, or if none did."""
|
|
102
|
+
known = [g.utilization for g in self.gpus if g.utilization is not None]
|
|
103
|
+
return sum(known) / len(known) if known else None
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def vram(self) -> tuple[int, int] | None:
|
|
107
|
+
total = sum(g.memory_total for g in self.gpus)
|
|
108
|
+
return (sum(g.memory_used for g in self.gpus), total) if total else None
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def ram(self) -> tuple[int, int]:
|
|
112
|
+
return self.memory_total - self.memory_free, self.memory_total
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def disk(self) -> tuple[int, int]:
|
|
116
|
+
return self.disk_used, self.disk_total
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class Samples:
|
|
120
|
+
"""Recent resource readings per runtime. A failed reading is a gap (None), never a zero."""
|
|
121
|
+
|
|
122
|
+
KEEP = 900 # 30 minutes at one reading every two seconds
|
|
123
|
+
|
|
124
|
+
def __init__(self):
|
|
125
|
+
self.readings: dict[str, deque[tuple[datetime, Resources | None]]] = {}
|
|
126
|
+
|
|
127
|
+
def add(self, endpoint: str, at: datetime, reading: Resources | None) -> None:
|
|
128
|
+
self.readings.setdefault(endpoint, deque(maxlen=self.KEEP)).append((at, reading))
|
|
129
|
+
|
|
130
|
+
def latest(self, endpoint: str) -> Resources | None:
|
|
131
|
+
for _, reading in reversed(self.readings.get(endpoint, ())):
|
|
132
|
+
if reading is not None:
|
|
133
|
+
return reading
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
def series(self, endpoint: str, metric) -> list[float | None]:
|
|
137
|
+
"""metric(reading) for each reading, oldest first; gaps, and readings it has no value for, are None."""
|
|
138
|
+
return [metric(r) if r is not None else None for _, r in self.readings.get(endpoint, ())]
|
|
139
|
+
|
|
140
|
+
def time_of(self, endpoint: str, n: int) -> datetime | None:
|
|
141
|
+
"""When the n-th newest reading was taken, or the oldest if there are fewer."""
|
|
142
|
+
readings = self.readings.get(endpoint)
|
|
143
|
+
return readings[-min(n, len(readings))][0] if readings else None
|
|
144
|
+
|
|
145
|
+
def keep_only(self, endpoints: set[str]) -> None:
|
|
146
|
+
for gone in self.readings.keys() - endpoints:
|
|
147
|
+
del self.readings[gone]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True)
|
|
151
|
+
class Status:
|
|
152
|
+
"""What the runtime's Jupyter server says about its kernels."""
|
|
153
|
+
|
|
154
|
+
started: datetime
|
|
155
|
+
kernels: tuple[Kernel, ...]
|
|
156
|
+
|
|
157
|
+
@classmethod
|
|
158
|
+
def parse(cls, status: dict, kernels: list) -> Status:
|
|
159
|
+
return cls(
|
|
160
|
+
started=parse_time(status["started"]),
|
|
161
|
+
kernels=tuple(
|
|
162
|
+
Kernel(k["execution_state"], parse_time(k["last_activity"]), k["connections"])
|
|
163
|
+
for k in kernels
|
|
164
|
+
),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def busy(self) -> bool:
|
|
169
|
+
return any(k.state == "busy" for k in self.kernels)
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def clients(self) -> int:
|
|
173
|
+
return sum(k.connections for k in self.kernels)
|
|
174
|
+
|
|
175
|
+
def idle_for(self, now: datetime) -> timedelta | None:
|
|
176
|
+
"""How long every kernel has been idle; None while one is busy."""
|
|
177
|
+
if self.busy:
|
|
178
|
+
return None
|
|
179
|
+
if not self.kernels:
|
|
180
|
+
return now - self.started
|
|
181
|
+
return now - max(k.last_activity for k in self.kernels)
|
|
182
|
+
|
|
183
|
+
def uptime(self, now: datetime) -> timedelta:
|
|
184
|
+
return now - self.started
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(frozen=True)
|
|
188
|
+
class Runtime:
|
|
189
|
+
"""One Colab assignment: a VM the account is paying for."""
|
|
190
|
+
|
|
191
|
+
endpoint: str
|
|
192
|
+
accelerator: str
|
|
193
|
+
shape: str
|
|
194
|
+
variant: str
|
|
195
|
+
url: str
|
|
196
|
+
token: str = field(default="", repr=False) # runtime proxy token: used for requests, never shown
|
|
197
|
+
name: str | None = None # the colab CLI's local name, if it made this one
|
|
198
|
+
running: str | None = None
|
|
199
|
+
assigned_at: datetime | None = None # from the CLI's history; Colab boots VMs before handing them out
|
|
200
|
+
status: Status | None = None
|
|
201
|
+
error: str | None = None
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def kind(self) -> str:
|
|
205
|
+
return f"{self.hardware}/{self.shape}"
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def hardware(self) -> str:
|
|
209
|
+
return "CPU" if self.accelerator == "NONE" else self.accelerator
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def label(self) -> str:
|
|
213
|
+
return self.name or "untracked"
|
|
214
|
+
|
|
215
|
+
def idle_for(self, now: datetime) -> timedelta | None:
|
|
216
|
+
"""Idle time on the meter: a VM's kernel can sit idle before Colab hands it out."""
|
|
217
|
+
idle = self.status.idle_for(now) if self.status else None
|
|
218
|
+
if idle is None or self.assigned_at is None:
|
|
219
|
+
return idle
|
|
220
|
+
return min(idle, now - self.assigned_at)
|
|
221
|
+
|
|
222
|
+
def billed_for(self, now: datetime) -> tuple[timedelta, bool] | None:
|
|
223
|
+
"""Time on the meter, and whether it is exact or only an upper bound (VM boot)."""
|
|
224
|
+
if self.assigned_at:
|
|
225
|
+
return now - self.assigned_at, True
|
|
226
|
+
if self.status:
|
|
227
|
+
return self.status.uptime(now), False
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
def concerns(self, now: datetime, resources: Resources | None) -> list[str]:
|
|
231
|
+
"""What is wrong with this runtime, worst first, each short enough for a table cell."""
|
|
232
|
+
if self.error:
|
|
233
|
+
return [f"unreachable: {self.error}"]
|
|
234
|
+
found = []
|
|
235
|
+
idle = self.idle_for(now)
|
|
236
|
+
util = resources.gpu_utilization if resources else None
|
|
237
|
+
if idle and idle >= IDLE_ALERT and (util is None or util < 0.05):
|
|
238
|
+
found.append(f"idle {fmt_duration(idle)}")
|
|
239
|
+
if resources:
|
|
240
|
+
used, total = resources.ram
|
|
241
|
+
if total and used / total > 0.9:
|
|
242
|
+
found.append(f"RAM {used / total:.0%} full")
|
|
243
|
+
used, total = resources.disk
|
|
244
|
+
if total and used / total > 0.9:
|
|
245
|
+
found.append(f"disk {used / total:.0%} full")
|
|
246
|
+
return found
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@dataclass(frozen=True)
|
|
250
|
+
class Rate:
|
|
251
|
+
hourly: float | None
|
|
252
|
+
basis: str # reported | split | learned | unknown
|
|
253
|
+
|
|
254
|
+
@property
|
|
255
|
+
def exact(self) -> bool:
|
|
256
|
+
return self.basis == "reported"
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class RateBook:
|
|
260
|
+
"""Hourly compute-unit rates per hardware kind, learned from the account total.
|
|
261
|
+
|
|
262
|
+
Colab reports only the account's total rate. When every runtime is the
|
|
263
|
+
same kind, total / count is that kind's rate; the book remembers it so a
|
|
264
|
+
mixed set of runtimes can still be split later.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
def __init__(self, path: Path):
|
|
268
|
+
self.path = path
|
|
269
|
+
self.rates: dict[str, float] = json.loads(path.read_text()) if path.exists() else {}
|
|
270
|
+
|
|
271
|
+
def observe(self, account: Account, runtimes: list[Runtime]) -> None:
|
|
272
|
+
kinds = {r.kind for r in runtimes}
|
|
273
|
+
if len(kinds) != 1 or account.rate_hourly <= 0:
|
|
274
|
+
return
|
|
275
|
+
if account.assignments_count != len(runtimes): # the two reads straddled a start or stop
|
|
276
|
+
return
|
|
277
|
+
rate = round(account.rate_hourly / len(runtimes), 4)
|
|
278
|
+
kind = kinds.pop()
|
|
279
|
+
if self.rates.get(kind) != rate:
|
|
280
|
+
self.rates[kind] = rate
|
|
281
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
282
|
+
self.path.write_text(json.dumps(self.rates, indent=2, sort_keys=True))
|
|
283
|
+
|
|
284
|
+
def rate(self, runtime: Runtime, runtimes: list[Runtime], account: Account) -> Rate:
|
|
285
|
+
if len(runtimes) == 1:
|
|
286
|
+
return Rate(account.rate_hourly, "reported")
|
|
287
|
+
if len({r.kind for r in runtimes}) == 1:
|
|
288
|
+
return Rate(account.rate_hourly / len(runtimes), "split")
|
|
289
|
+
if runtime.kind in self.rates:
|
|
290
|
+
return Rate(self.rates[runtime.kind], "learned")
|
|
291
|
+
return Rate(None, "unknown")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class UsageLog:
|
|
295
|
+
"""Balance and hourly rate on disk, for measured burn and the usage graph.
|
|
296
|
+
|
|
297
|
+
Colab settles the balance in steps a few minutes apart, so burn is measured
|
|
298
|
+
between the first and last step inside the window, not between polls.
|
|
299
|
+
"""
|
|
300
|
+
|
|
301
|
+
KEEP = timedelta(days=7)
|
|
302
|
+
|
|
303
|
+
def __init__(self, path: Path):
|
|
304
|
+
self.path = path
|
|
305
|
+
self.samples: list[tuple[datetime, float, float]] = []
|
|
306
|
+
if path.exists():
|
|
307
|
+
for line in path.read_text().splitlines():
|
|
308
|
+
ts, balance, rate = json.loads(line)
|
|
309
|
+
self.samples.append((parse_time(ts), balance, rate))
|
|
310
|
+
|
|
311
|
+
def record(self, now: datetime, balance: float, rate: float) -> None:
|
|
312
|
+
if self.samples and self.samples[-1][1:] == (balance, rate):
|
|
313
|
+
return
|
|
314
|
+
self.samples.append((now, balance, rate))
|
|
315
|
+
self.samples = [s for s in self.samples if now - s[0] <= self.KEEP]
|
|
316
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
317
|
+
self.path.write_text("".join(json.dumps([t.isoformat(), b, r]) + "\n" for t, b, r in self.samples))
|
|
318
|
+
|
|
319
|
+
def burn(self, now: datetime, window: timedelta = timedelta(hours=1)) -> float | None:
|
|
320
|
+
"""Compute units per hour actually charged within the window."""
|
|
321
|
+
steps = []
|
|
322
|
+
for t, b, _ in self.samples:
|
|
323
|
+
if now - t <= window and (not steps or steps[-1][1] != b):
|
|
324
|
+
steps.append((t, b))
|
|
325
|
+
if len(steps) < 3: # the first is a baseline, not a settled step
|
|
326
|
+
return None
|
|
327
|
+
(t0, b0), (t1, b1) = steps[1], steps[-1]
|
|
328
|
+
hours = (t1 - t0).total_seconds() / 3600
|
|
329
|
+
if hours < 0.1 or b1 > b0: # too short, or a top-up landed
|
|
330
|
+
return None
|
|
331
|
+
return (b0 - b1) / hours
|
|
332
|
+
|
|
333
|
+
def rate_series(self, now: datetime, columns: int, window: timedelta = timedelta(hours=24)) -> list[float | None]:
|
|
334
|
+
"""The hourly rate across the window in equal steps, oldest first; None before the first sample."""
|
|
335
|
+
out, i, rate = [], 0, None
|
|
336
|
+
for c in range(columns):
|
|
337
|
+
at = now - window + window * (c + 1) / columns
|
|
338
|
+
while i < len(self.samples) and self.samples[i][0] <= at:
|
|
339
|
+
rate = self.samples[i][2]
|
|
340
|
+
i += 1
|
|
341
|
+
out.append(rate)
|
|
342
|
+
return out
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
@dataclass(frozen=True)
|
|
346
|
+
class PastSession:
|
|
347
|
+
name: str
|
|
348
|
+
endpoint: str
|
|
349
|
+
kind: str
|
|
350
|
+
created: datetime
|
|
351
|
+
ended: datetime | None # None: no terminate event; live, or reclaimed without the CLI noticing
|
|
352
|
+
reason: str | None
|
|
353
|
+
|
|
354
|
+
def lasted(self, now: datetime, live: set[str]) -> tuple[timedelta, str] | None:
|
|
355
|
+
"""How long it ran, and "" if exact, "≤" if an upper bound; None if unknown."""
|
|
356
|
+
if self.ended:
|
|
357
|
+
# "pruned" is when the CLI noticed the VM was gone, not when it went
|
|
358
|
+
return self.ended - self.created, "≤" if self.reason == "pruned" else ""
|
|
359
|
+
if self.endpoint in live:
|
|
360
|
+
return now - self.created, ""
|
|
361
|
+
return None
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@dataclass(frozen=True)
|
|
365
|
+
class Event:
|
|
366
|
+
at: datetime
|
|
367
|
+
summary: str
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def summarize_event(e: dict) -> str:
|
|
371
|
+
"""One line per CLI history event; never code, outputs or stdin replies."""
|
|
372
|
+
match e["event_type"]:
|
|
373
|
+
case "file_operation":
|
|
374
|
+
return f"{e.get('op', 'file')} {Path(e.get('remote') or '').name}"
|
|
375
|
+
case "execution":
|
|
376
|
+
return f"exec cell {e['cell_index']}" if e.get("cell_index") is not None else "exec"
|
|
377
|
+
case "session_created":
|
|
378
|
+
return "created"
|
|
379
|
+
case "session_terminated":
|
|
380
|
+
return f"terminated ({e.get('reason', '?')})"
|
|
381
|
+
case other:
|
|
382
|
+
return other
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
@dataclass
|
|
386
|
+
class History:
|
|
387
|
+
"""The colab CLI's per-session event logs (~/.config/colab-cli/history)."""
|
|
388
|
+
|
|
389
|
+
events: dict[str, list[dict]] = field(default_factory=dict)
|
|
390
|
+
|
|
391
|
+
@classmethod
|
|
392
|
+
def load(cls, directory: Path) -> History:
|
|
393
|
+
events = {}
|
|
394
|
+
for f in directory.glob("*.jsonl"):
|
|
395
|
+
events[f.stem] = [json.loads(l) for l in f.read_text().splitlines() if l.strip()]
|
|
396
|
+
return cls(events)
|
|
397
|
+
|
|
398
|
+
def assigned_at(self) -> dict[str, datetime]:
|
|
399
|
+
"""When the CLI was handed each endpoint."""
|
|
400
|
+
return {
|
|
401
|
+
e["endpoint"]: parse_time(e["timestamp"])
|
|
402
|
+
for events in self.events.values()
|
|
403
|
+
for e in events
|
|
404
|
+
if e["event_type"] == "session_created"
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
def recent(self, name: str, limit: int = 8) -> list[Event]:
|
|
408
|
+
"""Newest first, with runs of the same event folded into one line."""
|
|
409
|
+
runs: list[tuple[str, datetime, int]] = []
|
|
410
|
+
for e in reversed(self.events.get(name, [])):
|
|
411
|
+
summary, at = summarize_event(e), parse_time(e["timestamp"])
|
|
412
|
+
if runs and runs[-1][0] == summary:
|
|
413
|
+
runs[-1] = (summary, runs[-1][1], runs[-1][2] + 1)
|
|
414
|
+
elif len(runs) == limit:
|
|
415
|
+
break
|
|
416
|
+
else:
|
|
417
|
+
runs.append((summary, at, 1))
|
|
418
|
+
return [Event(at, summary if n == 1 else f"{summary} ×{n}") for summary, at, n in runs]
|
|
419
|
+
|
|
420
|
+
def past_sessions(self) -> list[PastSession]:
|
|
421
|
+
"""One entry per `colab new`; the CLI reuses names, so a file can hold several."""
|
|
422
|
+
sessions = []
|
|
423
|
+
for name, events in self.events.items():
|
|
424
|
+
created = None
|
|
425
|
+
for e in events + [{"event_type": "session_created"}]: # sentinel flushes the last
|
|
426
|
+
if e["event_type"] == "session_created":
|
|
427
|
+
if created:
|
|
428
|
+
sessions.append(_past(name, created, None))
|
|
429
|
+
created = e if "timestamp" in e else None
|
|
430
|
+
elif e["event_type"] == "session_terminated" and created:
|
|
431
|
+
sessions.append(_past(name, created, e))
|
|
432
|
+
created = None
|
|
433
|
+
return sorted(sessions, key=lambda s: s.created, reverse=True)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _past(name: str, created: dict, ended: dict | None) -> PastSession:
|
|
437
|
+
hardware = "CPU" if created["accelerator"] == "NONE" else created["accelerator"]
|
|
438
|
+
return PastSession(
|
|
439
|
+
name=name,
|
|
440
|
+
endpoint=created["endpoint"],
|
|
441
|
+
kind=f"{hardware}/{created['machine_shape']}",
|
|
442
|
+
created=parse_time(created["timestamp"]),
|
|
443
|
+
ended=parse_time(ended["timestamp"]) if ended else None,
|
|
444
|
+
reason=ended.get("reason") if ended else None,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def runtimes_from(
|
|
449
|
+
assignments: list[dict],
|
|
450
|
+
local: dict[str, dict],
|
|
451
|
+
status: dict[str, Status | Exception],
|
|
452
|
+
assigned_at: dict[str, datetime] | None = None,
|
|
453
|
+
) -> list[Runtime]:
|
|
454
|
+
"""Join the server's assignments with the CLI's names, history and kernel status."""
|
|
455
|
+
by_endpoint = {s["endpoint"]: s for s in local.values()}
|
|
456
|
+
runtimes = []
|
|
457
|
+
for a in assignments:
|
|
458
|
+
mine = by_endpoint.get(a["endpoint"], {})
|
|
459
|
+
t = status.get(a["endpoint"])
|
|
460
|
+
runtimes.append(
|
|
461
|
+
Runtime(
|
|
462
|
+
endpoint=a["endpoint"],
|
|
463
|
+
accelerator=a["accelerator"],
|
|
464
|
+
shape=SHAPES.get(a.get("machineShape", 0), str(a.get("machineShape"))),
|
|
465
|
+
variant=VARIANTS.get(a.get("variant", 0), str(a.get("variant"))),
|
|
466
|
+
url=a["runtimeProxyInfo"]["url"],
|
|
467
|
+
token=a["runtimeProxyInfo"]["token"],
|
|
468
|
+
name=mine.get("name"),
|
|
469
|
+
running=mine.get("running"),
|
|
470
|
+
assigned_at=(assigned_at or {}).get(a["endpoint"]),
|
|
471
|
+
status=t if isinstance(t, Status) else None,
|
|
472
|
+
error=f"{type(t).__name__}: {t}" if isinstance(t, Exception) else None,
|
|
473
|
+
)
|
|
474
|
+
)
|
|
475
|
+
return runtimes
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def utc_now() -> datetime:
|
|
479
|
+
return datetime.now(timezone.utc)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
IDLE_ALERT = timedelta(minutes=10)
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def fmt_duration(d: timedelta) -> str:
|
|
486
|
+
minutes = int(d.total_seconds() // 60)
|
|
487
|
+
if minutes < 60:
|
|
488
|
+
return f"{minutes}m"
|
|
489
|
+
hours, minutes = divmod(minutes, 60)
|
|
490
|
+
if hours < 48:
|
|
491
|
+
return f"{hours}h{minutes:02d}m"
|
|
492
|
+
return f"{hours // 24}d{hours % 24}h"
|