cotop 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cotop/__init__.py +0 -0
- cotop/app.py +572 -0
- cotop/colab.py +158 -0
- cotop/model.py +492 -0
- cotop/widgets.py +132 -0
- cotop-0.1.0.dist-info/METADATA +100 -0
- cotop-0.1.0.dist-info/RECORD +10 -0
- cotop-0.1.0.dist-info/WHEEL +4 -0
- cotop-0.1.0.dist-info/entry_points.txt +2 -0
- cotop-0.1.0.dist-info/licenses/LICENSE +202 -0
cotop/__init__.py
ADDED
|
File without changes
|
cotop/app.py
ADDED
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
"""cotop: top for Google Colab."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import datetime, timedelta
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from rich.text import Text
|
|
13
|
+
from textual import work
|
|
14
|
+
from textual.app import App, ComposeResult
|
|
15
|
+
from textual.binding import Binding
|
|
16
|
+
from textual.containers import Horizontal, Vertical
|
|
17
|
+
from textual.screen import ModalScreen, Screen
|
|
18
|
+
from textual.theme import Theme
|
|
19
|
+
from textual.widgets import Button, DataTable, Footer, Label, Static
|
|
20
|
+
from textual.worker import Worker
|
|
21
|
+
|
|
22
|
+
from cotop.colab import CLI_HOME, Colab, local_sessions
|
|
23
|
+
from cotop.model import (
|
|
24
|
+
Account,
|
|
25
|
+
History,
|
|
26
|
+
RateBook,
|
|
27
|
+
Resources,
|
|
28
|
+
Runtime,
|
|
29
|
+
Samples,
|
|
30
|
+
UsageLog,
|
|
31
|
+
fmt_duration,
|
|
32
|
+
runtimes_from,
|
|
33
|
+
utc_now,
|
|
34
|
+
)
|
|
35
|
+
from cotop.widgets import AMBER, DIM, ORANGE, PRESSURE, RED, WORK, Graph, TimeAxis, meter, sparkline
|
|
36
|
+
|
|
37
|
+
CACHE = Path("~/.cache/cotop").expanduser()
|
|
38
|
+
POLL_SECONDS = 15
|
|
39
|
+
SAMPLE_SECONDS = 2
|
|
40
|
+
LOW_RUNWAY = timedelta(hours=24)
|
|
41
|
+
ENDINGS = {"user_requested": "stopped", "pruned": "gone"} # the CLI's reason codes, as people say them
|
|
42
|
+
|
|
43
|
+
THEME = Theme(
|
|
44
|
+
name="cotop",
|
|
45
|
+
primary=AMBER,
|
|
46
|
+
secondary=ORANGE,
|
|
47
|
+
accent=AMBER,
|
|
48
|
+
warning=ORANGE,
|
|
49
|
+
error=RED,
|
|
50
|
+
success=AMBER,
|
|
51
|
+
foreground="#D5DAE1",
|
|
52
|
+
background="#101318",
|
|
53
|
+
surface="#101318",
|
|
54
|
+
panel="#1B2028",
|
|
55
|
+
dark=True,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class Snapshot:
|
|
61
|
+
taken_at: datetime
|
|
62
|
+
account: Account
|
|
63
|
+
runtimes: list[Runtime]
|
|
64
|
+
history: History
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def gb(n: int) -> str:
|
|
68
|
+
return f"{n / 1e9:.0f}G"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def clock(at: datetime) -> str:
|
|
72
|
+
return f"{at.astimezone():%H:%M}"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Usage(Vertical):
|
|
76
|
+
"""The account: what is left, how fast it goes, and the rate over the last day."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, log: UsageLog):
|
|
79
|
+
super().__init__()
|
|
80
|
+
self.usage_log = log
|
|
81
|
+
|
|
82
|
+
def compose(self) -> ComposeResult:
|
|
83
|
+
with Horizontal():
|
|
84
|
+
yield Static(id="usage-text")
|
|
85
|
+
yield Graph(WORK, source=self.rate_values, id="usage-graph")
|
|
86
|
+
|
|
87
|
+
def rate_values(self, n: int) -> list[float | None]:
|
|
88
|
+
series = self.usage_log.rate_series(utc_now(), n)
|
|
89
|
+
top = max((r for r in series if r), default=0) or 1
|
|
90
|
+
return [None if r is None else r / top for r in series]
|
|
91
|
+
|
|
92
|
+
def show(self, s: Snapshot, log: UsageLog) -> None:
|
|
93
|
+
a, now = s.account, s.taken_at
|
|
94
|
+
self.border_title = a.email
|
|
95
|
+
runway = a.runway
|
|
96
|
+
left = Text.assemble((f"{a.balance:,.2f}", "bold"), " compute units left")
|
|
97
|
+
if runway is None:
|
|
98
|
+
pace = Text("nothing running", style=DIM)
|
|
99
|
+
else:
|
|
100
|
+
pace = Text.assemble(
|
|
101
|
+
f"{a.rate_hourly:.2f} an hour, lasts ",
|
|
102
|
+
(fmt_duration(runway), f"bold {RED}" if runway < LOW_RUNWAY else "bold"),
|
|
103
|
+
(f" until {(now + runway).astimezone():%a %H:%M}", DIM),
|
|
104
|
+
)
|
|
105
|
+
burn = log.burn(now)
|
|
106
|
+
charged = Text(f"charged {burn:.2f} an hour over the last hour" if burn is not None else "", style=DIM)
|
|
107
|
+
self.query_one("#usage-text", Static).update(Text("\n").join([left, pace, charged]))
|
|
108
|
+
self.query_one("#usage-graph", Graph).refresh()
|
|
109
|
+
self.border_subtitle = "rate over 24h"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class RuntimePanel(Vertical):
|
|
113
|
+
"""The selected runtime: its GPU graph, meters and what it costs.
|
|
114
|
+
|
|
115
|
+
There is one panel, pointed at whichever runtime is selected, so changing
|
|
116
|
+
the selection never tears widgets down while readings arrive.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
def __init__(self, samples: Samples, **kwargs):
|
|
120
|
+
super().__init__(**kwargs)
|
|
121
|
+
self.samples = samples
|
|
122
|
+
self.runtime: Runtime | None = None
|
|
123
|
+
|
|
124
|
+
def compose(self) -> ComposeResult:
|
|
125
|
+
with Horizontal():
|
|
126
|
+
with Vertical(classes="plot"):
|
|
127
|
+
yield Static(classes="plot-label")
|
|
128
|
+
yield Graph(WORK)
|
|
129
|
+
yield TimeAxis(self.reading_time)
|
|
130
|
+
with Vertical(classes="side"):
|
|
131
|
+
yield Static(classes="meters")
|
|
132
|
+
yield Static(classes="facts")
|
|
133
|
+
yield Static(classes="activity")
|
|
134
|
+
|
|
135
|
+
def reading_time(self, n: int) -> datetime | None:
|
|
136
|
+
return self.samples.time_of(self.runtime.endpoint, n) if self.runtime else None
|
|
137
|
+
|
|
138
|
+
def show(self, r: Runtime, s: Snapshot, rates: RateBook) -> None:
|
|
139
|
+
self.runtime = r
|
|
140
|
+
now = s.taken_at
|
|
141
|
+
latest = self.samples.latest(r.endpoint)
|
|
142
|
+
self.border_title = f"{r.label} {r.hardware}{' high-RAM' if r.shape == 'HIGH_RAM' else ''}"
|
|
143
|
+
self.border_subtitle = state_text(r, now, latest).plain + (f" {r.running}" if r.running else "")
|
|
144
|
+
self.show_readings()
|
|
145
|
+
|
|
146
|
+
rate = rates.rate(r, s.runtimes, s.account)
|
|
147
|
+
billed = r.billed_for(now)
|
|
148
|
+
lines = []
|
|
149
|
+
if r.error:
|
|
150
|
+
lines.append(Text(f"unreachable\n{r.error}", style=RED))
|
|
151
|
+
for concern in r.concerns(now, latest):
|
|
152
|
+
billing = ", still billing" if concern.startswith("idle") else ""
|
|
153
|
+
lines.append(Text(concern + billing, style=ORANGE))
|
|
154
|
+
if billed:
|
|
155
|
+
duration, exact = billed
|
|
156
|
+
since = f"since {clock(r.assigned_at)}" if r.assigned_at else "since VM boot"
|
|
157
|
+
lines.append(Text.assemble(("up ", DIM), ("" if exact else "≤") + fmt_duration(duration), (f" {since}", DIM)))
|
|
158
|
+
if rate.hourly:
|
|
159
|
+
spent = rate.hourly * duration.total_seconds() / 3600
|
|
160
|
+
lines.append(Text.assemble(("spent ", DIM), f"{'≈' if exact else '≤'}{spent:.1f}", (f" at {rate_text(rate)}/hr", DIM)))
|
|
161
|
+
if r.status:
|
|
162
|
+
lines.append(Text.assemble(("clients ", DIM), str(r.status.clients)))
|
|
163
|
+
self.query_one(".facts", Static).update(Text("\n").join(lines))
|
|
164
|
+
events = s.history.recent(r.name, limit=12) if r.name else []
|
|
165
|
+
activity = Text("\n").join(Text.assemble((clock(e.at) + " ", DIM), e.summary) for e in events)
|
|
166
|
+
self.query_one(".activity", Static).update(activity or Text("no CLI activity", style=DIM))
|
|
167
|
+
|
|
168
|
+
def show_readings(self) -> None:
|
|
169
|
+
if self.runtime is None:
|
|
170
|
+
return
|
|
171
|
+
latest = self.samples.latest(self.runtime.endpoint)
|
|
172
|
+
graph = self.query_one(Graph)
|
|
173
|
+
has_gpu = bool(latest and latest.gpus)
|
|
174
|
+
metric = (lambda x: x.gpu_utilization) if has_gpu or latest is None else (lambda x: fraction(*x.ram))
|
|
175
|
+
graph.gradient = WORK if has_gpu or latest is None else PRESSURE
|
|
176
|
+
graph.values = self.samples.series(self.runtime.endpoint, metric)
|
|
177
|
+
self.query_one(TimeAxis).refresh()
|
|
178
|
+
now_value = metric(latest) if latest else None
|
|
179
|
+
name = latest.gpus[0].name if has_gpu else ("RAM" if latest else "GPU")
|
|
180
|
+
label = Text.assemble((f"{now_value:.0%}" if now_value is not None else "–", "bold"), (" " + name, DIM))
|
|
181
|
+
self.query_one(".plot-label", Static).update(label)
|
|
182
|
+
self.query_one(".meters", Static).update(meters(latest, width=14))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def fraction(used: int, total: int) -> float | None:
|
|
186
|
+
"""Colab's client treats every size as optional, so a total can be 0."""
|
|
187
|
+
return used / total if total else None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def meters(res: Resources | None, width: int) -> Text:
|
|
191
|
+
if res is None:
|
|
192
|
+
return Text("waiting for readings", style=DIM)
|
|
193
|
+
rows = []
|
|
194
|
+
for label, (used, total), gradient, show_sizes in (
|
|
195
|
+
("vram", res.vram or (0, 0), WORK, True),
|
|
196
|
+
("ram ", res.ram, PRESSURE, True),
|
|
197
|
+
("disk", res.disk, PRESSURE, False),
|
|
198
|
+
):
|
|
199
|
+
f = fraction(used, total)
|
|
200
|
+
if f is not None:
|
|
201
|
+
rows.append((label, f, f"{gb(used)}/{gb(total)}" if show_sizes else f"{f:.0%}", gradient))
|
|
202
|
+
return Text("\n").join(
|
|
203
|
+
Text.assemble((label + " ", DIM), meter(f, width, g), " " + value) for label, f, value, g in rows
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def rate_text(rate) -> str:
|
|
208
|
+
if rate.hourly is None:
|
|
209
|
+
return "?"
|
|
210
|
+
return f"{'' if rate.exact else '≈'}{rate.hourly:.2f}"
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def state_text(r: Runtime, now: datetime, latest: Resources | None) -> Text:
|
|
214
|
+
if r.error:
|
|
215
|
+
return Text("unreachable", style=RED)
|
|
216
|
+
concerns = r.concerns(now, latest)
|
|
217
|
+
if concerns:
|
|
218
|
+
return Text(concerns[0], style=ORANGE)
|
|
219
|
+
if r.status is None:
|
|
220
|
+
return Text("starting", style=DIM)
|
|
221
|
+
if r.status.busy:
|
|
222
|
+
return Text("busy", style=AMBER)
|
|
223
|
+
idle = r.idle_for(now)
|
|
224
|
+
return Text(f"idle {fmt_duration(idle)}" if idle else "idle", style=DIM)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class Runtimes(DataTable):
|
|
228
|
+
"""The list to pick from; the GPU column moves every two seconds."""
|
|
229
|
+
|
|
230
|
+
COLUMNS = ("Name", "Hardware", "GPU", "CU/hr", "Spent", "Up", "State")
|
|
231
|
+
|
|
232
|
+
def on_mount(self) -> None:
|
|
233
|
+
for c in self.COLUMNS:
|
|
234
|
+
self.add_column(c, key=c)
|
|
235
|
+
|
|
236
|
+
def show(self, s: Snapshot, rates: RateBook, samples: Samples) -> None:
|
|
237
|
+
keys = [r.endpoint for r in s.runtimes]
|
|
238
|
+
if [row.value for row in self.rows] != keys:
|
|
239
|
+
selected = self.selected_endpoint
|
|
240
|
+
self.clear()
|
|
241
|
+
for r in s.runtimes:
|
|
242
|
+
self.add_row(*["" for _ in self.COLUMNS], key=r.endpoint)
|
|
243
|
+
if selected in keys:
|
|
244
|
+
self.move_cursor(row=keys.index(selected))
|
|
245
|
+
now = s.taken_at
|
|
246
|
+
for r in s.runtimes:
|
|
247
|
+
rate = rates.rate(r, s.runtimes, s.account)
|
|
248
|
+
billed = r.billed_for(now)
|
|
249
|
+
up, spent = "?", "?"
|
|
250
|
+
if billed:
|
|
251
|
+
duration, exact = billed
|
|
252
|
+
up = ("" if exact else "≤") + fmt_duration(duration)
|
|
253
|
+
if rate.hourly:
|
|
254
|
+
spent = f"{'≈' if exact else '≤'}{rate.hourly * duration.total_seconds() / 3600:.1f}"
|
|
255
|
+
cells = {
|
|
256
|
+
"Name": Text(r.label, style="" if r.name else DIM),
|
|
257
|
+
"Hardware": r.hardware + (" hi-RAM" if r.shape == "HIGH_RAM" else ""),
|
|
258
|
+
"CU/hr": rate_text(rate),
|
|
259
|
+
"Spent": spent,
|
|
260
|
+
"Up": up,
|
|
261
|
+
"State": state_text(r, now, samples.latest(r.endpoint)),
|
|
262
|
+
}
|
|
263
|
+
for column, value in cells.items():
|
|
264
|
+
self.update_cell(r.endpoint, column, value)
|
|
265
|
+
self.show_readings(s, samples)
|
|
266
|
+
|
|
267
|
+
def show_readings(self, s: Snapshot, samples: Samples) -> None:
|
|
268
|
+
for r in s.runtimes:
|
|
269
|
+
latest = samples.latest(r.endpoint)
|
|
270
|
+
util = latest.gpu_utilization if latest else None
|
|
271
|
+
cell = sparkline(samples.series(r.endpoint, lambda x: x.gpu_utilization), 8)
|
|
272
|
+
cell.append(f" {util:4.0%}" if util is not None else " –")
|
|
273
|
+
self.update_cell(r.endpoint, "GPU", cell)
|
|
274
|
+
|
|
275
|
+
@property
|
|
276
|
+
def selected_endpoint(self) -> str | None:
|
|
277
|
+
if not self.row_count:
|
|
278
|
+
return None
|
|
279
|
+
return self.coordinate_to_cell_key(self.cursor_coordinate).row_key.value
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class Confirm(ModalScreen[bool]):
|
|
283
|
+
"""A yes/no question; No is focused, so Enter alone never stops a VM."""
|
|
284
|
+
|
|
285
|
+
CSS = """
|
|
286
|
+
Confirm { align: center middle; }
|
|
287
|
+
#dialog { width: 64; height: auto; border: round $warning; padding: 1 2; background: $panel; }
|
|
288
|
+
#buttons { height: auto; margin-top: 1; align-horizontal: right; }
|
|
289
|
+
#buttons Button { margin-left: 2; }
|
|
290
|
+
"""
|
|
291
|
+
BINDINGS = [("y", "answer(True)", "Yes"), ("n,escape", "answer(False)", "No")]
|
|
292
|
+
|
|
293
|
+
def __init__(self, question: str, yes: str):
|
|
294
|
+
super().__init__()
|
|
295
|
+
self.question, self.yes = question, yes
|
|
296
|
+
|
|
297
|
+
def compose(self) -> ComposeResult:
|
|
298
|
+
with Vertical(id="dialog"):
|
|
299
|
+
yield Label(self.question)
|
|
300
|
+
with Horizontal(id="buttons"):
|
|
301
|
+
yield Button(f"{self.yes} (y)", variant="error", id="yes")
|
|
302
|
+
yield Button("Cancel (n)", id="no")
|
|
303
|
+
|
|
304
|
+
def on_mount(self) -> None:
|
|
305
|
+
self.query_one("#no", Button).focus()
|
|
306
|
+
|
|
307
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
308
|
+
self.dismiss(event.button.id == "yes")
|
|
309
|
+
|
|
310
|
+
def action_answer(self, yes: bool) -> None:
|
|
311
|
+
self.dismiss(yes)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
class HistoryScreen(Screen):
|
|
315
|
+
"""Past sessions and what they cost; the account's eligible hardware."""
|
|
316
|
+
|
|
317
|
+
BINDINGS = [Binding("h,escape", "app.pop_screen", "Back")]
|
|
318
|
+
|
|
319
|
+
def __init__(self, s: Snapshot, rates: RateBook):
|
|
320
|
+
super().__init__()
|
|
321
|
+
self.snapshot, self.rates = s, rates
|
|
322
|
+
|
|
323
|
+
def compose(self) -> ComposeResult:
|
|
324
|
+
s = self.snapshot
|
|
325
|
+
table = DataTable(cursor_type="row", zebra_stripes=True, classes="panel")
|
|
326
|
+
table.border_title = "Sessions started with the colab CLI"
|
|
327
|
+
yield table
|
|
328
|
+
a = s.account
|
|
329
|
+
eligible = Static(
|
|
330
|
+
Text.assemble(("GPU ", DIM), " ".join(a.eligible_gpus) or "none", (" TPU ", DIM), " ".join(a.eligible_tpus) or "none"),
|
|
331
|
+
classes="panel",
|
|
332
|
+
)
|
|
333
|
+
eligible.border_title = "Hardware this account can use"
|
|
334
|
+
yield eligible
|
|
335
|
+
yield Footer()
|
|
336
|
+
|
|
337
|
+
def on_mount(self) -> None:
|
|
338
|
+
s = self.snapshot
|
|
339
|
+
table = self.query_one(DataTable)
|
|
340
|
+
table.add_columns("Name", "Hardware", "Started", "Lasted", "CU", "Ended")
|
|
341
|
+
live = {r.endpoint for r in s.runtimes}
|
|
342
|
+
for p in s.history.past_sessions():
|
|
343
|
+
rate = self.rates.rates.get(p.kind)
|
|
344
|
+
lasted, cost = "?", "?"
|
|
345
|
+
if run := p.lasted(s.taken_at, live):
|
|
346
|
+
duration, bound = run
|
|
347
|
+
lasted = bound + fmt_duration(duration)
|
|
348
|
+
if rate:
|
|
349
|
+
cost = f"{bound or '≈'}{rate * duration.total_seconds() / 3600:.1f}"
|
|
350
|
+
ended = ENDINGS.get(p.reason, p.reason) if p.reason else ("running" if p.endpoint in live else "unknown")
|
|
351
|
+
table.add_row(
|
|
352
|
+
p.name,
|
|
353
|
+
p.kind.replace("/STANDARD", "").replace("/HIGH_RAM", " hi-RAM"),
|
|
354
|
+
f"{p.created.astimezone():%b %-d %H:%M}",
|
|
355
|
+
lasted,
|
|
356
|
+
cost,
|
|
357
|
+
ended,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class Dashboard(App):
|
|
362
|
+
TITLE = "cotop"
|
|
363
|
+
CSS = """
|
|
364
|
+
Screen { layout: vertical; }
|
|
365
|
+
.panel, Usage, RuntimePanel, Runtimes {
|
|
366
|
+
border: round $panel-lighten-2; border-title-color: $foreground; border-subtitle-color: $text-muted;
|
|
367
|
+
padding: 0 1; background: $background;
|
|
368
|
+
}
|
|
369
|
+
Usage { height: 5; }
|
|
370
|
+
Usage Horizontal { height: 3; }
|
|
371
|
+
#usage-text { width: 46; }
|
|
372
|
+
#usage-graph { height: 3; }
|
|
373
|
+
Runtimes { height: auto; max-height: 9; }
|
|
374
|
+
Runtimes:focus { border: round $accent; }
|
|
375
|
+
Runtimes > .datatable--cursor, Runtimes:focus > .datatable--cursor { background: $panel-lighten-1; text-style: bold; }
|
|
376
|
+
Runtimes > .datatable--header { background: $background; color: $text-muted; }
|
|
377
|
+
RuntimePanel { height: 1fr; min-height: 8; }
|
|
378
|
+
RuntimePanel Horizontal { height: 1fr; }
|
|
379
|
+
RuntimePanel .plot { width: 1fr; }
|
|
380
|
+
RuntimePanel .plot-label { height: 1; }
|
|
381
|
+
RuntimePanel .side { width: 34; padding-left: 2; }
|
|
382
|
+
RuntimePanel .meters { height: auto; margin-bottom: 1; }
|
|
383
|
+
RuntimePanel .facts { height: auto; margin-bottom: 1; }
|
|
384
|
+
RuntimePanel .activity { height: 1fr; }
|
|
385
|
+
#empty { height: 1fr; content-align: center middle; color: $text-muted; display: none; }
|
|
386
|
+
HistoryScreen DataTable { height: 1fr; }
|
|
387
|
+
HistoryScreen .panel { height: auto; }
|
|
388
|
+
"""
|
|
389
|
+
BINDINGS = [
|
|
390
|
+
("r", "refresh", "Refresh"),
|
|
391
|
+
("s", "stop", "Stop"),
|
|
392
|
+
("k", "restart_kernel", "Restart kernel"),
|
|
393
|
+
("x", "ssh", "SSH"),
|
|
394
|
+
("h", "history", "History"),
|
|
395
|
+
Binding("q", "quit", "Quit"),
|
|
396
|
+
]
|
|
397
|
+
|
|
398
|
+
def __init__(self, colab: Colab):
|
|
399
|
+
super().__init__()
|
|
400
|
+
self.colab = colab
|
|
401
|
+
self.rates = RateBook(CACHE / "rates.json")
|
|
402
|
+
self.usage = UsageLog(CACHE / "usage.jsonl")
|
|
403
|
+
self.samples = Samples()
|
|
404
|
+
self.email: str | None = None
|
|
405
|
+
self.snapshot: Snapshot | None = None
|
|
406
|
+
self.polling: Worker | None = None
|
|
407
|
+
self.sampling: Worker | None = None
|
|
408
|
+
|
|
409
|
+
def compose(self) -> ComposeResult:
|
|
410
|
+
yield Usage(self.usage)
|
|
411
|
+
yield Runtimes(cursor_type="row", cursor_foreground_priority="renderable")
|
|
412
|
+
yield RuntimePanel(self.samples)
|
|
413
|
+
yield Static("No runtimes. Start one with colab new.", id="empty")
|
|
414
|
+
yield Footer()
|
|
415
|
+
|
|
416
|
+
def on_mount(self) -> None:
|
|
417
|
+
self.register_theme(THEME)
|
|
418
|
+
self.theme = "cotop"
|
|
419
|
+
self.query_one(Runtimes).border_title = "Runtimes"
|
|
420
|
+
self.action_refresh()
|
|
421
|
+
self.set_interval(POLL_SECONDS, self.action_refresh)
|
|
422
|
+
self.set_interval(SAMPLE_SECONDS, self.action_sample)
|
|
423
|
+
|
|
424
|
+
# Polling: the account and kernel status every 15 s, resources every 2 s.
|
|
425
|
+
|
|
426
|
+
def action_refresh(self) -> None:
|
|
427
|
+
if not (self.polling and self.polling.is_running):
|
|
428
|
+
self.polling = self.poll()
|
|
429
|
+
|
|
430
|
+
def action_sample(self) -> None:
|
|
431
|
+
if self.snapshot and not (self.sampling and self.sampling.is_running):
|
|
432
|
+
self.sampling = self.sample(self.snapshot.runtimes)
|
|
433
|
+
|
|
434
|
+
@work(thread=True)
|
|
435
|
+
def poll(self) -> None:
|
|
436
|
+
try:
|
|
437
|
+
self.email = self.email or self.colab.email()
|
|
438
|
+
account = Account.parse(self.email, self.colab.ccu_info())
|
|
439
|
+
assignments = self.colab.assignments()
|
|
440
|
+
history = History.load(CLI_HOME / "history")
|
|
441
|
+
status = self.colab.status(assignments)
|
|
442
|
+
runtimes = runtimes_from(assignments, local_sessions(), status, history.assigned_at())
|
|
443
|
+
snapshot = Snapshot(utc_now(), account, runtimes, history)
|
|
444
|
+
except Exception as e:
|
|
445
|
+
self.call_from_thread(self.show_error, e)
|
|
446
|
+
return
|
|
447
|
+
self.call_from_thread(self.show, snapshot)
|
|
448
|
+
|
|
449
|
+
@work(thread=True)
|
|
450
|
+
def sample(self, runtimes: list[Runtime]) -> None:
|
|
451
|
+
at = utc_now()
|
|
452
|
+
readings = self.colab.resources(runtimes)
|
|
453
|
+
self.call_from_thread(self.add_readings, at, readings)
|
|
454
|
+
|
|
455
|
+
def add_readings(self, at: datetime, readings: dict[str, Resources | None]) -> None:
|
|
456
|
+
for endpoint, reading in readings.items():
|
|
457
|
+
self.samples.add(endpoint, at, reading)
|
|
458
|
+
if self.snapshot:
|
|
459
|
+
self.query_one(RuntimePanel).show_readings()
|
|
460
|
+
self.query_one(Runtimes).show_readings(self.snapshot, self.samples)
|
|
461
|
+
|
|
462
|
+
# Showing.
|
|
463
|
+
|
|
464
|
+
def show(self, s: Snapshot) -> None:
|
|
465
|
+
self.snapshot = s
|
|
466
|
+
self.usage.record(s.taken_at, s.account.balance, s.account.rate_hourly)
|
|
467
|
+
self.rates.observe(s.account, s.runtimes)
|
|
468
|
+
self.samples.keep_only({r.endpoint for r in s.runtimes})
|
|
469
|
+
self.query_one(Usage).show(s, self.usage)
|
|
470
|
+
# The table stays visible when empty: hiding a focused widget blurs it, and
|
|
471
|
+
# nothing would give the selection's only home its focus back.
|
|
472
|
+
table = self.query_one(Runtimes)
|
|
473
|
+
table.show(s, self.rates, self.samples)
|
|
474
|
+
self.show_selected()
|
|
475
|
+
|
|
476
|
+
def show_selected(self) -> None:
|
|
477
|
+
"""Point the panel at the runtime under the table's cursor, the selection's only home."""
|
|
478
|
+
r = self.selected_runtime()
|
|
479
|
+
panel = self.query_one(RuntimePanel)
|
|
480
|
+
panel.display = r is not None
|
|
481
|
+
self.query_one("#empty").display = bool(self.snapshot) and not self.snapshot.runtimes
|
|
482
|
+
if r:
|
|
483
|
+
panel.show(r, self.snapshot, self.rates)
|
|
484
|
+
|
|
485
|
+
def show_error(self, e: Exception) -> None:
|
|
486
|
+
age = f" Showing data from {fmt_duration(utc_now() - self.snapshot.taken_at)} ago." if self.snapshot else ""
|
|
487
|
+
self.notify(f"Refresh failed: {type(e).__name__}: {e}.{age}", severity="error", timeout=POLL_SECONDS)
|
|
488
|
+
|
|
489
|
+
# Selection and actions.
|
|
490
|
+
|
|
491
|
+
def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
|
|
492
|
+
if isinstance(event.data_table, Runtimes) and self.snapshot:
|
|
493
|
+
self.show_selected()
|
|
494
|
+
|
|
495
|
+
def selected_runtime(self) -> Runtime | None:
|
|
496
|
+
endpoint = self.query_one(Runtimes).selected_endpoint
|
|
497
|
+
if not self.snapshot or endpoint is None:
|
|
498
|
+
return None
|
|
499
|
+
return next((r for r in self.snapshot.runtimes if r.endpoint == endpoint), None)
|
|
500
|
+
|
|
501
|
+
def named(self, verb: str) -> Runtime | None:
|
|
502
|
+
r = self.selected_runtime()
|
|
503
|
+
if r and r.name is None:
|
|
504
|
+
self.notify(f"Can only {verb} runtimes started with the colab CLI.", severity="warning")
|
|
505
|
+
return None
|
|
506
|
+
return r
|
|
507
|
+
|
|
508
|
+
def action_stop(self) -> None:
|
|
509
|
+
if not (r := self.selected_runtime()):
|
|
510
|
+
return
|
|
511
|
+
busy = f" It is busy running {r.running}." if r.status and r.status.busy and r.running else ""
|
|
512
|
+
question = f"Stop {r.name or r.endpoint} ({r.hardware})? Its VM and files will be deleted.{busy}"
|
|
513
|
+
self.push_screen(Confirm(question, "Stop"), lambda yes: yes and self.act(self.colab.stop, r))
|
|
514
|
+
|
|
515
|
+
def action_restart_kernel(self) -> None:
|
|
516
|
+
if not (r := self.named("restart the kernel of")):
|
|
517
|
+
return
|
|
518
|
+
busy = " Its running code is interrupted." if r.status and r.status.busy else ""
|
|
519
|
+
question = f"Restart {r.name}'s kernel? Everything in memory is lost; files stay.{busy}"
|
|
520
|
+
self.push_screen(Confirm(question, "Restart"), lambda yes: yes and self.act(self.colab.restart_kernel, r))
|
|
521
|
+
|
|
522
|
+
def action_ssh(self) -> None:
|
|
523
|
+
if r := self.named("ssh into"):
|
|
524
|
+
self.open_ssh(r)
|
|
525
|
+
|
|
526
|
+
def action_history(self) -> None:
|
|
527
|
+
if self.snapshot:
|
|
528
|
+
self.push_screen(HistoryScreen(self.snapshot, self.rates))
|
|
529
|
+
|
|
530
|
+
@work(thread=True)
|
|
531
|
+
def act(self, action, runtime: Runtime) -> None:
|
|
532
|
+
label = runtime.name or runtime.endpoint
|
|
533
|
+
self.call_from_thread(self.notify, f"{action.__name__.replace('_', ' ')} {label}…")
|
|
534
|
+
try:
|
|
535
|
+
result = action(runtime)
|
|
536
|
+
except Exception as e:
|
|
537
|
+
self.call_from_thread(self.notify, f"{label}: {e}", severity="error", timeout=10)
|
|
538
|
+
else:
|
|
539
|
+
self.call_from_thread(self.notify, f"{label}: {result}")
|
|
540
|
+
self.call_from_thread(self.action_refresh)
|
|
541
|
+
|
|
542
|
+
@work(thread=True)
|
|
543
|
+
def open_ssh(self, runtime: Runtime) -> None:
|
|
544
|
+
try:
|
|
545
|
+
argv = self.colab.ssh_command(runtime)
|
|
546
|
+
except Exception as e:
|
|
547
|
+
self.call_from_thread(self.notify, f"{runtime.name}: {e}", severity="error", timeout=10)
|
|
548
|
+
return
|
|
549
|
+
self.call_from_thread(self.run_in_terminal, argv)
|
|
550
|
+
|
|
551
|
+
def run_in_terminal(self, argv: list[str]) -> None:
|
|
552
|
+
# suspend() blocks the event loop, so no timer redraws behind the shell
|
|
553
|
+
with self.suspend():
|
|
554
|
+
code = subprocess.run(argv).returncode
|
|
555
|
+
if code != 0: # keep colab's error on screen; resuming would paint over it
|
|
556
|
+
input(f"\ncolab ssh exited {code}. Press Enter to return to cotop.")
|
|
557
|
+
self.action_refresh()
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def main() -> None:
|
|
561
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
562
|
+
parser.add_argument("-i", "--identity", help="SSH key for x (passed to colab ssh -i)")
|
|
563
|
+
args = parser.parse_args()
|
|
564
|
+
try:
|
|
565
|
+
colab = Colab.sign_in(ssh_identity=args.identity) # before the UI: sign-in may prompt in the terminal
|
|
566
|
+
except Exception as e:
|
|
567
|
+
sys.exit(f"cotop: could not sign in with the colab CLI's credentials: {e}")
|
|
568
|
+
Dashboard(colab).run()
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
if __name__ == "__main__":
|
|
572
|
+
main()
|