cotop 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cotop-0.1.0 → cotop-0.2.0}/PKG-INFO +7 -7
- {cotop-0.1.0 → cotop-0.2.0}/README.md +6 -5
- {cotop-0.1.0 → cotop-0.2.0}/docs/data-sources.md +24 -5
- cotop-0.2.0/docs/images/dashboard.png +0 -0
- cotop-0.2.0/docs/images/stop.png +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/pyproject.toml +1 -2
- cotop-0.2.0/scripts/screenshots.py +38 -0
- {cotop-0.1.0 → cotop-0.2.0}/src/cotop/app.py +72 -20
- {cotop-0.1.0 → cotop-0.2.0}/src/cotop/colab.py +12 -10
- cotop-0.2.0/src/cotop/demo.py +221 -0
- {cotop-0.1.0 → cotop-0.2.0}/src/cotop/model.py +27 -7
- {cotop-0.1.0 → cotop-0.2.0}/tests/test_app.py +86 -25
- {cotop-0.1.0 → cotop-0.2.0}/tests/test_model.py +7 -0
- {cotop-0.1.0 → cotop-0.2.0}/uv.lock +1 -3
- cotop-0.1.0/docs/images/dashboard.png +0 -0
- cotop-0.1.0/docs/images/stop.png +0 -0
- cotop-0.1.0/scripts/screenshots.py +0 -205
- {cotop-0.1.0 → cotop-0.2.0}/.github/workflows/publish.yml +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/.gitignore +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/LICENSE +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/src/cotop/__init__.py +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/src/cotop/widgets.py +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/tests/__init__.py +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/tests/fixtures/resources_g4.txt +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/tests/test_colab.py +0 -0
- {cotop-0.1.0 → cotop-0.2.0}/tests/test_widgets.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cotop
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A top-like terminal dashboard for Google Colab runtimes and compute units
|
|
5
5
|
Project-URL: Homepage, https://github.com/benngarcia/cotop
|
|
6
6
|
Project-URL: Issues, https://github.com/benngarcia/cotop/issues
|
|
@@ -17,7 +17,6 @@ Classifier: Programming Language :: Python :: 3
|
|
|
17
17
|
Classifier: Topic :: System :: Monitoring
|
|
18
18
|
Requires-Python: >=3.12
|
|
19
19
|
Requires-Dist: google-colab-cli~=0.8.0
|
|
20
|
-
Requires-Dist: packaging
|
|
21
20
|
Requires-Dist: requests
|
|
22
21
|
Requires-Dist: textual>=6
|
|
23
22
|
Description-Content-Type: text/markdown
|
|
@@ -26,10 +25,10 @@ Description-Content-Type: text/markdown
|
|
|
26
25
|
|
|
27
26
|
`top` for Google Colab: compute units, runtimes and what they cost, live in your terminal.
|
|
28
27
|
|
|
29
|
-

|
|
30
29
|
|
|
31
|
-
- **Your account:** compute units left, the hourly rate, when you run out, and the rate over the last day.
|
|
32
|
-
- **Runtimes:** every VM on your account, including ones opened in a browser, with a live GPU sparkline, rate, spend, uptime and state. A runtime idle while it bills, out of RAM or disk, or unreachable says so in its row.
|
|
30
|
+
- **Your account:** compute units left, the hourly rate, when you run out, and the rate over the last day. Compute units left turn red when they run out within a day.
|
|
31
|
+
- **Runtimes:** every VM on your account, most expensive first, including ones opened in a browser, with a live GPU sparkline, rate, spend, uptime and state. A runtime idle while it bills, out of RAM or disk, or unreachable says so in its row.
|
|
33
32
|
- **The selected runtime:** a braille graph of GPU utilization, read every two seconds, with VRAM, RAM and disk meters, what it has cost and its recent activity. On a runtime without a GPU, the graph shows RAM.
|
|
34
33
|
- **History:** press `h` for past sessions, how long each ran and roughly what it cost.
|
|
35
34
|
- **Actions:** stop a runtime, restart its kernel, or open a shell on it.
|
|
@@ -39,8 +38,9 @@ cotop works on top of the official [Colab CLI](https://pypi.org/project/google-c
|
|
|
39
38
|
## Install
|
|
40
39
|
|
|
41
40
|
```sh
|
|
42
|
-
uv tool install
|
|
41
|
+
uv tool install cotop
|
|
43
42
|
cotop
|
|
43
|
+
cotop --demo # a made-up account with twelve runtimes; no sign-in
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
cotop needs Python 3.12 or later. It is tested on macOS and should work on Linux. If you have not signed in to the Colab CLI yet, cotop runs the same Google sign-in on first launch. On macOS, Keychain may ask once whether cotop's Python can read the Colab CLI's token.
|
|
@@ -90,7 +90,7 @@ cotop is an independent project. It relies on internal parts of `google-colab-cl
|
|
|
90
90
|
```sh
|
|
91
91
|
uv sync
|
|
92
92
|
uv run pytest
|
|
93
|
-
uv run python scripts/screenshots.py OUT_DIR # README screenshots, rendered from
|
|
93
|
+
uv run python scripts/screenshots.py OUT_DIR # README screenshots, rendered from the demo account
|
|
94
94
|
```
|
|
95
95
|
|
|
96
96
|
To release, set `version` in `pyproject.toml`, then push a matching tag such as `v0.1.0`. The [publish workflow](.github/workflows/publish.yml) builds the package and uploads it to PyPI with trusted publishing.
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
`top` for Google Colab: compute units, runtimes and what they cost, live in your terminal.
|
|
4
4
|
|
|
5
|
-

|
|
6
6
|
|
|
7
|
-
- **Your account:** compute units left, the hourly rate, when you run out, and the rate over the last day.
|
|
8
|
-
- **Runtimes:** every VM on your account, including ones opened in a browser, with a live GPU sparkline, rate, spend, uptime and state. A runtime idle while it bills, out of RAM or disk, or unreachable says so in its row.
|
|
7
|
+
- **Your account:** compute units left, the hourly rate, when you run out, and the rate over the last day. Compute units left turn red when they run out within a day.
|
|
8
|
+
- **Runtimes:** every VM on your account, most expensive first, including ones opened in a browser, with a live GPU sparkline, rate, spend, uptime and state. A runtime idle while it bills, out of RAM or disk, or unreachable says so in its row.
|
|
9
9
|
- **The selected runtime:** a braille graph of GPU utilization, read every two seconds, with VRAM, RAM and disk meters, what it has cost and its recent activity. On a runtime without a GPU, the graph shows RAM.
|
|
10
10
|
- **History:** press `h` for past sessions, how long each ran and roughly what it cost.
|
|
11
11
|
- **Actions:** stop a runtime, restart its kernel, or open a shell on it.
|
|
@@ -15,8 +15,9 @@ cotop works on top of the official [Colab CLI](https://pypi.org/project/google-c
|
|
|
15
15
|
## Install
|
|
16
16
|
|
|
17
17
|
```sh
|
|
18
|
-
uv tool install
|
|
18
|
+
uv tool install cotop
|
|
19
19
|
cotop
|
|
20
|
+
cotop --demo # a made-up account with twelve runtimes; no sign-in
|
|
20
21
|
```
|
|
21
22
|
|
|
22
23
|
cotop needs Python 3.12 or later. It is tested on macOS and should work on Linux. If you have not signed in to the Colab CLI yet, cotop runs the same Google sign-in on first launch. On macOS, Keychain may ask once whether cotop's Python can read the Colab CLI's token.
|
|
@@ -66,7 +67,7 @@ cotop is an independent project. It relies on internal parts of `google-colab-cl
|
|
|
66
67
|
```sh
|
|
67
68
|
uv sync
|
|
68
69
|
uv run pytest
|
|
69
|
-
uv run python scripts/screenshots.py OUT_DIR # README screenshots, rendered from
|
|
70
|
+
uv run python scripts/screenshots.py OUT_DIR # README screenshots, rendered from the demo account
|
|
70
71
|
```
|
|
71
72
|
|
|
72
73
|
To release, set `version` in `pyproject.toml`, then push a matching tag such as `v0.1.0`. The [publish workflow](.github/workflows/publish.yml) builds the package and uploads it to PyPI with trusted publishing.
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# Where cotop's numbers come from
|
|
2
2
|
|
|
3
3
|
cotop polls the account, the runtime list and each runtime's kernels every 15
|
|
4
|
-
seconds, and each runtime's resources every 2 seconds for the graphs.
|
|
5
|
-
|
|
4
|
+
seconds, and each runtime's resources every 2 seconds for the graphs. Watching
|
|
5
|
+
sends only GET requests. Stop sends a POST: the colab CLI's own `colab stop`,
|
|
6
|
+
or for a runtime the CLI didn't start, the CLI client's `unassign` call.
|
|
7
|
+
|
|
8
|
+
`cotop --demo` runs on `src/cotop/demo.py`, a made-up account, and sends no
|
|
9
|
+
requests at all.
|
|
6
10
|
|
|
7
11
|
| Panel | Source | Notes |
|
|
8
12
|
|---|---|---|
|
|
@@ -37,12 +41,27 @@ the fall in balance between the first and last settled step in the past hour,
|
|
|
37
41
|
logged in `~/.cache/cotop/usage.jsonl`. You can compare it with the rate
|
|
38
42
|
Colab reports.
|
|
39
43
|
|
|
44
|
+
## Watching did not keep an idle runtime alive
|
|
45
|
+
|
|
46
|
+
Tested on 2026-11-17 with two idle CPU runtimes started with `colab new`, neither
|
|
47
|
+
with a kernel. One was read the way cotop reads a runtime: the same endpoints
|
|
48
|
+
and flag, its resources about every 2 seconds and its kernels about every 16 to
|
|
49
|
+
20 seconds. The other was never touched. Colab reclaimed both in the same
|
|
50
|
+
one-minute check, 3 hours 1 minute after they started.
|
|
51
|
+
|
|
52
|
+
The read requests carry `no_track_activity=1`, and Jupyter honors it: the
|
|
53
|
+
runtime's `last_activity` did not move after flagged reads, and did move after
|
|
54
|
+
the same reads without the flag. Colab does not document what it counts as
|
|
55
|
+
idle activity.
|
|
56
|
+
|
|
40
57
|
## Not verified
|
|
41
58
|
|
|
42
|
-
- Whether reading a runtime's Jupyter API delays Colab's idle shutdown. Requests
|
|
43
|
-
send `no_track_activity=1`, including the reads every 2 seconds. Colab's own
|
|
44
|
-
idle signal is not documented.
|
|
45
59
|
- That billing starts at assignment rather than at VM boot.
|
|
60
|
+
- Whether Colab reclaimed these runtimes for being idle or at a fixed time
|
|
61
|
+
limit near 3 hours. The test had no runtime that stayed active, so it cannot
|
|
62
|
+
tell the two apart.
|
|
63
|
+
- Runtimes with an idle kernel, and GPU or TPU runtimes. The test covered CPU
|
|
64
|
+
runtimes without a kernel only.
|
|
46
65
|
|
|
47
66
|
## Actions
|
|
48
67
|
|
|
Binary file
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cotop"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "A top-like terminal dashboard for Google Colab runtimes and compute units"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -20,7 +20,6 @@ classifiers = [
|
|
|
20
20
|
dependencies = [
|
|
21
21
|
# cotop reads the CLI's internals (auth, session file, client), so it pins the minor version.
|
|
22
22
|
"google-colab-cli~=0.8.0",
|
|
23
|
-
"packaging",
|
|
24
23
|
"requests",
|
|
25
24
|
"textual>=6",
|
|
26
25
|
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Render the README screenshots from the demo account; no Colab account is touched.
|
|
2
|
+
|
|
3
|
+
uv run python scripts/screenshots.py OUT_DIR
|
|
4
|
+
|
|
5
|
+
writes OUT_DIR/dashboard.svg (138x40), small.svg (80x24), large.svg (180x52) and
|
|
6
|
+
stop.svg. To get the PNGs in docs/images:
|
|
7
|
+
|
|
8
|
+
chrome --headless --force-device-scale-factor=2 --window-size=1380,880 \\
|
|
9
|
+
--screenshot=docs/images/dashboard.png OUT_DIR/dashboard.svg
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from cotop.app import demo_dashboard
|
|
18
|
+
|
|
19
|
+
SIZES = {"dashboard": (138, 40), "small": (80, 24), "large": (180, 52)}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def shoot(out: Path) -> None:
|
|
23
|
+
for name, size in SIZES.items():
|
|
24
|
+
with tempfile.TemporaryDirectory() as cache:
|
|
25
|
+
app = demo_dashboard(Path(cache))
|
|
26
|
+
async with app.run_test(size=size) as pilot:
|
|
27
|
+
for _ in range(3):
|
|
28
|
+
await app.workers.wait_for_complete()
|
|
29
|
+
await pilot.pause()
|
|
30
|
+
app.save_screenshot(f"{name}.svg", path=str(out))
|
|
31
|
+
if name == "dashboard":
|
|
32
|
+
await pilot.press("down", "down", "down", "down", "down", "s")
|
|
33
|
+
await pilot.pause()
|
|
34
|
+
app.save_screenshot("stop.svg", path=str(out))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
asyncio.run(shoot(Path(sys.argv[1])))
|
|
@@ -10,7 +10,7 @@ from datetime import datetime, timedelta
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
|
|
12
12
|
from rich.text import Text
|
|
13
|
-
from textual import work
|
|
13
|
+
from textual import events, work
|
|
14
14
|
from textual.app import App, ComposeResult
|
|
15
15
|
from textual.binding import Binding
|
|
16
16
|
from textual.containers import Horizontal, Vertical
|
|
@@ -19,7 +19,7 @@ from textual.theme import Theme
|
|
|
19
19
|
from textual.widgets import Button, DataTable, Footer, Label, Static
|
|
20
20
|
from textual.worker import Worker
|
|
21
21
|
|
|
22
|
-
from cotop.colab import
|
|
22
|
+
from cotop.colab import Colab
|
|
23
23
|
from cotop.model import (
|
|
24
24
|
Account,
|
|
25
25
|
History,
|
|
@@ -93,13 +93,14 @@ class Usage(Vertical):
|
|
|
93
93
|
a, now = s.account, s.taken_at
|
|
94
94
|
self.border_title = a.email
|
|
95
95
|
runway = a.runway
|
|
96
|
-
|
|
96
|
+
low = runway is not None and runway < LOW_RUNWAY
|
|
97
|
+
left = Text.assemble((f"{a.balance:,.2f}", f"bold {RED}" if low else "bold"), (" compute units left", RED if low else ""))
|
|
97
98
|
if runway is None:
|
|
98
99
|
pace = Text("nothing running", style=DIM)
|
|
99
100
|
else:
|
|
100
101
|
pace = Text.assemble(
|
|
101
102
|
f"{a.rate_hourly:.2f} an hour, lasts ",
|
|
102
|
-
(fmt_duration(runway), f"bold {RED}" if
|
|
103
|
+
(fmt_duration(runway), f"bold {RED}" if low else "bold"),
|
|
103
104
|
(f" until {(now + runway).astimezone():%a %H:%M}", DIM),
|
|
104
105
|
)
|
|
105
106
|
burn = log.burn(now)
|
|
@@ -225,26 +226,38 @@ def state_text(r: Runtime, now: datetime, latest: Resources | None) -> Text:
|
|
|
225
226
|
|
|
226
227
|
|
|
227
228
|
class Runtimes(DataTable):
|
|
228
|
-
"""The list to pick from; the GPU column moves every two seconds."""
|
|
229
|
+
"""The list to pick from, most expensive first; the GPU column moves every two seconds."""
|
|
229
230
|
|
|
230
|
-
COLUMNS = ("Name", "Hardware", "GPU", "CU/hr", "Spent", "
|
|
231
|
+
COLUMNS = ("Name", "Hardware", "GPU", "CU/hr", "Spent", "State")
|
|
232
|
+
WIDE = 100 # below this many columns "Up" is dropped; the panel still shows it
|
|
231
233
|
|
|
232
234
|
def on_mount(self) -> None:
|
|
233
235
|
for c in self.COLUMNS:
|
|
234
236
|
self.add_column(c, key=c)
|
|
235
237
|
|
|
238
|
+
def fit(self, width: int) -> None:
|
|
239
|
+
"""Show "Up" only when the terminal is wide enough for every column."""
|
|
240
|
+
has_up = "Up" in self.columns
|
|
241
|
+
if width >= self.WIDE and not has_up:
|
|
242
|
+
self.add_column("Up", key="Up")
|
|
243
|
+
elif width < self.WIDE and has_up:
|
|
244
|
+
self.remove_column("Up")
|
|
245
|
+
|
|
236
246
|
def show(self, s: Snapshot, rates: RateBook, samples: Samples) -> None:
|
|
237
|
-
|
|
247
|
+
rate_of = {r.endpoint: rates.rate(r, s.runtimes, s.account) for r in s.runtimes}
|
|
248
|
+
ordered = sorted(s.runtimes, key=lambda r: (rate_of[r.endpoint].hourly is None, -(rate_of[r.endpoint].hourly or 0), r.label))
|
|
249
|
+
keys = [r.endpoint for r in ordered]
|
|
238
250
|
if [row.value for row in self.rows] != keys:
|
|
239
251
|
selected = self.selected_endpoint
|
|
240
252
|
self.clear()
|
|
241
|
-
for r in
|
|
242
|
-
self.add_row(*["" for _ in self.
|
|
253
|
+
for r in ordered:
|
|
254
|
+
self.add_row(*["" for _ in self.columns], key=r.endpoint)
|
|
243
255
|
if selected in keys:
|
|
244
256
|
self.move_cursor(row=keys.index(selected))
|
|
257
|
+
self.border_title = f"Runtimes ({len(keys)})" if keys else "Runtimes"
|
|
245
258
|
now = s.taken_at
|
|
246
|
-
for r in
|
|
247
|
-
rate =
|
|
259
|
+
for r in ordered:
|
|
260
|
+
rate = rate_of[r.endpoint]
|
|
248
261
|
billed = r.billed_for(now)
|
|
249
262
|
up, spent = "?", "?"
|
|
250
263
|
if billed:
|
|
@@ -261,7 +274,8 @@ class Runtimes(DataTable):
|
|
|
261
274
|
"State": state_text(r, now, samples.latest(r.endpoint)),
|
|
262
275
|
}
|
|
263
276
|
for column, value in cells.items():
|
|
264
|
-
|
|
277
|
+
if column in self.columns:
|
|
278
|
+
self.update_cell(r.endpoint, column, value)
|
|
265
279
|
self.show_readings(s, samples)
|
|
266
280
|
|
|
267
281
|
def show_readings(self, s: Snapshot, samples: Samples) -> None:
|
|
@@ -285,6 +299,7 @@ class Confirm(ModalScreen[bool]):
|
|
|
285
299
|
CSS = """
|
|
286
300
|
Confirm { align: center middle; }
|
|
287
301
|
#dialog { width: 64; height: auto; border: round $warning; padding: 1 2; background: $panel; }
|
|
302
|
+
#dialog Label { width: 100%; }
|
|
288
303
|
#buttons { height: auto; margin-top: 1; align-horizontal: right; }
|
|
289
304
|
#buttons Button { margin-left: 2; }
|
|
290
305
|
"""
|
|
@@ -370,11 +385,11 @@ class Dashboard(App):
|
|
|
370
385
|
Usage Horizontal { height: 3; }
|
|
371
386
|
#usage-text { width: 46; }
|
|
372
387
|
#usage-graph { height: 3; }
|
|
373
|
-
Runtimes { height: auto; max-height:
|
|
388
|
+
Runtimes { height: auto; max-height: 45%; }
|
|
374
389
|
Runtimes:focus { border: round $accent; }
|
|
375
390
|
Runtimes > .datatable--cursor, Runtimes:focus > .datatable--cursor { background: $panel-lighten-1; text-style: bold; }
|
|
376
391
|
Runtimes > .datatable--header { background: $background; color: $text-muted; }
|
|
377
|
-
RuntimePanel { height: 1fr; min-height:
|
|
392
|
+
RuntimePanel { height: 1fr; min-height: 7; }
|
|
378
393
|
RuntimePanel Horizontal { height: 1fr; }
|
|
379
394
|
RuntimePanel .plot { width: 1fr; }
|
|
380
395
|
RuntimePanel .plot-label { height: 1; }
|
|
@@ -395,11 +410,11 @@ class Dashboard(App):
|
|
|
395
410
|
Binding("q", "quit", "Quit"),
|
|
396
411
|
]
|
|
397
412
|
|
|
398
|
-
def __init__(self, colab: Colab):
|
|
413
|
+
def __init__(self, colab: Colab, cache: Path):
|
|
399
414
|
super().__init__()
|
|
400
415
|
self.colab = colab
|
|
401
|
-
self.rates = RateBook(
|
|
402
|
-
self.usage = UsageLog(
|
|
416
|
+
self.rates = RateBook(cache / "rates.json")
|
|
417
|
+
self.usage = UsageLog(cache / "usage.jsonl")
|
|
403
418
|
self.samples = Samples()
|
|
404
419
|
self.email: str | None = None
|
|
405
420
|
self.snapshot: Snapshot | None = None
|
|
@@ -417,6 +432,7 @@ class Dashboard(App):
|
|
|
417
432
|
self.register_theme(THEME)
|
|
418
433
|
self.theme = "cotop"
|
|
419
434
|
self.query_one(Runtimes).border_title = "Runtimes"
|
|
435
|
+
self.query_one(Runtimes).fit(self.size.width)
|
|
420
436
|
self.action_refresh()
|
|
421
437
|
self.set_interval(POLL_SECONDS, self.action_refresh)
|
|
422
438
|
self.set_interval(SAMPLE_SECONDS, self.action_sample)
|
|
@@ -437,9 +453,9 @@ class Dashboard(App):
|
|
|
437
453
|
self.email = self.email or self.colab.email()
|
|
438
454
|
account = Account.parse(self.email, self.colab.ccu_info())
|
|
439
455
|
assignments = self.colab.assignments()
|
|
440
|
-
history =
|
|
456
|
+
history = self.colab.history()
|
|
441
457
|
status = self.colab.status(assignments)
|
|
442
|
-
runtimes = runtimes_from(assignments,
|
|
458
|
+
runtimes = runtimes_from(assignments, self.colab.sessions(), status, history.assigned_at())
|
|
443
459
|
snapshot = Snapshot(utc_now(), account, runtimes, history)
|
|
444
460
|
except Exception as e:
|
|
445
461
|
self.call_from_thread(self.show_error, e)
|
|
@@ -482,6 +498,12 @@ class Dashboard(App):
|
|
|
482
498
|
if r:
|
|
483
499
|
panel.show(r, self.snapshot, self.rates)
|
|
484
500
|
|
|
501
|
+
def on_resize(self, event: events.Resize) -> None:
|
|
502
|
+
table = self.query_one(Runtimes)
|
|
503
|
+
table.fit(event.size.width)
|
|
504
|
+
if self.snapshot:
|
|
505
|
+
table.show(self.snapshot, self.rates, self.samples)
|
|
506
|
+
|
|
485
507
|
def show_error(self, e: Exception) -> None:
|
|
486
508
|
age = f" Showing data from {fmt_duration(utc_now() - self.snapshot.taken_at)} ago." if self.snapshot else ""
|
|
487
509
|
self.notify(f"Refresh failed: {type(e).__name__}: {e}.{age}", severity="error", timeout=POLL_SECONDS)
|
|
@@ -523,6 +545,12 @@ class Dashboard(App):
|
|
|
523
545
|
if r := self.named("ssh into"):
|
|
524
546
|
self.open_ssh(r)
|
|
525
547
|
|
|
548
|
+
def check_action(self, action: str, parameters: tuple) -> bool | None:
|
|
549
|
+
"""Runtime keys work only on the dashboard, where you can see the runtime they act on."""
|
|
550
|
+
if action in {"stop", "restart_kernel", "ssh", "history"} and self.screen is not self.screen_stack[0]:
|
|
551
|
+
return False
|
|
552
|
+
return True
|
|
553
|
+
|
|
526
554
|
def action_history(self) -> None:
|
|
527
555
|
if self.snapshot:
|
|
528
556
|
self.push_screen(HistoryScreen(self.snapshot, self.rates))
|
|
@@ -560,12 +588,36 @@ class Dashboard(App):
|
|
|
560
588
|
def main() -> None:
|
|
561
589
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
562
590
|
parser.add_argument("-i", "--identity", help="SSH key for x (passed to colab ssh -i)")
|
|
591
|
+
parser.add_argument("--demo", action="store_true", help="run on a made-up heavy account: no sign-in, nothing touched")
|
|
563
592
|
args = parser.parse_args()
|
|
593
|
+
if args.demo:
|
|
594
|
+
run_demo()
|
|
595
|
+
return
|
|
564
596
|
try:
|
|
565
597
|
colab = Colab.sign_in(ssh_identity=args.identity) # before the UI: sign-in may prompt in the terminal
|
|
566
598
|
except Exception as e:
|
|
567
599
|
sys.exit(f"cotop: could not sign in with the colab CLI's credentials: {e}")
|
|
568
|
-
Dashboard(colab).run()
|
|
600
|
+
Dashboard(colab, CACHE).run()
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def demo_dashboard(cache: Path) -> Dashboard:
|
|
604
|
+
"""The dashboard on the demo account, its graphs already full."""
|
|
605
|
+
from cotop.demo import STEP, DemoColab, seed_cache
|
|
606
|
+
|
|
607
|
+
colab = DemoColab()
|
|
608
|
+
seed_cache(cache, colab.start)
|
|
609
|
+
app = Dashboard(colab, cache=cache)
|
|
610
|
+
for endpoint, readings in colab.backfill().items():
|
|
611
|
+
for i, reading in enumerate(readings):
|
|
612
|
+
app.samples.add(endpoint, colab.start - STEP * (len(readings) - i), reading)
|
|
613
|
+
return app
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def run_demo() -> None:
|
|
617
|
+
import tempfile
|
|
618
|
+
|
|
619
|
+
with tempfile.TemporaryDirectory(prefix="cotop-demo-") as cache:
|
|
620
|
+
demo_dashboard(Path(cache)).run()
|
|
569
621
|
|
|
570
622
|
|
|
571
623
|
if __name__ == "__main__":
|
|
@@ -19,7 +19,7 @@ from colab_cli.auth import get_credentials
|
|
|
19
19
|
from colab_cli.client import Client, Prod
|
|
20
20
|
from colab_cli.state import StateStore
|
|
21
21
|
|
|
22
|
-
from cotop.model import Resources, Runtime, Status, parse_json, utc_now
|
|
22
|
+
from cotop.model import History, Resources, Runtime, Status, parse_json, utc_now
|
|
23
23
|
|
|
24
24
|
COLAB = "https://colab.research.google.com"
|
|
25
25
|
USERINFO = "https://openidconnect.googleapis.com/v1/userinfo"
|
|
@@ -80,9 +80,19 @@ class Colab:
|
|
|
80
80
|
the last poll would point the action at another VM, and `colab ssh`
|
|
81
81
|
creates a new runtime for a name it does not know.
|
|
82
82
|
"""
|
|
83
|
-
if not still_assigned(runtime, self.assignments(),
|
|
83
|
+
if not still_assigned(runtime, self.assignments(), self.sessions()):
|
|
84
84
|
raise RuntimeGone(f"{runtime.name or runtime.endpoint} is no longer this runtime; refresh and retry")
|
|
85
85
|
|
|
86
|
+
def sessions(self) -> dict[str, dict]:
|
|
87
|
+
"""The CLI's named sessions, without their tokens, read under the CLI's own lock."""
|
|
88
|
+
return {
|
|
89
|
+
name: {"name": s.name, "endpoint": s.endpoint, "running": s.running}
|
|
90
|
+
for name, s in StateStore().list().items()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
def history(self) -> History:
|
|
94
|
+
return History.load(CLI_HOME / "history")
|
|
95
|
+
|
|
86
96
|
def status(self, assignments: list[dict]) -> dict[str, Status | Exception]:
|
|
87
97
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
88
98
|
results = pool.map(_runtime_status, assignments)
|
|
@@ -148,11 +158,3 @@ def _runtime_resources(runtime: Runtime) -> Resources | None:
|
|
|
148
158
|
return Resources.parse(utc_now(), _proxy_get(runtime.url, runtime.token, "/api/colab/resources", RESOURCES_TIMEOUT))
|
|
149
159
|
except Exception: # a gap in the graph; the slow loop reports an unreachable runtime
|
|
150
160
|
return None
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
def local_sessions() -> dict[str, dict]:
|
|
154
|
-
"""The CLI's named sessions, without their tokens."""
|
|
155
|
-
return {
|
|
156
|
-
name: {"name": s.name, "endpoint": s.endpoint, "running": s.running}
|
|
157
|
-
for name, s in StateStore().list().items()
|
|
158
|
-
}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""A made-up heavy Colab account, for `cotop --demo` and the README screenshots.
|
|
2
|
+
|
|
3
|
+
It answers the same calls as `cotop.colab.Colab`, so the dashboard runs on it
|
|
4
|
+
unchanged. Nothing here touches Google, the colab CLI's files or ~/.cache/cotop.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import random
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from datetime import datetime, timedelta
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from cotop.model import Gpu, History, Kernel, Resources, Runtime, Status, utc_now
|
|
17
|
+
|
|
18
|
+
GB = 1_000_000_000
|
|
19
|
+
STEP = timedelta(seconds=2) # one reading every two seconds, like the real sampler
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Machine:
|
|
24
|
+
"""One demo runtime: what it is, and what its readings do over time."""
|
|
25
|
+
|
|
26
|
+
name: str | None # None: opened in a browser, so the colab CLI has no name for it
|
|
27
|
+
endpoint: str
|
|
28
|
+
accelerator: str
|
|
29
|
+
shape: int # 0 standard, 1 high-RAM
|
|
30
|
+
gpu: str | None
|
|
31
|
+
vram_total: float
|
|
32
|
+
ram_total: float
|
|
33
|
+
disk: float
|
|
34
|
+
rate: float # CU/hr, illustrative
|
|
35
|
+
started_ago: timedelta
|
|
36
|
+
idle_for: timedelta | None # None: the kernel is busy
|
|
37
|
+
load: str # train | sweep | infer | idle | data | tpu
|
|
38
|
+
running: str | None = None
|
|
39
|
+
unreachable: bool = False
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
MACHINES = [
|
|
43
|
+
Machine("pretrain-7b", "gpu-h100-s-usc1a1-demo01", "H100", 0, "NVIDIA H100 80GB HBM3", 80, 234, 0.58, 15.04, timedelta(hours=9, minutes=40), None, "train", "run(pretrain.py)"),
|
|
44
|
+
Machine("finetune", "gpu-a100-s-usw1b2-demo02", "A100", 1, "NVIDIA A100-SXM4-40GB", 40, 83, 0.41, 11.77, timedelta(hours=3, minutes=12), None, "train", "run(train.py)"),
|
|
45
|
+
Machine("sweep-lr1", "gpu-l4-s-use4a1-demo03", "L4", 0, "NVIDIA L4", 23, 53, 0.22, 4.82, timedelta(hours=1, minutes=5), None, "sweep", "run(sweep.py --lr 1e-4)"),
|
|
46
|
+
Machine("sweep-lr2", "gpu-l4-s-use4a1-demo04", "L4", 0, "NVIDIA L4", 23, 53, 0.22, 4.82, timedelta(hours=1, minutes=5), None, "sweep", "run(sweep.py --lr 3e-4)"),
|
|
47
|
+
Machine("sweep-lr3", "gpu-l4-s-use4a2-demo05", "L4", 0, "NVIDIA L4", 23, 53, 0.97, 4.82, timedelta(hours=1, minutes=4), None, "sweep", "run(sweep.py --lr 1e-3)"),
|
|
48
|
+
Machine("eval", "gpu-t4-s-usc1a3-demo06", "T4", 0, "Tesla T4", 15, 13, 0.12, 1.76, timedelta(minutes=58), timedelta(minutes=34), "idle"),
|
|
49
|
+
Machine("data-prep", "m-s-usc1b1-demo07", "NONE", 1, None, 0, 52, 0.66, 0.29, timedelta(hours=2), None, "data", "run(tokenize.py)"),
|
|
50
|
+
Machine("jax-tpu", "tpu-v6e1-s-usc1a1-demo08", "V6E1", 0, None, 0, 176, 0.31, 3.52, timedelta(minutes=47), None, "tpu", "run(train_jax.py)"),
|
|
51
|
+
Machine(None, "gpu-g4-s-euw4a0-demo09", "G4", 0, "NVIDIA RTX PRO 6000 Blackwell", 96, 190, 0.18, 10.05, timedelta(minutes=21), None, "infer"),
|
|
52
|
+
Machine(None, "gpu-t4-s-euw4b1-demo10", "T4", 0, "Tesla T4", 15, 13, 0.09, 1.76, timedelta(hours=3, minutes=10), timedelta(hours=2, minutes=48), "idle"),
|
|
53
|
+
Machine("render", "gpu-l4-s-use1c2-demo11", "L4", 0, "NVIDIA L4", 23, 53, 0.2, 4.82, timedelta(hours=5), None, "infer", "run(render.py)", unreachable=True),
|
|
54
|
+
Machine("notebook", "gpu-a100-s-usw1b4-demo12", "A100", 0, "NVIDIA A100-SXM4-40GB", 40, 83, 0.11, 11.77, timedelta(minutes=12), timedelta(minutes=3), "idle"),
|
|
55
|
+
]
|
|
56
|
+
BALANCE = 612.40 # with the rates above this lasts under a day, so it shows in red
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def utilization(m: Machine, i: int, rng: random.Random) -> float:
|
|
60
|
+
"""GPU (or, without one, RAM) use for reading i, shaped like the job."""
|
|
61
|
+
match m.load:
|
|
62
|
+
case "train": # near full, with an evaluation dip every three minutes
|
|
63
|
+
return 0.18 + 0.15 * rng.random() if i % 90 < 7 else min(1.0, 0.9 + 0.08 * rng.random())
|
|
64
|
+
case "sweep":
|
|
65
|
+
return min(1.0, 0.72 + 0.2 * math.sin(i / 23 + len(m.endpoint)) + 0.06 * rng.random())
|
|
66
|
+
case "infer": # bursts of requests
|
|
67
|
+
return max(0.0, min(1.0, 0.45 + 0.3 * math.sin(i / 11) + 0.2 * rng.random()))
|
|
68
|
+
case "data":
|
|
69
|
+
return min(1.0, 0.35 + 0.004 * (i % 200) + 0.05 * rng.random())
|
|
70
|
+
case "tpu":
|
|
71
|
+
return 0.55 + 0.1 * math.sin(i / 40) + 0.05 * rng.random()
|
|
72
|
+
case _:
|
|
73
|
+
return 0.0
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def reading(m: Machine, i: int, at: datetime) -> Resources:
|
|
77
|
+
rng = random.Random(f"{m.endpoint}:{i}")
|
|
78
|
+
u = utilization(m, i, rng)
|
|
79
|
+
gpus = ()
|
|
80
|
+
ram_used = m.ram_total * (0.12 + (0.6 * u if m.gpu is None else 0.25))
|
|
81
|
+
if m.gpu:
|
|
82
|
+
gpus = (Gpu(m.gpu, u, int(m.vram_total * (0.15 + 0.78 * (u > 0.05)) * GB), int(m.vram_total * GB)),)
|
|
83
|
+
return Resources(
|
|
84
|
+
at=at,
|
|
85
|
+
memory_total=int(m.ram_total * GB),
|
|
86
|
+
memory_free=int((m.ram_total - ram_used) * GB),
|
|
87
|
+
disk_total=235 * GB,
|
|
88
|
+
disk_used=int(m.disk * 235 * GB),
|
|
89
|
+
gpus=gpus,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class DemoColab:
|
|
94
|
+
"""The account behind `cotop --demo`. Stopping a runtime removes it, as on Colab."""
|
|
95
|
+
|
|
96
|
+
def __init__(self, now: datetime | None = None):
|
|
97
|
+
self.start = now or utc_now()
|
|
98
|
+
self.machines = {m.endpoint: m for m in MACHINES}
|
|
99
|
+
|
|
100
|
+
# What the dashboard reads.
|
|
101
|
+
|
|
102
|
+
def email(self) -> str:
|
|
103
|
+
return "you@example.com"
|
|
104
|
+
|
|
105
|
+
def ccu_info(self) -> dict:
|
|
106
|
+
return {
|
|
107
|
+
"currentBalance": BALANCE,
|
|
108
|
+
"consumptionRateHourly": round(sum(m.rate for m in self.machines.values()), 2),
|
|
109
|
+
"assignmentsCount": len(self.machines),
|
|
110
|
+
"eligibleGpus": ["H100", "G4", "A100", "L4", "T4"],
|
|
111
|
+
"eligibleTpus": ["V6E1", "V5E1"],
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
def assignments(self) -> list[dict]:
|
|
115
|
+
return [
|
|
116
|
+
{
|
|
117
|
+
"endpoint": m.endpoint,
|
|
118
|
+
"accelerator": m.accelerator,
|
|
119
|
+
"machineShape": m.shape,
|
|
120
|
+
"variant": 2 if m.load == "tpu" else (0 if m.accelerator == "NONE" else 1),
|
|
121
|
+
"runtimeProxyInfo": {"token": "demo", "url": f"https://8080-{m.endpoint}-a.us-central1.prod.r.colab.dev"},
|
|
122
|
+
}
|
|
123
|
+
for m in self.machines.values()
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
def status(self, assignments: list[dict]) -> dict[str, Status | Exception]:
|
|
127
|
+
now = utc_now()
|
|
128
|
+
out = {}
|
|
129
|
+
for a in assignments:
|
|
130
|
+
m = self.machines[a["endpoint"]]
|
|
131
|
+
if m.unreachable:
|
|
132
|
+
out[m.endpoint] = ConnectionError("runtime proxy returned 502")
|
|
133
|
+
continue
|
|
134
|
+
state = "busy" if m.idle_for is None else "idle"
|
|
135
|
+
last = now - (m.idle_for or timedelta(seconds=1))
|
|
136
|
+
out[m.endpoint] = Status(started=now - m.started_ago, kernels=(Kernel(state, last, 1),))
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
def resources(self, runtimes: list[Runtime]) -> dict[str, Resources | None]:
|
|
140
|
+
now = utc_now()
|
|
141
|
+
i = HISTORY + int((now - self.start) / STEP)
|
|
142
|
+
return {
|
|
143
|
+
r.endpoint: None if self.machines[r.endpoint].unreachable else reading(self.machines[r.endpoint], i, now)
|
|
144
|
+
for r in runtimes
|
|
145
|
+
if r.endpoint in self.machines
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
def backfill(self) -> dict[str, list[Resources | None]]:
|
|
149
|
+
"""Readings for the last HISTORY steps, so graphs open full instead of empty."""
|
|
150
|
+
return {
|
|
151
|
+
m.endpoint: [None if m.unreachable else reading(m, i, self.start - STEP * (HISTORY - i)) for i in range(HISTORY)]
|
|
152
|
+
for m in self.machines.values()
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
def sessions(self) -> dict[str, dict]:
|
|
156
|
+
return {m.name: {"name": m.name, "endpoint": m.endpoint, "running": m.running} for m in self.machines.values() if m.name}
|
|
157
|
+
|
|
158
|
+
def history(self) -> History:
|
|
159
|
+
return History(demo_history(self.start))
|
|
160
|
+
|
|
161
|
+
# What the dashboard does.
|
|
162
|
+
|
|
163
|
+
def stop(self, runtime: Runtime) -> str:
|
|
164
|
+
self.machines.pop(runtime.endpoint, None)
|
|
165
|
+
return "stopped (demo)"
|
|
166
|
+
|
|
167
|
+
def restart_kernel(self, runtime: Runtime) -> str:
|
|
168
|
+
return "kernel restarted (demo)"
|
|
169
|
+
|
|
170
|
+
def ssh_command(self, runtime: Runtime) -> list[str]:
|
|
171
|
+
raise RuntimeError("the demo has no machines to open a shell on")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
HISTORY = 450 # fifteen minutes of readings
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def demo_history(now: datetime) -> dict[str, list[dict]]:
|
|
178
|
+
"""The colab CLI's event log for the demo: live sessions plus a week of finished ones."""
|
|
179
|
+
|
|
180
|
+
def event(at, kind, **fields):
|
|
181
|
+
return {"timestamp": at.isoformat(), "event_type": kind, **fields}
|
|
182
|
+
|
|
183
|
+
def created(at, m_endpoint, accelerator, shape="STANDARD"):
|
|
184
|
+
return event(at, "session_created", endpoint=m_endpoint, variant="GPU", accelerator=accelerator, machine_shape=shape)
|
|
185
|
+
|
|
186
|
+
events: dict[str, list[dict]] = {}
|
|
187
|
+
for m in MACHINES:
|
|
188
|
+
if not m.name:
|
|
189
|
+
continue
|
|
190
|
+
start = now - m.started_ago
|
|
191
|
+
log = [created(start, m.endpoint, m.accelerator, "HIGH_RAM" if m.shape else "STANDARD")]
|
|
192
|
+
log.append(event(start + timedelta(minutes=2), "file_operation", op="upload", remote="/content/data.tar.zst", local="-"))
|
|
193
|
+
log.append(event(start + timedelta(minutes=3), "execution", cell_index=0, cell_id="-", code="-", outputs=[]))
|
|
194
|
+
if m.idle_for is None:
|
|
195
|
+
log += [event(now - timedelta(minutes=k), "file_operation", op="download", remote="/content/runs/metrics.json", local="-") for k in range(40, 0, -5)]
|
|
196
|
+
events[m.name] = log
|
|
197
|
+
rng = random.Random(7)
|
|
198
|
+
kinds = [("A100", "HIGH_RAM", 11.77), ("H100", "STANDARD", 15.04), ("L4", "STANDARD", 4.82), ("T4", "STANDARD", 1.76), ("NONE", "STANDARD", 0.08)]
|
|
199
|
+
for k in range(18):
|
|
200
|
+
accelerator, shape, _ = kinds[k % len(kinds)]
|
|
201
|
+
start = now - timedelta(hours=10 + 9 * k, minutes=rng.randrange(60))
|
|
202
|
+
end = start + timedelta(minutes=rng.randrange(8, 400))
|
|
203
|
+
name = f"run-{k:02d}"
|
|
204
|
+
events[name] = [
|
|
205
|
+
created(start, f"demo-ended-{k:02d}", accelerator, shape),
|
|
206
|
+
event(end, "session_terminated", reason="pruned" if k % 7 == 3 else "user_requested"),
|
|
207
|
+
]
|
|
208
|
+
return events
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def seed_cache(cache: Path, now: datetime) -> None:
|
|
212
|
+
"""A day of balance and rate history, and the per-hardware rates cotop learns over time."""
|
|
213
|
+
cache.mkdir(parents=True, exist_ok=True)
|
|
214
|
+
rates = {f"{'CPU' if m.accelerator == 'NONE' else m.accelerator}/{'HIGH_RAM' if m.shape else 'STANDARD'}": m.rate for m in MACHINES}
|
|
215
|
+
(cache / "rates.json").write_text(json.dumps(rates))
|
|
216
|
+
steps = [(24, 0.0), (21, 4.82), (17, 16.59), (12, 31.0), (8, 45.6), (4, 62.3), (1.5, 74.1), (0.3, round(sum(m.rate for m in MACHINES), 2))]
|
|
217
|
+
balance, lines = BALANCE + 520.0, []
|
|
218
|
+
for (hours_ago, rate), (next_ago, _) in zip(steps, steps[1:] + [(0, 0)]):
|
|
219
|
+
lines.append(json.dumps([(now - timedelta(hours=hours_ago)).isoformat(), round(balance, 2), rate]))
|
|
220
|
+
balance -= rate * (hours_ago - next_ago)
|
|
221
|
+
(cache / "usage.jsonl").write_text("\n".join(lines) + "\n")
|
|
@@ -7,6 +7,8 @@ single place that knows the wire format (XSSI prefix, camelCase, fractions).
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import json
|
|
10
|
+
import os
|
|
11
|
+
import tempfile
|
|
10
12
|
from collections import deque
|
|
11
13
|
from dataclasses import dataclass, field
|
|
12
14
|
from datetime import datetime, timedelta, timezone
|
|
@@ -25,6 +27,22 @@ def parse_time(value: str) -> datetime:
|
|
|
25
27
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
26
28
|
|
|
27
29
|
|
|
30
|
+
def write_atomic(path: Path, text: str) -> None:
|
|
31
|
+
"""Replace path in one step, so a reader never sees half a file, even with two cotops running."""
|
|
32
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
with tempfile.NamedTemporaryFile("w", dir=path.parent, prefix=path.name, suffix=".tmp", delete=False) as f:
|
|
34
|
+
f.write(text)
|
|
35
|
+
os.replace(f.name, path)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def read_cache(path: Path):
|
|
39
|
+
"""A cache file's JSON, or None if it is missing or damaged; cotop rebuilds what it loses."""
|
|
40
|
+
try:
|
|
41
|
+
return json.loads(path.read_text())
|
|
42
|
+
except (FileNotFoundError, json.JSONDecodeError):
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
28
46
|
@dataclass(frozen=True)
|
|
29
47
|
class Account:
|
|
30
48
|
email: str
|
|
@@ -266,7 +284,8 @@ class RateBook:
|
|
|
266
284
|
|
|
267
285
|
def __init__(self, path: Path):
|
|
268
286
|
self.path = path
|
|
269
|
-
|
|
287
|
+
rates = read_cache(path)
|
|
288
|
+
self.rates: dict[str, float] = rates if isinstance(rates, dict) else {}
|
|
270
289
|
|
|
271
290
|
def observe(self, account: Account, runtimes: list[Runtime]) -> None:
|
|
272
291
|
kinds = {r.kind for r in runtimes}
|
|
@@ -278,8 +297,7 @@ class RateBook:
|
|
|
278
297
|
kind = kinds.pop()
|
|
279
298
|
if self.rates.get(kind) != rate:
|
|
280
299
|
self.rates[kind] = rate
|
|
281
|
-
self.path.
|
|
282
|
-
self.path.write_text(json.dumps(self.rates, indent=2, sort_keys=True))
|
|
300
|
+
write_atomic(self.path, json.dumps(self.rates, indent=2, sort_keys=True))
|
|
283
301
|
|
|
284
302
|
def rate(self, runtime: Runtime, runtimes: list[Runtime], account: Account) -> Rate:
|
|
285
303
|
if len(runtimes) == 1:
|
|
@@ -305,16 +323,18 @@ class UsageLog:
|
|
|
305
323
|
self.samples: list[tuple[datetime, float, float]] = []
|
|
306
324
|
if path.exists():
|
|
307
325
|
for line in path.read_text().splitlines():
|
|
308
|
-
|
|
309
|
-
|
|
326
|
+
try:
|
|
327
|
+
ts, balance, rate = json.loads(line)
|
|
328
|
+
self.samples.append((parse_time(ts), balance, rate))
|
|
329
|
+
except ValueError: # a damaged line from an older cotop: skip it
|
|
330
|
+
continue
|
|
310
331
|
|
|
311
332
|
def record(self, now: datetime, balance: float, rate: float) -> None:
|
|
312
333
|
if self.samples and self.samples[-1][1:] == (balance, rate):
|
|
313
334
|
return
|
|
314
335
|
self.samples.append((now, balance, rate))
|
|
315
336
|
self.samples = [s for s in self.samples if now - s[0] <= self.KEEP]
|
|
316
|
-
self.path.
|
|
317
|
-
self.path.write_text("".join(json.dumps([t.isoformat(), b, r]) + "\n" for t, b, r in self.samples))
|
|
337
|
+
write_atomic(self.path, "".join(json.dumps([t.isoformat(), b, r]) + "\n" for t, b, r in self.samples))
|
|
318
338
|
|
|
319
339
|
def burn(self, now: datetime, window: timedelta = timedelta(hours=1)) -> float | None:
|
|
320
340
|
"""Compute units per hour actually charged within the window."""
|
|
@@ -2,18 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
|
-
import cotop.app as app_module
|
|
6
5
|
from cotop.app import Dashboard, RuntimePanel, Runtimes
|
|
7
|
-
from
|
|
6
|
+
from rich.color import Color as RichColor
|
|
7
|
+
from rich.style import Style
|
|
8
|
+
|
|
9
|
+
from cotop.widgets import RED
|
|
10
|
+
from cotop.model import History, utc_now
|
|
8
11
|
from tests.test_model import CPU, G4, LOCAL, T4, ccu, resources, status
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
class StubColab:
|
|
12
15
|
def __init__(self, assignments, status, readings, rate):
|
|
13
16
|
self._assignments, self._status, self._readings, self._rate = assignments, status, readings, rate
|
|
17
|
+
self.local = LOCAL
|
|
14
18
|
self.fail = False
|
|
15
19
|
self.calls = []
|
|
16
20
|
|
|
21
|
+
def sessions(self):
|
|
22
|
+
return self.local
|
|
23
|
+
|
|
24
|
+
def history(self):
|
|
25
|
+
return History({})
|
|
26
|
+
|
|
17
27
|
def email(self):
|
|
18
28
|
return "me@example.com"
|
|
19
29
|
|
|
@@ -41,12 +51,9 @@ class StubColab:
|
|
|
41
51
|
|
|
42
52
|
|
|
43
53
|
@pytest.fixture
|
|
44
|
-
def isolated(tmp_path
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
monkeypatch.setattr(app_module, "local_sessions", lambda: LOCAL)
|
|
48
|
-
(tmp_path / "cli" / "history").mkdir(parents=True)
|
|
49
|
-
return tmp_path
|
|
54
|
+
def isolated(tmp_path):
|
|
55
|
+
"""The cache dir for one test; nothing else is read from or written to disk."""
|
|
56
|
+
return tmp_path / "cache"
|
|
50
57
|
|
|
51
58
|
|
|
52
59
|
async def settle(app, pilot):
|
|
@@ -69,15 +76,15 @@ def mixed():
|
|
|
69
76
|
|
|
70
77
|
|
|
71
78
|
async def test_mixed_untracked_unreachable_and_idle(isolated):
|
|
72
|
-
app = Dashboard(mixed())
|
|
79
|
+
app = Dashboard(mixed(), isolated)
|
|
73
80
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
74
81
|
await settle(app, pilot)
|
|
75
82
|
table = app.query_one(Runtimes)
|
|
76
83
|
assert table.row_count == 3
|
|
77
|
-
states = [str(table.get_row_at(i)[
|
|
84
|
+
states = [str(table.get_row_at(i)[5]) for i in range(3)]
|
|
78
85
|
assert states[0].startswith("idle") and states[1] == "busy" and states[2] == "unreachable"
|
|
79
86
|
assert [str(table.get_row_at(i)[3]) for i in range(3)] == ["?", "?", "?"] # mixed: no rate known
|
|
80
|
-
assert not (isolated / "
|
|
87
|
+
assert not (isolated / "rates.json").exists()
|
|
81
88
|
panel = app.query_one(RuntimePanel)
|
|
82
89
|
assert panel.border_title == "train G4"
|
|
83
90
|
await pilot.press("down")
|
|
@@ -87,7 +94,7 @@ async def test_mixed_untracked_unreachable_and_idle(isolated):
|
|
|
87
94
|
|
|
88
95
|
async def test_failed_refresh_keeps_the_last_data(isolated):
|
|
89
96
|
colab = mixed()
|
|
90
|
-
app = Dashboard(colab)
|
|
97
|
+
app = Dashboard(colab, isolated)
|
|
91
98
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
92
99
|
await settle(app, pilot)
|
|
93
100
|
colab.fail = True
|
|
@@ -98,7 +105,7 @@ async def test_failed_refresh_keeps_the_last_data(isolated):
|
|
|
98
105
|
|
|
99
106
|
|
|
100
107
|
async def test_no_runtimes(isolated):
|
|
101
|
-
app = Dashboard(StubColab([], {}, {}, rate=0))
|
|
108
|
+
app = Dashboard(StubColab([], {}, {}, rate=0), isolated)
|
|
102
109
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
103
110
|
await settle(app, pilot)
|
|
104
111
|
assert "nothing running" in str(app.query_one("#usage-text").render())
|
|
@@ -107,7 +114,7 @@ async def test_no_runtimes(isolated):
|
|
|
107
114
|
|
|
108
115
|
async def test_stop_needs_a_yes_and_acts_on_the_selected_runtime(isolated):
|
|
109
116
|
colab = StubColab([G4, T4], {G4["endpoint"]: status(), T4["endpoint"]: status()}, {}, rate=11.89)
|
|
110
|
-
app = Dashboard(colab)
|
|
117
|
+
app = Dashboard(colab, isolated)
|
|
111
118
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
112
119
|
await settle(app, pilot)
|
|
113
120
|
await pilot.press("s")
|
|
@@ -123,7 +130,7 @@ async def test_stop_needs_a_yes_and_acts_on_the_selected_runtime(isolated):
|
|
|
123
130
|
|
|
124
131
|
async def test_kernel_and_ssh_actions_refuse_untracked_runtimes(isolated):
|
|
125
132
|
colab = StubColab([T4], {T4["endpoint"]: status()}, {}, rate=1.84)
|
|
126
|
-
app = Dashboard(colab)
|
|
133
|
+
app = Dashboard(colab, isolated)
|
|
127
134
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
128
135
|
await settle(app, pilot)
|
|
129
136
|
await pilot.press("k", "x")
|
|
@@ -133,7 +140,7 @@ async def test_kernel_and_ssh_actions_refuse_untracked_runtimes(isolated):
|
|
|
133
140
|
|
|
134
141
|
|
|
135
142
|
async def test_history_screen_opens_and_closes(isolated):
|
|
136
|
-
app = Dashboard(mixed())
|
|
143
|
+
app = Dashboard(mixed(), isolated)
|
|
137
144
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
138
145
|
await settle(app, pilot)
|
|
139
146
|
await pilot.press("h")
|
|
@@ -147,7 +154,7 @@ async def test_history_screen_opens_and_closes(isolated):
|
|
|
147
154
|
async def test_selection_survives_the_runtime_list_changing(isolated):
|
|
148
155
|
"""A selection below row 0 must stay put, not flip between runtimes, when the list changes."""
|
|
149
156
|
colab = mixed()
|
|
150
|
-
app = Dashboard(colab)
|
|
157
|
+
app = Dashboard(colab, isolated)
|
|
151
158
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
152
159
|
await settle(app, pilot)
|
|
153
160
|
await pilot.press("down") # T4, on row 1
|
|
@@ -166,7 +173,7 @@ async def test_selection_survives_the_runtime_list_changing(isolated):
|
|
|
166
173
|
|
|
167
174
|
|
|
168
175
|
async def test_readings_arriving_while_the_selection_moves(isolated):
|
|
169
|
-
app = Dashboard(mixed())
|
|
176
|
+
app = Dashboard(mixed(), isolated)
|
|
170
177
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
171
178
|
await settle(app, pilot)
|
|
172
179
|
for step in range(6):
|
|
@@ -190,7 +197,7 @@ async def empty_then_two(app, pilot, colab, empty_at_launch):
|
|
|
190
197
|
@pytest.mark.parametrize("empty_at_launch", [True, False])
|
|
191
198
|
async def test_arrows_move_the_selection_after_the_list_was_empty(isolated, empty_at_launch):
|
|
192
199
|
colab = StubColab([] if empty_at_launch else [G4], {} if empty_at_launch else {G4["endpoint"]: status()}, {}, rate=0)
|
|
193
|
-
app = Dashboard(colab)
|
|
200
|
+
app = Dashboard(colab, isolated)
|
|
194
201
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
195
202
|
await empty_then_two(app, pilot, colab, empty_at_launch)
|
|
196
203
|
await pilot.press("down")
|
|
@@ -200,28 +207,29 @@ async def test_arrows_move_the_selection_after_the_list_was_empty(isolated, empt
|
|
|
200
207
|
assert app.query_one(RuntimePanel).border_title == "untracked T4"
|
|
201
208
|
|
|
202
209
|
|
|
203
|
-
async def test_ssh_after_the_empty_state_targets_the_highlighted_row(isolated
|
|
210
|
+
async def test_ssh_after_the_empty_state_targets_the_highlighted_row(isolated):
|
|
204
211
|
"""x is the one key with no confirmation, so it gets its own test."""
|
|
205
|
-
named = {**LOCAL, "eval": {"name": "eval", "endpoint": T4["endpoint"], "running": None}}
|
|
206
|
-
monkeypatch.setattr(app_module, "local_sessions", lambda: named)
|
|
207
212
|
colab = StubColab([], {}, {}, rate=0)
|
|
213
|
+
colab.local = {**LOCAL, "eval": {"name": "eval", "endpoint": T4["endpoint"], "running": None}}
|
|
208
214
|
|
|
209
215
|
def ssh_command(runtime):
|
|
210
216
|
colab.calls.append(("ssh", runtime.endpoint))
|
|
211
217
|
raise RuntimeError("stubbed: no shell in tests")
|
|
212
218
|
|
|
213
219
|
colab.ssh_command = ssh_command
|
|
214
|
-
app = Dashboard(colab)
|
|
220
|
+
app = Dashboard(colab, isolated)
|
|
215
221
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
216
222
|
await empty_then_two(app, pilot, colab, empty_at_launch=True)
|
|
223
|
+
table = app.query_one(Runtimes)
|
|
224
|
+
second = table.coordinate_to_cell_key((1, 0)).row_key.value
|
|
217
225
|
await pilot.press("down", "x")
|
|
218
226
|
await settle(app, pilot)
|
|
219
|
-
assert colab.calls == [("ssh",
|
|
227
|
+
assert colab.calls == [("ssh", second)]
|
|
220
228
|
|
|
221
229
|
|
|
222
230
|
async def test_the_panel_follows_a_poll_without_a_cursor_move(isolated):
|
|
223
231
|
colab = StubColab([G4], {G4["endpoint"]: status()}, {}, rate=10.05)
|
|
224
|
-
app = Dashboard(colab)
|
|
232
|
+
app = Dashboard(colab, isolated)
|
|
225
233
|
async with app.run_test(size=(150, 46)) as pilot:
|
|
226
234
|
await settle(app, pilot)
|
|
227
235
|
assert app.query_one(RuntimePanel).border_subtitle.startswith("busy")
|
|
@@ -229,3 +237,56 @@ async def test_the_panel_follows_a_poll_without_a_cursor_move(isolated):
|
|
|
229
237
|
await pilot.press("r")
|
|
230
238
|
await settle(app, pilot)
|
|
231
239
|
assert app.query_one(RuntimePanel).border_subtitle.startswith("idle")
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
async def test_runtime_keys_do_nothing_on_the_history_screen(isolated):
|
|
243
|
+
colab = mixed()
|
|
244
|
+
colab.local = {**LOCAL, "eval": {"name": "eval", "endpoint": T4["endpoint"], "running": None}}
|
|
245
|
+
|
|
246
|
+
def ssh_command(runtime):
|
|
247
|
+
colab.calls.append(("ssh", runtime.endpoint))
|
|
248
|
+
raise RuntimeError("stubbed: no shell in tests")
|
|
249
|
+
|
|
250
|
+
colab.ssh_command = ssh_command
|
|
251
|
+
app = Dashboard(colab, isolated)
|
|
252
|
+
async with app.run_test(size=(150, 46)) as pilot:
|
|
253
|
+
await settle(app, pilot)
|
|
254
|
+
await pilot.press("h", "x", "s", "k")
|
|
255
|
+
await settle(app, pilot)
|
|
256
|
+
assert type(app.screen).__name__ == "HistoryScreen"
|
|
257
|
+
assert colab.calls == []
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
async def test_the_demo_runs_and_sorts_the_most_expensive_first(tmp_path):
|
|
261
|
+
from cotop.app import demo_dashboard
|
|
262
|
+
|
|
263
|
+
app = demo_dashboard(tmp_path)
|
|
264
|
+
async with app.run_test(size=(138, 40)) as pilot:
|
|
265
|
+
await settle(app, pilot)
|
|
266
|
+
table = app.query_one(Runtimes)
|
|
267
|
+
assert table.row_count == 12 and table.border_title == "Runtimes (12)"
|
|
268
|
+
rates = [float(str(table.get_row_at(i)[3]).lstrip("≈")) for i in range(table.row_count)]
|
|
269
|
+
assert rates == sorted(rates, reverse=True)
|
|
270
|
+
assert "Up" in table.columns
|
|
271
|
+
await pilot.resize_terminal(80, 24)
|
|
272
|
+
await settle(app, pilot)
|
|
273
|
+
assert "Up" not in table.columns # too narrow: the panel still shows uptime
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
async def test_compute_units_turn_red_when_they_last_under_a_day(isolated):
|
|
277
|
+
colab = StubColab([G4], {G4["endpoint"]: status()}, {}, rate=10.05)
|
|
278
|
+
app = Dashboard(colab, isolated)
|
|
279
|
+
async with app.run_test(size=(150, 46)) as pilot:
|
|
280
|
+
await settle(app, pilot)
|
|
281
|
+
assert not balance_is_red(app) # 5,227 left at 10.05 an hour lasts weeks
|
|
282
|
+
colab._rate = 300.0 # now it lasts under a day
|
|
283
|
+
await pilot.press("r")
|
|
284
|
+
await settle(app, pilot)
|
|
285
|
+
assert balance_is_red(app)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def balance_is_red(app) -> bool:
|
|
289
|
+
text = app.query_one("#usage-text").render()
|
|
290
|
+
span = next(s for s in text.spans if text.plain[s.start:s.end].startswith("5,227"))
|
|
291
|
+
color = Style.parse(str(span.style)).color
|
|
292
|
+
return color is not None and color.get_truecolor() == RichColor.parse(RED).get_truecolor()
|
|
@@ -284,3 +284,10 @@ def test_a_rate_change_is_not_a_billing_step(tmp_path):
|
|
|
284
284
|
def test_a_missing_gpu_reading_is_a_gap_not_an_idle_gpu():
|
|
285
285
|
r = Resources.parse(NOW, {"gpus": [{"name": "Tesla T4"}]})
|
|
286
286
|
assert r.gpu_utilization is None
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def test_damaged_cache_files_do_not_stop_cotop(tmp_path):
|
|
290
|
+
(tmp_path / "u.jsonl").write_text('["2026-11-17T21:00:00+00:00", 100.0, 10.0]\n["2026-11-17T21:05:00+00:00", 99.')
|
|
291
|
+
(tmp_path / "r.json").write_text('{"G4/STANDARD": 10.0')
|
|
292
|
+
assert len(UsageLog(tmp_path / "u.jsonl").samples) == 1
|
|
293
|
+
assert RateBook(tmp_path / "r.json").rates == {}
|
|
@@ -278,11 +278,10 @@ wheels = [
|
|
|
278
278
|
|
|
279
279
|
[[package]]
|
|
280
280
|
name = "cotop"
|
|
281
|
-
version = "0.
|
|
281
|
+
version = "0.2.0"
|
|
282
282
|
source = { editable = "." }
|
|
283
283
|
dependencies = [
|
|
284
284
|
{ name = "google-colab-cli" },
|
|
285
|
-
{ name = "packaging" },
|
|
286
285
|
{ name = "requests" },
|
|
287
286
|
{ name = "textual" },
|
|
288
287
|
]
|
|
@@ -296,7 +295,6 @@ dev = [
|
|
|
296
295
|
[package.metadata]
|
|
297
296
|
requires-dist = [
|
|
298
297
|
{ name = "google-colab-cli", specifier = "~=0.8.0" },
|
|
299
|
-
{ name = "packaging" },
|
|
300
298
|
{ name = "requests" },
|
|
301
299
|
{ name = "textual", specifier = ">=6" },
|
|
302
300
|
]
|
|
Binary file
|
cotop-0.1.0/docs/images/stop.png
DELETED
|
Binary file
|
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
"""Render the README screenshots from made-up data; no Colab account is touched.
|
|
2
|
-
|
|
3
|
-
uv run python scripts/screenshots.py OUT_DIR
|
|
4
|
-
|
|
5
|
-
writes OUT_DIR/dashboard.svg, OUT_DIR/stop.svg and OUT_DIR/small.svg (80x24).
|
|
6
|
-
To get the PNGs in docs/images:
|
|
7
|
-
|
|
8
|
-
chrome --headless --force-device-scale-factor=2 --window-size=1380,880 \\
|
|
9
|
-
--screenshot=docs/images/dashboard.png OUT_DIR/dashboard.svg
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
import asyncio
|
|
13
|
-
import json
|
|
14
|
-
import math
|
|
15
|
-
import random
|
|
16
|
-
import sys
|
|
17
|
-
import tempfile
|
|
18
|
-
from datetime import timedelta
|
|
19
|
-
from pathlib import Path
|
|
20
|
-
|
|
21
|
-
import cotop.app as app_module
|
|
22
|
-
from cotop.app import Dashboard
|
|
23
|
-
from cotop.model import Gpu, Kernel, Resources, Status, utc_now
|
|
24
|
-
|
|
25
|
-
NOW = utc_now()
|
|
26
|
-
GB = 1_000_000_000
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def ago(**kw):
|
|
30
|
-
return NOW - timedelta(**kw)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def assignment(endpoint, accelerator, shape=0):
|
|
34
|
-
return {
|
|
35
|
-
"endpoint": endpoint,
|
|
36
|
-
"accelerator": accelerator,
|
|
37
|
-
"machineShape": shape,
|
|
38
|
-
"variant": 1,
|
|
39
|
-
"runtimeProxyInfo": {"token": "-", "url": f"https://8080-{endpoint}-a.us-west1.prod.r.colab.dev"},
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def status(started, state, last_activity):
|
|
44
|
-
return Status(started=started, kernels=(Kernel(state, last_activity, 1),))
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def reading(at, gpu, util, vram, ram, disk):
|
|
48
|
-
return Resources(
|
|
49
|
-
at=at,
|
|
50
|
-
memory_total=ram[1] * GB,
|
|
51
|
-
memory_free=int((ram[1] - ram[0]) * GB),
|
|
52
|
-
disk_total=235 * GB,
|
|
53
|
-
disk_used=int(disk * 235 * GB),
|
|
54
|
-
gpus=(Gpu(gpu, util, int(vram[0] * GB), vram[1] * GB),),
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def training(i, rng):
|
|
59
|
-
"""A fine-tune: near-full GPU, with a dip for evaluation every three minutes."""
|
|
60
|
-
if i % 90 < 7:
|
|
61
|
-
return 0.18 + 0.15 * rng.random()
|
|
62
|
-
return min(1.0, 0.9 + 0.08 * rng.random())
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def inference(i, rng):
|
|
66
|
-
"""A notebook serving requests in bursts."""
|
|
67
|
-
return max(0.0, min(1.0, 0.45 + 0.3 * math.sin(i / 11) + 0.2 * rng.random()))
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
SERIES = 450 # fifteen minutes of readings, one every two seconds
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def readings(endpoint):
|
|
74
|
-
rng = random.Random(endpoint)
|
|
75
|
-
out = []
|
|
76
|
-
for i in range(SERIES):
|
|
77
|
-
at = NOW - timedelta(seconds=2 * (SERIES - i))
|
|
78
|
-
if endpoint == A100["endpoint"]:
|
|
79
|
-
out.append(reading(at, "NVIDIA A100-SXM4-40GB", training(i, rng), (37.2, 40), (61 + 2 * rng.random(), 83), 0.41))
|
|
80
|
-
elif endpoint == T4["endpoint"]:
|
|
81
|
-
out.append(reading(at, "Tesla T4", 0.0, (3.1, 15), (6.2, 13), 0.12))
|
|
82
|
-
else:
|
|
83
|
-
util = inference(i, rng)
|
|
84
|
-
out.append(reading(at, "NVIDIA L4", util, (14 + 4 * util, 23), (19, 53), 0.18))
|
|
85
|
-
return out
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
A100 = assignment("gpu-a100-s-usw1b2-demo0a1", "A100", shape=1)
|
|
89
|
-
T4 = assignment("gpu-t4-s-usc1a3-demo0b2", "T4")
|
|
90
|
-
L4 = assignment("gpu-l4-s-euw4a0-demo0c3", "L4")
|
|
91
|
-
ASSIGNMENTS = [A100, T4, L4]
|
|
92
|
-
STATUS = {
|
|
93
|
-
A100["endpoint"]: status(ago(hours=3, minutes=12), "busy", ago(seconds=2)),
|
|
94
|
-
T4["endpoint"]: status(ago(minutes=58), "idle", ago(minutes=34)),
|
|
95
|
-
L4["endpoint"]: status(ago(minutes=21), "busy", ago(seconds=9)),
|
|
96
|
-
}
|
|
97
|
-
LOCAL = {
|
|
98
|
-
"finetune": {"name": "finetune", "endpoint": A100["endpoint"], "running": "run(train.py)"},
|
|
99
|
-
"eval": {"name": "eval", "endpoint": T4["endpoint"], "running": None},
|
|
100
|
-
}
|
|
101
|
-
RATES = {"A100/HIGH_RAM": 11.77, "T4/STANDARD": 1.76, "L4/STANDARD": 4.82}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
def write_history(directory: Path) -> None:
|
|
105
|
-
def event(at, kind, **fields):
|
|
106
|
-
return json.dumps({"timestamp": at.isoformat(), "event_type": kind, **fields}) + "\n"
|
|
107
|
-
|
|
108
|
-
def created(at, endpoint, accelerator, shape="STANDARD"):
|
|
109
|
-
return event(at, "session_created", endpoint=endpoint, variant="GPU", accelerator=accelerator, machine_shape=shape)
|
|
110
|
-
|
|
111
|
-
finetune = [created(ago(hours=3, minutes=12), A100["endpoint"], "A100", "HIGH_RAM")]
|
|
112
|
-
finetune.append(event(ago(hours=3, minutes=10), "file_operation", op="upload", remote="/content/data.tar.zst", local="-"))
|
|
113
|
-
finetune.append(event(ago(hours=3, minutes=9), "file_operation", op="upload", remote="/content/train.py", local="-"))
|
|
114
|
-
finetune.append(event(ago(hours=3, minutes=8), "execution", cell_index=0, cell_id="-", code="-", outputs=[]))
|
|
115
|
-
finetune.append(event(ago(hours=3, minutes=7), "execution", cell_index=1, cell_id="-", code="-", outputs=[]))
|
|
116
|
-
finetune += [event(ago(minutes=m), "file_operation", op="download", remote="/content/runs/metrics.json", local="-") for m in range(40, 0, -5)]
|
|
117
|
-
eval_ = [
|
|
118
|
-
created(ago(minutes=58), T4["endpoint"], "T4"),
|
|
119
|
-
event(ago(minutes=50), "file_operation", op="upload", remote="/content/eval.py", local="-"),
|
|
120
|
-
event(ago(minutes=49), "execution", cell_index=0, cell_id="-", code="-", outputs=[]),
|
|
121
|
-
event(ago(minutes=34), "file_operation", op="download", remote="/content/scores.csv", local="-"),
|
|
122
|
-
]
|
|
123
|
-
past = {
|
|
124
|
-
"sweep": [(ago(days=1, hours=2), ago(days=1), "A100", "HIGH_RAM", "user_requested")],
|
|
125
|
-
"tokenize": [(ago(days=1, hours=5), ago(days=1, hours=4, minutes=25), "NONE", "STANDARD", "user_requested")],
|
|
126
|
-
"baseline": [(ago(days=2, hours=6), ago(days=2, hours=1), "L4", "STANDARD", "pruned")],
|
|
127
|
-
"smoke": [(ago(days=2, hours=7), ago(days=2, hours=6, minutes=52), "T4", "STANDARD", "user_requested")],
|
|
128
|
-
}
|
|
129
|
-
(directory / "finetune.jsonl").write_text("".join(finetune))
|
|
130
|
-
(directory / "eval.jsonl").write_text("".join(eval_))
|
|
131
|
-
for name, runs in past.items():
|
|
132
|
-
lines = []
|
|
133
|
-
for start, end, accelerator, shape, reason in runs:
|
|
134
|
-
lines.append(created(start, f"gpu-{name}-ended", accelerator, shape))
|
|
135
|
-
lines.append(event(end, "session_terminated", reason=reason))
|
|
136
|
-
(directory / f"{name}.jsonl").write_text("".join(lines))
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
def write_cache(cache: Path) -> None:
|
|
140
|
-
"""A day of usage: a T4 in the morning, an A100 from midday, then three runtimes."""
|
|
141
|
-
cache.mkdir(parents=True)
|
|
142
|
-
(cache / "rates.json").write_text(json.dumps(RATES))
|
|
143
|
-
steps = [(24, 0.0), (19, 1.76), (16, 0.0), (11, 11.77), (6, 13.53), (3.2, 11.77), (1, 13.53), (0.35, 18.35)]
|
|
144
|
-
balance, lines = 512.80, []
|
|
145
|
-
for (hours_ago, rate), (next_ago, _) in zip(steps, steps[1:] + [(0, 0)]):
|
|
146
|
-
lines.append(json.dumps([ago(hours=hours_ago).isoformat(), round(balance, 2), rate]))
|
|
147
|
-
balance -= rate * (hours_ago - next_ago)
|
|
148
|
-
(cache / "usage.jsonl").write_text("\n".join(lines) + "\n")
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
class DemoColab:
|
|
152
|
-
def email(self):
|
|
153
|
-
return "you@example.com"
|
|
154
|
-
|
|
155
|
-
def ccu_info(self):
|
|
156
|
-
return {
|
|
157
|
-
"currentBalance": 412.37,
|
|
158
|
-
"consumptionRateHourly": round(sum(RATES.values()), 2),
|
|
159
|
-
"assignmentsCount": len(ASSIGNMENTS),
|
|
160
|
-
"eligibleGpus": ["H100", "G4", "A100", "L4", "T4"],
|
|
161
|
-
"eligibleTpus": ["V6E1", "V5E1"],
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
def assignments(self):
|
|
165
|
-
return ASSIGNMENTS
|
|
166
|
-
|
|
167
|
-
def status(self, assignments):
|
|
168
|
-
return STATUS
|
|
169
|
-
|
|
170
|
-
def resources(self, runtimes):
|
|
171
|
-
return {r.endpoint: readings(r.endpoint)[-1] for r in runtimes}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
async def shoot(out: Path) -> None:
|
|
175
|
-
with tempfile.TemporaryDirectory() as tmp:
|
|
176
|
-
root = Path(tmp)
|
|
177
|
-
(root / "cli" / "history").mkdir(parents=True)
|
|
178
|
-
write_history(root / "cli" / "history")
|
|
179
|
-
write_cache(root / "cache")
|
|
180
|
-
app_module.CACHE = root / "cache"
|
|
181
|
-
app_module.CLI_HOME = root / "cli"
|
|
182
|
-
app_module.local_sessions = lambda: LOCAL
|
|
183
|
-
|
|
184
|
-
for name, size in (("dashboard", (138, 40)), ("small", (80, 24))):
|
|
185
|
-
app = Dashboard(DemoColab())
|
|
186
|
-
for a in ASSIGNMENTS:
|
|
187
|
-
for r in readings(a["endpoint"]):
|
|
188
|
-
app.samples.add(a["endpoint"], r.at, r)
|
|
189
|
-
async with app.run_test(size=size) as pilot:
|
|
190
|
-
await app.workers.wait_for_complete()
|
|
191
|
-
await pilot.pause()
|
|
192
|
-
await app.workers.wait_for_complete()
|
|
193
|
-
await pilot.pause()
|
|
194
|
-
app.save_screenshot(f"{name}.svg", path=str(out))
|
|
195
|
-
if name == "dashboard":
|
|
196
|
-
await pilot.press("down")
|
|
197
|
-
await pilot.pause()
|
|
198
|
-
app.save_screenshot("eval.svg", path=str(out))
|
|
199
|
-
await pilot.press("s")
|
|
200
|
-
await pilot.pause()
|
|
201
|
-
app.save_screenshot("stop.svg", path=str(out))
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
if __name__ == "__main__":
|
|
205
|
-
asyncio.run(shoot(Path(sys.argv[1])))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|