bqtop 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bqtop/__init__.py +3 -0
- bqtop/__main__.py +5 -0
- bqtop/app.py +395 -0
- bqtop/cli.py +145 -0
- bqtop/config.example.toml +63 -0
- bqtop/config.py +144 -0
- bqtop/fmt.py +108 -0
- bqtop/model.py +138 -0
- bqtop/pricing.py +61 -0
- bqtop/render.py +246 -0
- bqtop/sources/__init__.py +19 -0
- bqtop/sources/audit_log.py +73 -0
- bqtop/sources/base.py +179 -0
- bqtop/sources/demo.py +120 -0
- bqtop/sources/information_schema.py +79 -0
- bqtop/store.py +162 -0
- bqtop/wizard.py +171 -0
- bqtop-0.3.1.dist-info/METADATA +189 -0
- bqtop-0.3.1.dist-info/RECORD +22 -0
- bqtop-0.3.1.dist-info/WHEEL +4 -0
- bqtop-0.3.1.dist-info/entry_points.txt +2 -0
- bqtop-0.3.1.dist-info/licenses/LICENSE +21 -0
bqtop/__init__.py
ADDED
bqtop/__main__.py
ADDED
bqtop/app.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
"""Textual TUI: summary + cost timeline, principals / projects, hot tables / job stream, with filter,
|
|
2
|
+
drill-down, job detail, help and pause."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from zoneinfo import ZoneInfo
|
|
7
|
+
|
|
8
|
+
from rich.text import Text
|
|
9
|
+
from textual import on, work
|
|
10
|
+
from textual.app import App, ComposeResult
|
|
11
|
+
from textual.binding import Binding
|
|
12
|
+
from textual.containers import Horizontal, Vertical, VerticalScroll
|
|
13
|
+
from textual.screen import ModalScreen
|
|
14
|
+
from textual.widgets import DataTable, Footer, Header, Input, Sparkline, Static
|
|
15
|
+
|
|
16
|
+
from bqtop import fmt
|
|
17
|
+
from bqtop.config import Config
|
|
18
|
+
from bqtop.model import Job, Snapshot
|
|
19
|
+
from bqtop.render import SORT_KEYS, cap_cell, err_cell, footer_text, sort_aggs, state_cell, summary_text
|
|
20
|
+
from bqtop.sources.base import Source
|
|
21
|
+
from bqtop.store import JobStore
|
|
22
|
+
|
|
23
|
+
WINDOWS = (1, 6, 24, 72, 168)
|
|
24
|
+
|
|
25
|
+
HELP = """\
|
|
26
|
+
[b]bqtop[/b] · htop for BigQuery
|
|
27
|
+
|
|
28
|
+
[b]q[/b] quit [b]r[/b] refresh now
|
|
29
|
+
[b]w[/b] wider window (1h → 6h → 24h → 72h → 168h → 1h)
|
|
30
|
+
[b]s[/b] cycle sort: cost, bytes, jobs, errors, slots
|
|
31
|
+
[b]j[/b] hide/show the tables panel (widens the job stream)
|
|
32
|
+
[b]d[/b] tables panel: hot tables ↔ dbt models (cost per model, from dbt's query comment)
|
|
33
|
+
[b]/[/b] filter (principal, project, table, query text, error)
|
|
34
|
+
[b]esc[/b] clear filter / close dialog
|
|
35
|
+
[b]p[/b] pause auto-refresh
|
|
36
|
+
[b]enter[/b] on a principal, project or table: drill down (filter on it)
|
|
37
|
+
on a job: details with the full query text
|
|
38
|
+
[b]tab[/b] move between panels · arrows to move inside
|
|
39
|
+
|
|
40
|
+
[dim]cost = bytes billed × on-demand price, or slot-hours × slot price for jobs that ran in a
|
|
41
|
+
reservation (see [pricing] in the config). "today" is since local midnight and is what the
|
|
42
|
+
quota/budget percentages compare against.[/dim]
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class BqTop(App):
|
|
47
|
+
TITLE = "bqtop"
|
|
48
|
+
AUTO_FOCUS = "#jobs"
|
|
49
|
+
CSS = """
|
|
50
|
+
#summary { height: auto; padding: 0 1; background: $panel; }
|
|
51
|
+
#timeline { height: 2; margin: 0 1; }
|
|
52
|
+
#status { height: 1; padding: 0 1; color: $text-muted; }
|
|
53
|
+
#filter { display: none; margin: 0 1; }
|
|
54
|
+
DataTable { border: round $primary; height: 1fr; }
|
|
55
|
+
DataTable:focus { border: round $accent; }
|
|
56
|
+
#top { height: 42%; }
|
|
57
|
+
#bottom { height: 58%; }
|
|
58
|
+
#principals { width: 3fr; }
|
|
59
|
+
#projects { width: 2fr; }
|
|
60
|
+
#tables { width: 2fr; }
|
|
61
|
+
#jobs { width: 3fr; }
|
|
62
|
+
"""
|
|
63
|
+
BINDINGS = [
|
|
64
|
+
Binding("q", "quit", "quit"),
|
|
65
|
+
Binding("r", "refresh_now", "refresh"),
|
|
66
|
+
Binding("w", "cycle_window", "window"),
|
|
67
|
+
Binding("s", "cycle_sort", "sort"),
|
|
68
|
+
Binding("j", "toggle_tables", "tables"),
|
|
69
|
+
Binding("d", "toggle_models", "dbt models"),
|
|
70
|
+
Binding("slash", "open_filter", "filter"),
|
|
71
|
+
Binding("escape", "clear_filter", "clear", show=False),
|
|
72
|
+
Binding("p", "toggle_pause", "pause"),
|
|
73
|
+
Binding("question_mark", "help", "help"),
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
def __init__(self, cfg: Config, source: Source) -> None:
|
|
77
|
+
super().__init__()
|
|
78
|
+
self.cfg = cfg
|
|
79
|
+
self.source = source
|
|
80
|
+
self.store = JobStore(
|
|
81
|
+
source,
|
|
82
|
+
cfg.pricing,
|
|
83
|
+
tz=cfg.ui.timezone,
|
|
84
|
+
max_hours=cfg.ui.max_window_hours,
|
|
85
|
+
stream_rows=cfg.ui.stream_rows,
|
|
86
|
+
top_n=cfg.ui.top_n,
|
|
87
|
+
buckets=cfg.ui.timeline_buckets,
|
|
88
|
+
)
|
|
89
|
+
self.window_hours = cfg.ui.window_hours
|
|
90
|
+
self.sort = "cost"
|
|
91
|
+
self.filter = ""
|
|
92
|
+
self.paused = False
|
|
93
|
+
self.show_models = False
|
|
94
|
+
self.snap: Snapshot | None = None
|
|
95
|
+
|
|
96
|
+
def compose(self) -> ComposeResult:
|
|
97
|
+
yield Header(show_clock=True)
|
|
98
|
+
yield Static(id="summary")
|
|
99
|
+
yield Sparkline([0.0], summary_function=max, id="timeline")
|
|
100
|
+
yield Static("starting…", id="status")
|
|
101
|
+
yield Input(
|
|
102
|
+
placeholder="filter: principal, project, table, query text… (enter to apply, esc to cancel)", id="filter"
|
|
103
|
+
)
|
|
104
|
+
with Vertical():
|
|
105
|
+
with Horizontal(id="top"):
|
|
106
|
+
yield DataTable(id="principals", cursor_type="row", zebra_stripes=True)
|
|
107
|
+
yield DataTable(id="projects", cursor_type="row", zebra_stripes=True)
|
|
108
|
+
with Horizontal(id="bottom"):
|
|
109
|
+
yield DataTable(id="tables", cursor_type="row", zebra_stripes=True)
|
|
110
|
+
yield DataTable(id="jobs", cursor_type="row", zebra_stripes=True)
|
|
111
|
+
yield Footer()
|
|
112
|
+
|
|
113
|
+
def on_mount(self) -> None:
|
|
114
|
+
for tid, title in (
|
|
115
|
+
("principals", "principals"),
|
|
116
|
+
("projects", "projects"),
|
|
117
|
+
("tables", "hot tables"),
|
|
118
|
+
("jobs", "jobs"),
|
|
119
|
+
):
|
|
120
|
+
self.query_one(f"#{tid}", DataTable).border_title = title
|
|
121
|
+
self.set_interval(self.cfg.ui.refresh_seconds, self._tick)
|
|
122
|
+
self.action_refresh_now()
|
|
123
|
+
|
|
124
|
+
# ---- actions -------------------------------------------------------------------------------
|
|
125
|
+
def _tick(self) -> None:
|
|
126
|
+
if not self.paused:
|
|
127
|
+
self.action_refresh_now()
|
|
128
|
+
|
|
129
|
+
def action_refresh_now(self) -> None:
|
|
130
|
+
self._status(Text("refreshing…", style="yellow"))
|
|
131
|
+
self.load_snapshot()
|
|
132
|
+
|
|
133
|
+
def action_cycle_window(self) -> None:
|
|
134
|
+
i = WINDOWS.index(self.window_hours) if self.window_hours in WINDOWS else -1
|
|
135
|
+
self.window_hours = WINDOWS[(i + 1) % len(WINDOWS)]
|
|
136
|
+
self.action_refresh_now() # may backfill; the store decides
|
|
137
|
+
|
|
138
|
+
def action_cycle_sort(self) -> None:
|
|
139
|
+
self.sort = SORT_KEYS[(SORT_KEYS.index(self.sort) + 1) % len(SORT_KEYS)]
|
|
140
|
+
self._render_local()
|
|
141
|
+
|
|
142
|
+
def action_toggle_tables(self) -> None:
|
|
143
|
+
t = self.query_one("#tables", DataTable)
|
|
144
|
+
t.display = not t.display
|
|
145
|
+
|
|
146
|
+
def action_toggle_models(self) -> None:
|
|
147
|
+
self.show_models = not self.show_models
|
|
148
|
+
t = self.query_one("#tables", DataTable)
|
|
149
|
+
t.display = True
|
|
150
|
+
self._render_local()
|
|
151
|
+
|
|
152
|
+
def action_toggle_pause(self) -> None:
|
|
153
|
+
self.paused = not self.paused
|
|
154
|
+
if self.snap:
|
|
155
|
+
self._render_local()
|
|
156
|
+
|
|
157
|
+
def action_open_filter(self) -> None:
|
|
158
|
+
box = self.query_one("#filter", Input)
|
|
159
|
+
box.display = True
|
|
160
|
+
box.value = self.filter
|
|
161
|
+
box.focus()
|
|
162
|
+
|
|
163
|
+
def action_clear_filter(self) -> None:
|
|
164
|
+
box = self.query_one("#filter", Input)
|
|
165
|
+
if box.display:
|
|
166
|
+
box.display = False
|
|
167
|
+
self.set_focus(None)
|
|
168
|
+
return
|
|
169
|
+
if self.filter:
|
|
170
|
+
self.filter = ""
|
|
171
|
+
self._render_local()
|
|
172
|
+
|
|
173
|
+
def action_help(self) -> None:
|
|
174
|
+
self.push_screen(HelpScreen())
|
|
175
|
+
|
|
176
|
+
@on(Input.Submitted, "#filter")
|
|
177
|
+
def _filter_submitted(self, event: Input.Submitted) -> None:
|
|
178
|
+
self.filter = event.value.strip()
|
|
179
|
+
event.input.display = False
|
|
180
|
+
self.set_focus(self.query_one("#jobs", DataTable))
|
|
181
|
+
self._render_local()
|
|
182
|
+
|
|
183
|
+
@on(DataTable.RowSelected)
|
|
184
|
+
def _row_selected(self, event: DataTable.RowSelected) -> None:
|
|
185
|
+
key = event.row_key.value if event.row_key else None
|
|
186
|
+
if key is None:
|
|
187
|
+
return
|
|
188
|
+
tid = event.data_table.id
|
|
189
|
+
if tid == "jobs":
|
|
190
|
+
project_id, job_id = key.split("|", 1)
|
|
191
|
+
job = self.store.get(project_id, job_id)
|
|
192
|
+
if job:
|
|
193
|
+
self.push_screen(JobDetail(job, self.source, self.cfg))
|
|
194
|
+
else:
|
|
195
|
+
self.filter = key
|
|
196
|
+
self._render_local()
|
|
197
|
+
|
|
198
|
+
# ---- data ----------------------------------------------------------------------------------
|
|
199
|
+
@work(thread=True, exclusive=True, group="load")
|
|
200
|
+
def load_snapshot(self) -> None:
|
|
201
|
+
try:
|
|
202
|
+
self.store.refresh(self.window_hours)
|
|
203
|
+
snap = self.store.snapshot(self.window_hours, self.filter)
|
|
204
|
+
except Exception as e: # keep the last good screen, show the error
|
|
205
|
+
self.call_from_thread(self._status, Text(f"error: {fmt.one_line(str(e), 300)}", style="bold red"))
|
|
206
|
+
return
|
|
207
|
+
self.call_from_thread(self._apply, snap)
|
|
208
|
+
|
|
209
|
+
def _render_local(self) -> None:
|
|
210
|
+
"""Re-aggregate from the store without touching BigQuery (sort, filter, drill-down)."""
|
|
211
|
+
if self.store.coverage_start is None:
|
|
212
|
+
return
|
|
213
|
+
self._apply(self.store.snapshot(self.window_hours, self.filter))
|
|
214
|
+
|
|
215
|
+
def _apply(self, snap: Snapshot) -> None:
|
|
216
|
+
self.snap = snap
|
|
217
|
+
self.render_snapshot(snap)
|
|
218
|
+
self._status(footer_text(snap, self.cfg, self.source.describe()))
|
|
219
|
+
|
|
220
|
+
def _status(self, text: Text) -> None:
|
|
221
|
+
self.query_one("#status", Static).update(text)
|
|
222
|
+
|
|
223
|
+
# ---- rendering -----------------------------------------------------------------------------
|
|
224
|
+
def render_snapshot(self, snap: Snapshot) -> None:
|
|
225
|
+
cfg = self.cfg
|
|
226
|
+
tz = ZoneInfo(cfg.ui.timezone)
|
|
227
|
+
self.query_one("#summary", Static).update(summary_text(snap, cfg, self.source.describe(), self.paused))
|
|
228
|
+
self.query_one("#timeline", Sparkline).data = snap.timeline or [0.0]
|
|
229
|
+
|
|
230
|
+
t = self.query_one("#principals", DataTable)
|
|
231
|
+
t.border_title = f"principals · by {self.sort}"
|
|
232
|
+
_reset(t, ("principal", "jobs", "run", "err", "billed", "cost", "today", "slot-h", "budget"))
|
|
233
|
+
for a in sort_aggs(snap.by_principal, self.sort)[: cfg.ui.top_n]:
|
|
234
|
+
t.add_row(
|
|
235
|
+
fmt.short_principal(a.key),
|
|
236
|
+
_n(a.jobs),
|
|
237
|
+
_n(a.running),
|
|
238
|
+
err_cell(a.errors),
|
|
239
|
+
fmt.bytes_(a.bytes_billed),
|
|
240
|
+
fmt.money(a.cost),
|
|
241
|
+
fmt.money(a.cost_today),
|
|
242
|
+
fmt.slot_hours(a.slot_ms),
|
|
243
|
+
cap_cell(a, cfg, "principal"),
|
|
244
|
+
key=a.key,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
t = self.query_one("#projects", DataTable)
|
|
248
|
+
t.border_title = f"projects · by {self.sort}"
|
|
249
|
+
_reset(t, ("project", "jobs", "run", "err", "billed", "cost", "today", "quota/budget"))
|
|
250
|
+
for a in sort_aggs(snap.by_project, self.sort)[: cfg.ui.top_n]:
|
|
251
|
+
t.add_row(
|
|
252
|
+
a.key,
|
|
253
|
+
_n(a.jobs),
|
|
254
|
+
_n(a.running),
|
|
255
|
+
err_cell(a.errors),
|
|
256
|
+
fmt.bytes_(a.bytes_billed),
|
|
257
|
+
fmt.money(a.cost),
|
|
258
|
+
fmt.money(a.cost_today),
|
|
259
|
+
cap_cell(a, cfg, "project"),
|
|
260
|
+
key=a.key,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
t = self.query_one("#tables", DataTable)
|
|
264
|
+
if self.show_models:
|
|
265
|
+
t.border_title = "dbt models · from the query comment (d: hot tables)"
|
|
266
|
+
_reset(t, ("model", "runs", "err", "billed", "cost", "slot-h"))
|
|
267
|
+
for a in snap.by_model[: cfg.ui.top_n]:
|
|
268
|
+
t.add_row(
|
|
269
|
+
fmt.dbt_node(a.key),
|
|
270
|
+
_n(a.jobs),
|
|
271
|
+
err_cell(a.errors),
|
|
272
|
+
fmt.bytes_(a.bytes_billed),
|
|
273
|
+
fmt.money(a.cost),
|
|
274
|
+
fmt.slot_hours(a.slot_ms),
|
|
275
|
+
key=a.key,
|
|
276
|
+
)
|
|
277
|
+
if not snap.by_model:
|
|
278
|
+
t.add_row("no dbt query comments in this window", "", "", "", "", "", key="")
|
|
279
|
+
else:
|
|
280
|
+
t.border_title = "hot tables (d: dbt models)"
|
|
281
|
+
_reset(t, ("table", "jobs", "who", "billed", "cost"))
|
|
282
|
+
for a in snap.by_table[: cfg.ui.top_n]:
|
|
283
|
+
t.add_row(
|
|
284
|
+
a.key,
|
|
285
|
+
_n(a.jobs),
|
|
286
|
+
str(len(a.principals)),
|
|
287
|
+
fmt.bytes_(a.bytes_billed),
|
|
288
|
+
fmt.money(a.cost),
|
|
289
|
+
key=a.key,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
t = self.query_one("#jobs", DataTable)
|
|
293
|
+
t.border_title = (
|
|
294
|
+
"jobs · running first, then newest"
|
|
295
|
+
if self.source.has_running_jobs
|
|
296
|
+
else "jobs · newest first (completed only)"
|
|
297
|
+
)
|
|
298
|
+
if snap.filter:
|
|
299
|
+
t.border_title += f" · «{snap.filter}»"
|
|
300
|
+
_reset(t, ("time", "state", "principal", "project", "type", "dur", "billed", "cost", "query"))
|
|
301
|
+
qwidth = max(30, (t.size.width or 120) - 105)
|
|
302
|
+
for j in snap.jobs[: cfg.ui.stream_rows]:
|
|
303
|
+
text = j.error_message if j.error_code else (j.query or j.destination_table or "")
|
|
304
|
+
t.add_row(
|
|
305
|
+
fmt.clock(j.creation_time, tz),
|
|
306
|
+
state_cell(j.state, bool(j.error_code)),
|
|
307
|
+
fmt.short_principal(j.principal),
|
|
308
|
+
j.project_id,
|
|
309
|
+
fmt.stmt(j),
|
|
310
|
+
fmt.duration(j.duration_s()),
|
|
311
|
+
fmt.bytes_(j.bytes_billed),
|
|
312
|
+
fmt.money(j.cost),
|
|
313
|
+
fmt.one_line(text, qwidth),
|
|
314
|
+
key=f"{j.project_id}|{j.job_id}",
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
class JobDetail(ModalScreen):
|
|
319
|
+
BINDINGS = [Binding("escape", "dismiss", "close"), Binding("q", "dismiss", "close")]
|
|
320
|
+
DEFAULT_CSS = """
|
|
321
|
+
JobDetail { align: center middle; }
|
|
322
|
+
#detail { width: 90%; height: 90%; border: thick $accent; background: $surface; padding: 1 2; }
|
|
323
|
+
"""
|
|
324
|
+
|
|
325
|
+
def __init__(self, job: Job, source: Source, cfg: Config) -> None:
|
|
326
|
+
super().__init__()
|
|
327
|
+
self.job, self.source, self.cfg = job, source, cfg
|
|
328
|
+
|
|
329
|
+
def compose(self) -> ComposeResult:
|
|
330
|
+
with VerticalScroll(id="detail"):
|
|
331
|
+
yield Static(self._header(), id="detail_head")
|
|
332
|
+
yield Static(Text(self.job.query or "(no query text)", style="cyan"), id="detail_query")
|
|
333
|
+
|
|
334
|
+
def on_mount(self) -> None:
|
|
335
|
+
if self.job.query and len(self.job.query) >= 300:
|
|
336
|
+
self.fetch_full_query()
|
|
337
|
+
|
|
338
|
+
def _header(self) -> Text:
|
|
339
|
+
j, tz = self.job, ZoneInfo(self.cfg.ui.timezone)
|
|
340
|
+
t = Text()
|
|
341
|
+
t.append(f"{j.project_id} / {j.job_id}\n", style="bold")
|
|
342
|
+
rows = [
|
|
343
|
+
("principal", j.principal),
|
|
344
|
+
("state", j.state),
|
|
345
|
+
("type", fmt.stmt(j)),
|
|
346
|
+
("created", j.creation_time.astimezone(tz).strftime("%Y-%m-%d %H:%M:%S")),
|
|
347
|
+
("duration", fmt.duration(j.duration_s())),
|
|
348
|
+
("bytes processed / billed", f"{fmt.bytes_(j.bytes_processed)} / {fmt.bytes_(j.bytes_billed)}"),
|
|
349
|
+
("cost", fmt.money(j.cost) + (" (reservation " + j.reservation_id + ")" if j.reservation_id else "")),
|
|
350
|
+
("slot-h", fmt.slot_hours(j.slot_ms)),
|
|
351
|
+
("cache hit", str(j.cache_hit)),
|
|
352
|
+
("destination", j.destination_table or "-"),
|
|
353
|
+
("tables", "\n" + "\n".join(f" {x}" for x in j.referenced_tables) if j.referenced_tables else "-"),
|
|
354
|
+
]
|
|
355
|
+
if j.error_code:
|
|
356
|
+
rows.append(("error", f"{j.error_code}: {j.error_message}"))
|
|
357
|
+
for k, v in rows:
|
|
358
|
+
t.append(f"{k:>26} ", style="dim")
|
|
359
|
+
t.append(f"{v}\n", style="red" if k == "error" else "")
|
|
360
|
+
t.append("\nquery ", style="dim")
|
|
361
|
+
t.append("(esc to close)\n", style="dim")
|
|
362
|
+
return t
|
|
363
|
+
|
|
364
|
+
@work(thread=True)
|
|
365
|
+
def fetch_full_query(self) -> None:
|
|
366
|
+
try:
|
|
367
|
+
full = self.source.fetch_query(self.job.project_id, self.job.job_id)
|
|
368
|
+
except Exception as e:
|
|
369
|
+
full = f"{self.job.query}\n\n[could not fetch the full text: {fmt.one_line(str(e), 200)}]"
|
|
370
|
+
if full:
|
|
371
|
+
self.app.call_from_thread(self.query_one("#detail_query", Static).update, Text(full, style="cyan"))
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
class HelpScreen(ModalScreen):
|
|
375
|
+
BINDINGS = [
|
|
376
|
+
Binding("escape", "dismiss", "close"),
|
|
377
|
+
Binding("q", "dismiss", "close"),
|
|
378
|
+
Binding("question_mark", "dismiss", "close"),
|
|
379
|
+
]
|
|
380
|
+
DEFAULT_CSS = """
|
|
381
|
+
HelpScreen { align: center middle; }
|
|
382
|
+
#help { width: 96; height: auto; border: thick $accent; background: $surface; padding: 1 2; }
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
def compose(self) -> ComposeResult:
|
|
386
|
+
yield Static(HELP, id="help")
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _reset(t: DataTable, cols: tuple[str, ...]) -> None:
|
|
390
|
+
t.clear(columns=True)
|
|
391
|
+
t.add_columns(*cols)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _n(n: int) -> str:
|
|
395
|
+
return f"{n:,}" if n else ""
|
bqtop/cli.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import asdict, is_dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from bqtop import __version__
|
|
11
|
+
from bqtop.config import Config, ConfigError, load
|
|
12
|
+
|
|
13
|
+
HINTS = (
|
|
14
|
+
(
|
|
15
|
+
"jobs.listAll",
|
|
16
|
+
"grant roles/bigquery.resourceViewer at that level (project, folder or org), "
|
|
17
|
+
'or use scope = "user" / kind = "audit_log"',
|
|
18
|
+
),
|
|
19
|
+
("bigquery.jobs.create", "grant roles/bigquery.jobUser on [source].billing_project"),
|
|
20
|
+
("Not found: Table", "check [source].table (audit_log) or [source].projects / regions"),
|
|
21
|
+
(
|
|
22
|
+
"could not automatically determine credentials",
|
|
23
|
+
"run `gcloud auth application-default login` or set GOOGLE_APPLICATION_CREDENTIALS",
|
|
24
|
+
),
|
|
25
|
+
("Reauthentication", "run `gcloud auth application-default login` again"),
|
|
26
|
+
("has not been used in project", "enable the BigQuery API on [source].billing_project"),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main(argv: list[str] | None = None) -> int:
|
|
31
|
+
ap = argparse.ArgumentParser(prog="bqtop", description="htop for BigQuery: jobs, principals, projects, cost.")
|
|
32
|
+
ap.add_argument("-c", "--config", help="config file (default: ./bqtop.toml, then ~/.config/bqtop/config.toml)")
|
|
33
|
+
ap.add_argument("-w", "--window", type=float, help="window in hours (overrides config)")
|
|
34
|
+
ap.add_argument(
|
|
35
|
+
"-f", "--filter", default="", help="only jobs matching this text (principal, project, table, query)"
|
|
36
|
+
)
|
|
37
|
+
ap.add_argument("--sort", default="cost", choices=("cost", "bytes", "jobs", "errors", "slots"))
|
|
38
|
+
ap.add_argument("--once", action="store_true", help="print one snapshot and exit")
|
|
39
|
+
ap.add_argument("--json", action="store_true", help="with --once: emit the snapshot as JSON")
|
|
40
|
+
ap.add_argument("--watch", type=int, metavar="SECONDS", help="plain-text mode: reprint every N seconds")
|
|
41
|
+
ap.add_argument("--demo", action="store_true", help="run on synthetic data, no GCP needed")
|
|
42
|
+
ap.add_argument("--check", action="store_true", help="check credentials, permissions and the source, then exit")
|
|
43
|
+
ap.add_argument("--init", action="store_true", help="interactive setup, writes ~/.config/bqtop/config.toml")
|
|
44
|
+
ap.add_argument("-y", "--yes", action="store_true", help="with --init: skip questions, write the example config")
|
|
45
|
+
ap.add_argument("--version", action="version", version=f"bqtop {__version__}")
|
|
46
|
+
args = ap.parse_args(argv)
|
|
47
|
+
|
|
48
|
+
if args.init:
|
|
49
|
+
from bqtop.wizard import run
|
|
50
|
+
|
|
51
|
+
return run(Path(args.config) if args.config else None, assume_yes=args.yes)
|
|
52
|
+
|
|
53
|
+
if args.demo:
|
|
54
|
+
cfg = Config.demo()
|
|
55
|
+
else:
|
|
56
|
+
try:
|
|
57
|
+
cfg = load(args.config)
|
|
58
|
+
except ConfigError as e:
|
|
59
|
+
print(f"bqtop: {e}", file=sys.stderr)
|
|
60
|
+
return 2
|
|
61
|
+
if args.window:
|
|
62
|
+
cfg.ui.window_hours = args.window
|
|
63
|
+
cfg.ui.max_window_hours = max(cfg.ui.max_window_hours, args.window)
|
|
64
|
+
|
|
65
|
+
from bqtop.sources import make_source
|
|
66
|
+
|
|
67
|
+
source = make_source(cfg)
|
|
68
|
+
|
|
69
|
+
if args.check:
|
|
70
|
+
return _check(cfg, source)
|
|
71
|
+
|
|
72
|
+
if args.once or args.json or args.watch:
|
|
73
|
+
from rich.console import Console
|
|
74
|
+
|
|
75
|
+
from bqtop.render import render_once
|
|
76
|
+
from bqtop.store import JobStore
|
|
77
|
+
|
|
78
|
+
store = JobStore(
|
|
79
|
+
source,
|
|
80
|
+
cfg.pricing,
|
|
81
|
+
tz=cfg.ui.timezone,
|
|
82
|
+
max_hours=cfg.ui.max_window_hours,
|
|
83
|
+
stream_rows=cfg.ui.stream_rows,
|
|
84
|
+
top_n=cfg.ui.top_n,
|
|
85
|
+
buckets=cfg.ui.timeline_buckets,
|
|
86
|
+
)
|
|
87
|
+
console = Console()
|
|
88
|
+
while True:
|
|
89
|
+
store.refresh(cfg.ui.window_hours)
|
|
90
|
+
snap = store.snapshot(cfg.ui.window_hours, args.filter)
|
|
91
|
+
if args.json:
|
|
92
|
+
out = _to_dict(snap)
|
|
93
|
+
out["totals"] = _to_dict(snap.totals)
|
|
94
|
+
json.dump(out, sys.stdout, default=str, indent=1)
|
|
95
|
+
print()
|
|
96
|
+
else:
|
|
97
|
+
if args.watch:
|
|
98
|
+
console.clear()
|
|
99
|
+
render_once(snap, cfg, source.describe(), console=console, sort=args.sort)
|
|
100
|
+
if not args.watch:
|
|
101
|
+
return 0
|
|
102
|
+
time.sleep(args.watch)
|
|
103
|
+
|
|
104
|
+
from bqtop.app import BqTop
|
|
105
|
+
|
|
106
|
+
BqTop(cfg, source).run()
|
|
107
|
+
return 0
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _check(cfg: Config, source) -> int:
|
|
111
|
+
print(f"config {cfg.path or '(demo)'}")
|
|
112
|
+
print(f"source {source.describe()}")
|
|
113
|
+
print(f"pricing {cfg.pricing.describe()}")
|
|
114
|
+
if cfg.quotas:
|
|
115
|
+
print(f"quotas {len(cfg.quotas)} project cap(s)")
|
|
116
|
+
if cfg.budgets:
|
|
117
|
+
print(f"budgets {len(cfg.budgets)} daily budget(s)")
|
|
118
|
+
ok_all = True
|
|
119
|
+
for what, ok, detail in source.probe():
|
|
120
|
+
mark = "✓" if ok else "✗"
|
|
121
|
+
print(f"{mark} {what:<8} {detail}")
|
|
122
|
+
if not ok:
|
|
123
|
+
ok_all = False
|
|
124
|
+
for needle, hint in HINTS:
|
|
125
|
+
if needle.lower() in detail.lower():
|
|
126
|
+
print(f" → {hint}")
|
|
127
|
+
if ok_all:
|
|
128
|
+
print("all good. run `bqtop` for the TUI or `bqtop --once` for a snapshot.")
|
|
129
|
+
return 0 if ok_all else 1
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _to_dict(obj):
|
|
133
|
+
if is_dataclass(obj):
|
|
134
|
+
return {k: _to_dict(v) for k, v in asdict(obj).items()}
|
|
135
|
+
if isinstance(obj, set):
|
|
136
|
+
return sorted(obj)
|
|
137
|
+
if isinstance(obj, list):
|
|
138
|
+
return [_to_dict(x) for x in obj]
|
|
139
|
+
if isinstance(obj, dict):
|
|
140
|
+
return {k: _to_dict(v) for k, v in obj.items()}
|
|
141
|
+
return obj
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__":
|
|
145
|
+
sys.exit(main())
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# bqtop configuration. Copy to ~/.config/bqtop/config.toml (or ./bqtop.toml in the current dir).
|
|
2
|
+
# Try `bqtop --demo` first if you just want to see what it looks like.
|
|
3
|
+
|
|
4
|
+
[source]
|
|
5
|
+
# information_schema: real time, includes RUNNING jobs, reads INFORMATION_SCHEMA.JOBS_BY_* (default).
|
|
6
|
+
# audit_log: reads a BigQuery audit-log sink table (cloudaudit_googleapis_com_data_access);
|
|
7
|
+
# completed jobs only, works across projects without jobs.listAll on each.
|
|
8
|
+
kind = "information_schema"
|
|
9
|
+
|
|
10
|
+
# Project that runs bqtop's own queries (and pays for them: ~10 MiB per refresh on INFORMATION_SCHEMA).
|
|
11
|
+
billing_project = "my-admin-project"
|
|
12
|
+
|
|
13
|
+
# BigQuery region(s) of the INFORMATION_SCHEMA views: "us", "eu", "europe-west1", ...
|
|
14
|
+
regions = ["us"]
|
|
15
|
+
|
|
16
|
+
# information_schema only. project | folder | organization | user
|
|
17
|
+
# project -> JOBS_BY_PROJECT for each entry in `projects` (needs bigquery.jobs.listAll per project,
|
|
18
|
+
# e.g. roles/bigquery.resourceViewer)
|
|
19
|
+
# folder -> JOBS_BY_FOLDER: every project in the folder that DIRECTLY contains billing_project, plus its
|
|
20
|
+
# sub-folders (jobs.listAll on that folder). To watch a whole tree, pick a billing_project that sits
|
|
21
|
+
# right under the top folder; `bqtop --check` prints how many projects it actually sees.
|
|
22
|
+
# organization -> JOBS_BY_ORGANIZATION (jobs.listAll on the org)
|
|
23
|
+
# user -> JOBS_BY_USER: your own jobs, no extra permissions
|
|
24
|
+
scope = "project"
|
|
25
|
+
projects = ["my-admin-project", "my-data-project"]
|
|
26
|
+
|
|
27
|
+
# audit_log only: the routed data-access table.
|
|
28
|
+
# table = "my-admin-project.bq_audit_logs.cloudaudit_googleapis_com_data_access"
|
|
29
|
+
|
|
30
|
+
# Optional: a service-account key to use instead of Application Default Credentials. Handy when your
|
|
31
|
+
# user login expires or you want a dedicated read-only identity for the dashboard.
|
|
32
|
+
# credentials_file = "~/.config/gcloud/bqtop-reader.json"
|
|
33
|
+
|
|
34
|
+
[pricing]
|
|
35
|
+
# auto: jobs that ran in a reservation are priced in slot-hours, everything else on demand
|
|
36
|
+
# on_demand: bytes billed × on_demand_usd_per_tib
|
|
37
|
+
# slots: slot-ms × slot_usd_per_hour (Editions pay-as-you-go: standard 0.04, enterprise 0.06, plus 0.10)
|
|
38
|
+
mode = "auto"
|
|
39
|
+
on_demand_usd_per_tib = 6.25
|
|
40
|
+
slot_usd_per_hour = 0.06
|
|
41
|
+
|
|
42
|
+
# Per-project overrides, same keys.
|
|
43
|
+
# [pricing.projects]
|
|
44
|
+
# "my-dbt-project" = { mode = "slots", slot_usd_per_hour = 0.04 }
|
|
45
|
+
|
|
46
|
+
[quotas]
|
|
47
|
+
# Optional daily QueryUsagePerDay caps per project, shown as % used today (bytes billed since local midnight).
|
|
48
|
+
# "my-data-project" = "10 TiB"
|
|
49
|
+
# "my-agents-project" = "5 TiB"
|
|
50
|
+
|
|
51
|
+
[budgets]
|
|
52
|
+
# Optional daily spend budgets in USD, keyed by principal or project. Shown as % and coloured.
|
|
53
|
+
# "svc-airflow@my-ingest-project.iam.gserviceaccount.com" = 50
|
|
54
|
+
# "my-agents-project" = "$10"
|
|
55
|
+
|
|
56
|
+
[ui]
|
|
57
|
+
refresh_seconds = 60 # audit_log sources: 120 is plenty, each refresh rescans today's partition
|
|
58
|
+
window_hours = 24 # initial window; `w` cycles 1h, 6h, 24h, 72h, 168h
|
|
59
|
+
max_window_hours = 168 # how much history the local store keeps
|
|
60
|
+
timezone = "UTC" # for "today" (quotas, budgets) and clock columns
|
|
61
|
+
top_n = 15
|
|
62
|
+
stream_rows = 40
|
|
63
|
+
timeline_buckets = 48
|