richqueue 0.6__py3-none-any.whl → 0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- richqueue/console.py +3 -0
- richqueue/layout.py +29 -0
- richqueue/main.py +90 -0
- richqueue/slurm.py +232 -379
- richqueue/table.py +368 -0
- richqueue/tools.py +7 -0
- {richqueue-0.6.dist-info → richqueue-0.7.dist-info}/METADATA +13 -1
- richqueue-0.7.dist-info/RECORD +12 -0
- richqueue-0.6.dist-info/RECORD +0 -8
- {richqueue-0.6.dist-info → richqueue-0.7.dist-info}/WHEEL +0 -0
- {richqueue-0.6.dist-info → richqueue-0.7.dist-info}/entry_points.txt +0 -0
- {richqueue-0.6.dist-info → richqueue-0.7.dist-info}/licenses/LICENSE +0 -0
richqueue/console.py
ADDED
richqueue/layout.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from rich.layout import Layout
|
|
2
|
+
from rich.panel import Panel
|
|
3
|
+
from .slurm import PANEL_PADDING
|
|
4
|
+
from rich.text import Text
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def dual_layout(function, **kwargs):
|
|
8
|
+
|
|
9
|
+
layout = Layout()
|
|
10
|
+
|
|
11
|
+
upper, lower = function(**kwargs)
|
|
12
|
+
|
|
13
|
+
upper = Layout(renderable=upper, name="upper")
|
|
14
|
+
lower = Layout(renderable=lower, name="lower")
|
|
15
|
+
|
|
16
|
+
upper.size = upper.renderable.renderable.row_count + PANEL_PADDING
|
|
17
|
+
lower.size = lower.renderable.renderable.row_count + PANEL_PADDING
|
|
18
|
+
|
|
19
|
+
layout.split_column(
|
|
20
|
+
upper,
|
|
21
|
+
lower,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
return layout
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def simple_layout():
|
|
28
|
+
|
|
29
|
+
raise NotImplementedError
|
richqueue/main.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from typer import Typer
|
|
2
|
+
from .layout import dual_layout
|
|
3
|
+
from rich.live import Live
|
|
4
|
+
import time
|
|
5
|
+
from .console import console
|
|
6
|
+
from .tools import curry
|
|
7
|
+
from .slurm import get_layout_pair, get_node_layout, get_hist_layout
|
|
8
|
+
|
|
9
|
+
# set up singletons
|
|
10
|
+
app = Typer()
|
|
11
|
+
|
|
12
|
+
# main CLI command
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@app.command()
|
|
16
|
+
def show(
|
|
17
|
+
user: None | str = None,
|
|
18
|
+
long: bool = False,
|
|
19
|
+
idle: bool = False,
|
|
20
|
+
loop: bool = True,
|
|
21
|
+
hist: int | None = None,
|
|
22
|
+
hist_unit: str = "weeks",
|
|
23
|
+
# screen: bool = True,
|
|
24
|
+
# disappear: bool = True,
|
|
25
|
+
):
|
|
26
|
+
|
|
27
|
+
screen = True
|
|
28
|
+
disappear = not screen
|
|
29
|
+
|
|
30
|
+
kwargs = {
|
|
31
|
+
"user": user,
|
|
32
|
+
"long": long,
|
|
33
|
+
"idle": idle,
|
|
34
|
+
"loop": loop,
|
|
35
|
+
"hist": hist,
|
|
36
|
+
"hist_unit": hist_unit,
|
|
37
|
+
"screen": screen,
|
|
38
|
+
"disappear": disappear,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
# console.print(kwargs)
|
|
42
|
+
|
|
43
|
+
match (bool(idle), bool(hist)):
|
|
44
|
+
case (True, False):
|
|
45
|
+
loop = False
|
|
46
|
+
layout_func = get_node_layout
|
|
47
|
+
case (False, True):
|
|
48
|
+
loop = False
|
|
49
|
+
layout_func = get_hist_layout
|
|
50
|
+
case (False, False):
|
|
51
|
+
layout_func = curry(dual_layout, get_layout_pair)
|
|
52
|
+
case _:
|
|
53
|
+
raise Exception("Unsupported CLI options")
|
|
54
|
+
|
|
55
|
+
# live updating layout
|
|
56
|
+
if loop:
|
|
57
|
+
|
|
58
|
+
layout = layout_func(**kwargs)
|
|
59
|
+
|
|
60
|
+
with Live(
|
|
61
|
+
layout,
|
|
62
|
+
refresh_per_second=4,
|
|
63
|
+
screen=screen,
|
|
64
|
+
transient=disappear,
|
|
65
|
+
vertical_overflow="visible",
|
|
66
|
+
) as live:
|
|
67
|
+
|
|
68
|
+
try:
|
|
69
|
+
while True:
|
|
70
|
+
layout = layout_func(**kwargs)
|
|
71
|
+
live.update(layout)
|
|
72
|
+
time.sleep(1)
|
|
73
|
+
except KeyboardInterrupt:
|
|
74
|
+
live.stop()
|
|
75
|
+
|
|
76
|
+
# static layout
|
|
77
|
+
else:
|
|
78
|
+
|
|
79
|
+
layout = layout_func(**kwargs)
|
|
80
|
+
console.print(layout)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# start Typer app
|
|
84
|
+
def main():
|
|
85
|
+
app()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# start Typer app
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
app()
|
richqueue/slurm.py
CHANGED
|
@@ -1,284 +1,147 @@
|
|
|
1
|
-
from
|
|
2
|
-
import json
|
|
3
|
-
import datetime
|
|
4
|
-
from rich.console import Console
|
|
1
|
+
from pandas import concat, DataFrame, isnull
|
|
5
2
|
from rich.panel import Panel
|
|
6
|
-
from rich.
|
|
7
|
-
from pandas import DataFrame
|
|
3
|
+
from rich.text import Text
|
|
8
4
|
import subprocess
|
|
5
|
+
import json
|
|
6
|
+
from .console import console
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import datetime
|
|
9
|
+
from .table import running_job_table, history_job_table, node_table
|
|
10
|
+
from .tools import human_timedelta
|
|
9
11
|
|
|
10
|
-
|
|
12
|
+
# from numpy import isnat
|
|
11
13
|
|
|
12
14
|
METADATA = {}
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
PANEL_PADDING = 4
|
|
15
17
|
|
|
18
|
+
### CONSTRUCT LAYOUT
|
|
16
19
|
|
|
17
|
-
def color_by_state(state):
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
return "[bold bright_green]Running"
|
|
21
|
-
elif state == "PENDING":
|
|
22
|
-
return "[bright_yellow]Pending"
|
|
23
|
-
elif state == "CANCELLED":
|
|
24
|
-
return "[orange3]Cancelled"
|
|
25
|
-
elif state == "FAILED":
|
|
26
|
-
return "[bold bright_red]Failed"
|
|
27
|
-
elif state == "COMPLETED":
|
|
28
|
-
return "[bold bright_green]Completed"
|
|
29
|
-
else:
|
|
30
|
-
return state
|
|
21
|
+
def get_layout_pair(user: str | None, **kwargs):
|
|
31
22
|
|
|
32
|
-
|
|
23
|
+
if user == "all":
|
|
24
|
+
user = None
|
|
25
|
+
elif user is None:
|
|
26
|
+
x = subprocess.Popen(["whoami"], shell=True, stdout=subprocess.PIPE)
|
|
27
|
+
output = x.communicate()
|
|
28
|
+
user = output[0].strip().decode("utf-8")
|
|
33
29
|
|
|
34
|
-
|
|
35
|
-
return "[orange1]Allocated"
|
|
36
|
-
elif state == "MIXED":
|
|
37
|
-
return "[bright_yellow]Mixed"
|
|
38
|
-
elif state == "IDLE":
|
|
39
|
-
return "[bright_green]Idle"
|
|
40
|
-
else:
|
|
41
|
-
return f"[red]{state}"
|
|
30
|
+
df = combined_df(user=user, **kwargs)
|
|
42
31
|
|
|
43
|
-
|
|
32
|
+
n_rows = len(df)
|
|
33
|
+
n_running = len(df[df["job_state"] == "RUNNING"])
|
|
34
|
+
n_pending = len(df[df["job_state"] == "PENDING"])
|
|
44
35
|
|
|
45
|
-
|
|
46
|
-
d = x[key]
|
|
47
|
-
if "set" in d:
|
|
48
|
-
if d["set"]:
|
|
49
|
-
return d[inner]
|
|
50
|
-
else:
|
|
51
|
-
return None
|
|
52
|
-
else:
|
|
53
|
-
return d[inner]
|
|
36
|
+
hide_pending = False
|
|
54
37
|
|
|
55
|
-
|
|
38
|
+
running_df = df[df["job_state"].isin(["RUNNING", "PENDING"])]
|
|
39
|
+
history_df = df[~df["job_state"].isin(["RUNNING", "PENDING"])]
|
|
56
40
|
|
|
41
|
+
console_height = console.size.height
|
|
42
|
+
max_rows = console_height - 2 * PANEL_PADDING
|
|
57
43
|
|
|
58
|
-
|
|
59
|
-
df[key] = df.apply(
|
|
60
|
-
lambda x: datetime.datetime.fromtimestamp(x[key]["number"]), axis=1
|
|
61
|
-
)
|
|
44
|
+
if n_rows > max_rows:
|
|
62
45
|
|
|
46
|
+
# hide history?
|
|
47
|
+
if n_running + n_pending + PANEL_PADDING + 1 < console_height:
|
|
48
|
+
history_limit = 0
|
|
49
|
+
running_limit = None
|
|
63
50
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
)
|
|
71
|
-
df["submit_time"] = df.apply(
|
|
72
|
-
lambda x: datetime.datetime.fromtimestamp(x["time"]["submission"]), axis=1
|
|
73
|
-
)
|
|
74
|
-
|
|
51
|
+
# hide pending?
|
|
52
|
+
elif n_rows - n_pending < max_rows:
|
|
53
|
+
# running_df = running_df[running_df["job_state"]=="RUNNING"]
|
|
54
|
+
running_limit = None
|
|
55
|
+
history_limit = None
|
|
56
|
+
hide_pending = n_pending
|
|
75
57
|
|
|
76
|
-
|
|
77
|
-
def inner(x):
|
|
78
|
-
if len(x[key]) == 1:
|
|
79
|
-
return x[key][0]
|
|
58
|
+
# fallback clip
|
|
80
59
|
else:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
df[key] = df.apply(inner, axis=1)
|
|
84
|
-
|
|
85
|
-
def mem_string(mb):
|
|
86
|
-
|
|
87
|
-
# gb, mb = divmod(mb, 1024)
|
|
88
|
-
|
|
89
|
-
# return f"{gb}G {mb}M"
|
|
60
|
+
running_limit = 5
|
|
61
|
+
history_limit = 5
|
|
90
62
|
|
|
91
|
-
return f"{mb/1024:.0f}G"
|
|
92
|
-
|
|
93
|
-
def node_table(
|
|
94
|
-
df,
|
|
95
|
-
title: str = None,
|
|
96
|
-
box: bool = False,
|
|
97
|
-
panel: bool = True,
|
|
98
|
-
idle: bool = True,
|
|
99
|
-
):
|
|
100
|
-
|
|
101
|
-
if idle:
|
|
102
|
-
df = df[df["node"].isin(["IDLE", "MIXED"])]
|
|
103
|
-
title = title or "idle nodes"
|
|
104
63
|
else:
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
table.add_column("[bold underline]State", justify='left', no_wrap=True, style="bold")
|
|
118
|
-
table.add_column("[green_yellow underline]Partition", justify='left', no_wrap=True, style="green_yellow")
|
|
119
|
-
table.add_column("[magenta underline]#C", justify='right', no_wrap=True, style="magenta")
|
|
120
|
-
table.add_column("[magenta underline]Memory", justify='right', no_wrap=True, style="magenta")
|
|
121
|
-
table.add_column("[dodger_blue2 underline]Features", justify='left', no_wrap=True, style="dodger_blue2")
|
|
122
|
-
|
|
123
|
-
reservation = any(df["reservation"].values)
|
|
124
|
-
|
|
125
|
-
if reservation:
|
|
126
|
-
table.add_column("[bold underline]Reservation", justify='left', no_wrap=True, style="bold")
|
|
127
|
-
|
|
128
|
-
for i,row in df.iterrows():
|
|
129
|
-
|
|
130
|
-
values = []
|
|
131
|
-
|
|
132
|
-
values.append(row.nodes)
|
|
133
|
-
# values.append(row.nodes)
|
|
134
|
-
values.append(color_by_node_state(row.node))
|
|
135
|
-
values.append(row.partition)
|
|
136
|
-
values.append(str(row.cpus_idle))
|
|
137
|
-
values.append(mem_string(row.memory_free))
|
|
138
|
-
values.append(row.features)
|
|
139
|
-
|
|
140
|
-
if reservation:
|
|
141
|
-
values.append(row.reservation)
|
|
142
|
-
|
|
143
|
-
table.add_row(*values)
|
|
144
|
-
|
|
145
|
-
# table.add_row(*[str(v) for v in row.values])
|
|
146
|
-
|
|
147
|
-
if panel:
|
|
148
|
-
return Panel(table, expand=False)
|
|
149
|
-
|
|
150
|
-
return table
|
|
151
|
-
|
|
152
|
-
def job_table(
|
|
153
|
-
df,
|
|
154
|
-
title: str = "jobs",
|
|
155
|
-
long: bool = False,
|
|
156
|
-
force_submit_time: bool = False,
|
|
157
|
-
return_table: bool = False,
|
|
158
|
-
box: bool = False,
|
|
159
|
-
panel: bool = True,
|
|
160
|
-
):
|
|
161
|
-
|
|
162
|
-
from rich.table import Table
|
|
163
|
-
|
|
164
|
-
if box:
|
|
165
|
-
from rich.box import SIMPLE
|
|
166
|
-
|
|
167
|
-
box = SIMPLE
|
|
168
|
-
|
|
169
|
-
from .tools import human_timedelta, human_datetime
|
|
170
|
-
|
|
171
|
-
num_nodes = "node_count" in df.columns
|
|
172
|
-
num_cores = "cpus" in df.columns
|
|
173
|
-
start_time = "start_time" in df.columns
|
|
174
|
-
end_time = "end_time" in df.columns
|
|
175
|
-
submit_time = "submit_time" in df.columns
|
|
176
|
-
|
|
177
|
-
table = Table(title=title, box=box, header_style="")
|
|
178
|
-
|
|
179
|
-
table.add_column(
|
|
180
|
-
"[bold underline]Job Id", justify="right", style="bold", no_wrap=True
|
|
64
|
+
running_limit = None
|
|
65
|
+
history_limit = None
|
|
66
|
+
|
|
67
|
+
running = Panel(
|
|
68
|
+
running_job_table(
|
|
69
|
+
running_df,
|
|
70
|
+
limit=running_limit,
|
|
71
|
+
hide_pending=hide_pending,
|
|
72
|
+
user=user,
|
|
73
|
+
**kwargs,
|
|
74
|
+
),
|
|
75
|
+
expand=False,
|
|
181
76
|
)
|
|
182
|
-
table.add_column(
|
|
183
|
-
"[underline cyan]Job Name", justify="left", style="cyan", no_wrap=False
|
|
184
|
-
)
|
|
185
|
-
if num_nodes:
|
|
186
|
-
table.add_column(
|
|
187
|
-
"[underline magenta]#N", justify="right", style="magenta", no_wrap=True
|
|
188
|
-
)
|
|
189
|
-
if num_cores:
|
|
190
|
-
table.add_column(
|
|
191
|
-
"[underline magenta]#C", justify="right", style="magenta", no_wrap=True
|
|
192
|
-
)
|
|
193
77
|
|
|
194
|
-
if
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
)
|
|
201
|
-
|
|
202
|
-
if long and start_time:
|
|
203
|
-
table.add_column(
|
|
204
|
-
"[underline dodger_blue2]Started",
|
|
205
|
-
justify="right",
|
|
206
|
-
style="dodger_blue2",
|
|
207
|
-
no_wrap=True,
|
|
78
|
+
if history_limit == 0:
|
|
79
|
+
history = Text("history hidden, resize window or use --hist")
|
|
80
|
+
else:
|
|
81
|
+
history = Panel(
|
|
82
|
+
history_job_table(history_df, limit=history_limit, user=user, **kwargs),
|
|
83
|
+
expand=False,
|
|
208
84
|
)
|
|
209
85
|
|
|
210
|
-
|
|
211
|
-
table.add_column(
|
|
212
|
-
"[underline dodger_blue2]Run Time",
|
|
213
|
-
justify="right",
|
|
214
|
-
style="dodger_blue2",
|
|
215
|
-
no_wrap=True,
|
|
216
|
-
)
|
|
86
|
+
return running, history
|
|
217
87
|
|
|
218
|
-
if long:
|
|
219
|
-
table.add_column(
|
|
220
|
-
"[underline green_yellow]Partition",
|
|
221
|
-
justify="right",
|
|
222
|
-
style="green_yellow",
|
|
223
|
-
no_wrap=True,
|
|
224
|
-
)
|
|
225
|
-
table.add_column(
|
|
226
|
-
"[underline green_yellow]Nodes",
|
|
227
|
-
justify="left",
|
|
228
|
-
style="green_yellow",
|
|
229
|
-
no_wrap=False,
|
|
230
|
-
)
|
|
231
88
|
|
|
232
|
-
|
|
89
|
+
def get_hist_layout(user: str | None, **kwargs):
|
|
233
90
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
91
|
+
if user == "all":
|
|
92
|
+
user = None
|
|
93
|
+
elif user is None:
|
|
94
|
+
x = subprocess.Popen(["whoami"], shell=True, stdout=subprocess.PIPE)
|
|
95
|
+
output = x.communicate()
|
|
96
|
+
user = output[0].strip().decode("utf-8")
|
|
238
97
|
|
|
239
|
-
|
|
240
|
-
start_time = human_datetime(row.start_time)
|
|
98
|
+
df = combined_df(user=user, **kwargs)
|
|
241
99
|
|
|
242
|
-
|
|
243
|
-
run_time = human_timedelta(row.end_time - row.start_time)
|
|
244
|
-
else:
|
|
245
|
-
run_time = human_timedelta(datetime.datetime.now() - row.start_time)
|
|
100
|
+
history_df = df[~df["job_state"].isin(["RUNNING", "PENDING"])]
|
|
246
101
|
|
|
247
|
-
|
|
102
|
+
history = Panel(
|
|
103
|
+
history_job_table(history_df, limit=None, user=user, **kwargs),
|
|
104
|
+
expand=False,
|
|
105
|
+
)
|
|
248
106
|
|
|
249
|
-
|
|
250
|
-
values.append(row.node_count)
|
|
251
|
-
if num_cores:
|
|
252
|
-
values.append(row.cpus)
|
|
107
|
+
return history
|
|
253
108
|
|
|
254
|
-
if force_submit_time or long and submit_time:
|
|
255
|
-
values.append(submit_time)
|
|
256
109
|
|
|
257
|
-
|
|
258
|
-
values.append(start_time)
|
|
110
|
+
def get_node_layout(idle: bool = True, **kwargs):
|
|
259
111
|
|
|
260
|
-
|
|
261
|
-
values.append(run_time)
|
|
112
|
+
df = get_sinfo(**kwargs)
|
|
262
113
|
|
|
263
|
-
|
|
264
|
-
|
|
114
|
+
if idle:
|
|
115
|
+
df = df[df["node_state"].isin(["IDLE", "MIXED"])]
|
|
265
116
|
|
|
266
|
-
|
|
117
|
+
table = node_table(df, **kwargs)
|
|
267
118
|
|
|
268
|
-
|
|
119
|
+
return Panel(table, expand=False)
|
|
269
120
|
|
|
270
|
-
if panel:
|
|
271
|
-
table = Panel(table, expand=False)
|
|
272
121
|
|
|
273
|
-
|
|
274
|
-
return table
|
|
122
|
+
### GET QUEUE DFs
|
|
275
123
|
|
|
276
|
-
console.print(table)
|
|
277
124
|
|
|
125
|
+
def get_squeue(user: str | None = None, **kwargs) -> "pandas.DataFrame":
|
|
278
126
|
|
|
279
|
-
|
|
127
|
+
if user:
|
|
128
|
+
command = f"squeue -u {user} --json"
|
|
129
|
+
else:
|
|
130
|
+
command = f"squeue --json"
|
|
280
131
|
|
|
281
|
-
|
|
132
|
+
try:
|
|
133
|
+
process = subprocess.Popen(
|
|
134
|
+
[command], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
135
|
+
)
|
|
136
|
+
output = process.communicate()
|
|
137
|
+
payload = json.loads(output[0])
|
|
138
|
+
except json.JSONDecodeError:
|
|
139
|
+
# console.print('[orange1 bold]Warning: using example data')
|
|
140
|
+
payload = json.load(
|
|
141
|
+
open(
|
|
142
|
+
Path(__file__).parent.parent / "example_data" / "squeue_long.json", "rt"
|
|
143
|
+
)
|
|
144
|
+
)
|
|
282
145
|
|
|
283
146
|
global METADATA
|
|
284
147
|
|
|
@@ -292,14 +155,11 @@ def parse_squeue_json(payload: dict) -> "DataFrame":
|
|
|
292
155
|
df = DataFrame(payload["jobs"])
|
|
293
156
|
|
|
294
157
|
# filter columns
|
|
295
|
-
|
|
296
|
-
columns = COLUMNS[command]
|
|
158
|
+
columns = COLUMNS["squeue"]
|
|
297
159
|
|
|
298
160
|
try:
|
|
299
161
|
df = df[columns]
|
|
300
|
-
|
|
301
162
|
except KeyError:
|
|
302
|
-
|
|
303
163
|
for key in columns:
|
|
304
164
|
if key not in df.columns:
|
|
305
165
|
raise KeyError(key)
|
|
@@ -318,32 +178,48 @@ def parse_squeue_json(payload: dict) -> "DataFrame":
|
|
|
318
178
|
return df
|
|
319
179
|
|
|
320
180
|
|
|
321
|
-
def
|
|
181
|
+
def get_sacct(
|
|
182
|
+
user: str | None = None, hist: int | None = 4, hist_unit: str = "weeks", **kwargs
|
|
183
|
+
) -> "pandas.DataFrame":
|
|
184
|
+
|
|
185
|
+
hist = hist or 4
|
|
186
|
+
|
|
187
|
+
if user:
|
|
188
|
+
command = f"sacct -u {user} --json -S now-{hist}{hist_unit}"
|
|
189
|
+
else:
|
|
190
|
+
command = f"sacct --json -S now-{hist}{hist_unit}"
|
|
322
191
|
|
|
323
|
-
|
|
192
|
+
try:
|
|
193
|
+
process = subprocess.Popen(
|
|
194
|
+
[command], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
195
|
+
)
|
|
196
|
+
output = process.communicate()
|
|
197
|
+
payload = json.loads(output[0])
|
|
198
|
+
except json.JSONDecodeError:
|
|
199
|
+
# console.print('[orange1 bold]Warning: using example data')
|
|
200
|
+
payload = json.load(
|
|
201
|
+
open(Path(__file__).parent.parent / "example_data" / "sacct.json", "rt")
|
|
202
|
+
)
|
|
324
203
|
|
|
325
204
|
global METADATA
|
|
326
205
|
|
|
327
|
-
METADATA
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
206
|
+
METADATA = {
|
|
207
|
+
"cluster_name": payload["meta"]["slurm"]["cluster"],
|
|
208
|
+
"user": payload["meta"]["client"]["user"],
|
|
209
|
+
"group": payload["meta"]["client"]["group"],
|
|
210
|
+
"hist": hist,
|
|
211
|
+
"hist_unit": hist_unit,
|
|
212
|
+
}
|
|
334
213
|
|
|
335
214
|
# parse payload
|
|
336
215
|
df = DataFrame(payload["jobs"])
|
|
337
216
|
|
|
338
217
|
# filter columns
|
|
339
|
-
|
|
340
|
-
columns = COLUMNS[command]
|
|
218
|
+
columns = COLUMNS["sacct"]
|
|
341
219
|
|
|
342
220
|
try:
|
|
343
221
|
df = df[columns]
|
|
344
|
-
|
|
345
222
|
except KeyError:
|
|
346
|
-
|
|
347
223
|
for key in columns:
|
|
348
224
|
if key not in df.columns:
|
|
349
225
|
raise KeyError(key)
|
|
@@ -357,167 +233,148 @@ def parse_sacct_json(payload: dict) -> "DataFrame":
|
|
|
357
233
|
extract_list(df, "job_state")
|
|
358
234
|
|
|
359
235
|
df = df[df["job_state"] != "RUNNING"]
|
|
236
|
+
df = df[df["job_state"] != "PENDING"]
|
|
360
237
|
|
|
361
238
|
return df
|
|
362
239
|
|
|
363
240
|
|
|
364
|
-
def
|
|
365
|
-
command: str,
|
|
366
|
-
user: None | str = None,
|
|
367
|
-
long: bool = False,
|
|
368
|
-
return_table: bool = False,
|
|
369
|
-
hist: int = 2,
|
|
370
|
-
hist_unit: str= "weeks",
|
|
371
|
-
box: bool = False,
|
|
372
|
-
):
|
|
241
|
+
def get_sinfo(**kwargs) -> "pandas.DataFrame":
|
|
373
242
|
|
|
374
|
-
|
|
375
|
-
user = None
|
|
376
|
-
elif user is None:
|
|
377
|
-
x = subprocess.Popen(["whoami"], shell=True, stdout=subprocess.PIPE)
|
|
378
|
-
output = x.communicate()
|
|
379
|
-
user = output[0].strip().decode("utf-8")
|
|
380
|
-
|
|
381
|
-
if user:
|
|
382
|
-
s_command = f"{command} -u {user} --json"
|
|
383
|
-
else:
|
|
384
|
-
s_command = f"{command} -u {user} --json"
|
|
385
|
-
|
|
386
|
-
if command == "sacct":
|
|
387
|
-
s_command += f" -S now-{hist}{hist_unit}"
|
|
388
|
-
|
|
389
|
-
x = subprocess.Popen([s_command], shell=True, stdout=subprocess.PIPE)
|
|
390
|
-
output = x.communicate()
|
|
391
|
-
|
|
392
|
-
payload = json.loads(output[0])
|
|
393
|
-
|
|
394
|
-
parser = PARSERS[command]
|
|
243
|
+
command = "sinfo -N --json"
|
|
395
244
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
245
|
+
try:
|
|
246
|
+
process = subprocess.Popen(
|
|
247
|
+
[command], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
248
|
+
)
|
|
249
|
+
output = process.communicate()
|
|
250
|
+
payload = json.loads(output[0])
|
|
251
|
+
except json.JSONDecodeError:
|
|
252
|
+
payload = json.load(
|
|
253
|
+
open(Path(__file__).parent.parent / "example_data" / "sinfo_N.json", "rt")
|
|
254
|
+
)
|
|
402
255
|
|
|
403
|
-
|
|
404
|
-
df = df[df["user_name"] == user]
|
|
405
|
-
title = f"[bold]{user}'s jobs on {cluster}"
|
|
406
|
-
else:
|
|
407
|
-
title = f"[bold]jobs on {cluster}"
|
|
256
|
+
global METADATA
|
|
408
257
|
|
|
409
|
-
|
|
258
|
+
METADATA = {
|
|
259
|
+
"cluster_name": payload["meta"]["slurm"]["cluster"],
|
|
260
|
+
"user": payload["meta"]["client"]["user"],
|
|
261
|
+
"group": payload["meta"]["client"]["group"],
|
|
262
|
+
}
|
|
410
263
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
264
|
+
df = DataFrame(payload["sinfo"])
|
|
265
|
+
|
|
266
|
+
df.drop(
|
|
267
|
+
columns=[
|
|
268
|
+
"port",
|
|
269
|
+
"weight",
|
|
270
|
+
"disk",
|
|
271
|
+
"sockets",
|
|
272
|
+
"threads",
|
|
273
|
+
"cluster",
|
|
274
|
+
"comment",
|
|
275
|
+
"extra",
|
|
276
|
+
"gres",
|
|
277
|
+
"reason",
|
|
278
|
+
"cores",
|
|
279
|
+
],
|
|
280
|
+
inplace=True,
|
|
418
281
|
)
|
|
419
282
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
panel1 = show_queue(user=user, command="squeue", long=long, return_table=True)
|
|
431
|
-
panel2 = show_queue(user=user, command="sacct", long=long, return_table=True)
|
|
432
|
-
|
|
433
|
-
upper = Layout(renderable=panel1, name="upper")
|
|
434
|
-
lower = Layout(renderable=panel2, name="lower")
|
|
435
|
-
|
|
436
|
-
upper.size = panel1.renderable.row_count + 4
|
|
437
|
-
lower.size = panel2.renderable.row_count + 4
|
|
438
|
-
|
|
439
|
-
layout.split_column(
|
|
440
|
-
upper,
|
|
441
|
-
lower,
|
|
283
|
+
df["node"] = df.apply(lambda x: x["node"]["state"][0], axis=1)
|
|
284
|
+
df["nodes"] = df.apply(lambda x: x["nodes"]["nodes"][0], axis=1)
|
|
285
|
+
df["cpus_max"] = df.apply(lambda x: x["cpus"]["maximum"], axis=1)
|
|
286
|
+
df["cpus_idle"] = df.apply(lambda x: x["cpus"]["idle"], axis=1)
|
|
287
|
+
df["cpus_allocated"] = df.apply(lambda x: x["cpus"]["allocated"], axis=1)
|
|
288
|
+
df["cpu_string"] = df.apply(lambda x: f"{x.cpus_idle}/{x.cpus_max}", axis=1)
|
|
289
|
+
df["memory_max"] = df.apply(lambda x: x["memory"]["maximum"], axis=1)
|
|
290
|
+
df["memory_free"] = df.apply(
|
|
291
|
+
lambda x: x["memory"]["free"]["maximum"]["number"], axis=1
|
|
442
292
|
)
|
|
293
|
+
df["memory_allocated"] = df.apply(lambda x: x["memory"]["allocated"], axis=1)
|
|
294
|
+
df["features"] = df.apply(lambda x: x["features"]["total"], axis=1)
|
|
295
|
+
df["partition"] = df.apply(lambda x: x["partition"]["name"], axis=1)
|
|
443
296
|
|
|
444
|
-
|
|
445
|
-
console.print(layout)
|
|
446
|
-
else:
|
|
447
|
-
return layout
|
|
297
|
+
df.drop(columns=["cpus", "memory"], inplace=True)
|
|
448
298
|
|
|
449
|
-
|
|
299
|
+
df.rename(columns={"node": "node_state", "nodes": "node_name"}, inplace=True)
|
|
450
300
|
|
|
451
|
-
df
|
|
301
|
+
return df
|
|
452
302
|
|
|
453
|
-
df2 = df.drop(columns=["port", "weight", "disk", "sockets", "threads", "cluster", "comment", "extra", "gres", "reason", "cores"])
|
|
454
|
-
df2["node"] = df2.apply(lambda x: x["node"]["state"][0], axis=1)
|
|
455
|
-
df2["nodes"] = df2.apply(lambda x: x["nodes"]["nodes"][0], axis=1)
|
|
456
|
-
df2["cpus_max"] = df2.apply(lambda x: x["cpus"]["maximum"], axis=1)
|
|
457
|
-
df2["cpus_idle"] = df2.apply(lambda x: x["cpus"]["idle"], axis=1)
|
|
458
|
-
df2["cpus_allocated"] = df2.apply(lambda x: x["cpus"]["allocated"], axis=1)
|
|
459
|
-
df2["memory_max"] = df2.apply(lambda x: x["memory"]["maximum"], axis=1)
|
|
460
|
-
df2["memory_free"] = df2.apply(lambda x: x["memory"]["free"]["maximum"]["number"], axis=1)
|
|
461
|
-
df2["memory_allocated"] = df2.apply(lambda x: x["memory"]["allocated"], axis=1)
|
|
462
|
-
df2["features"] = df2.apply(lambda x: x["features"]["total"], axis=1)
|
|
463
|
-
df2["partition"] = df2.apply(lambda x: x["partition"]["name"], axis=1)
|
|
464
303
|
|
|
465
|
-
|
|
304
|
+
def combined_df(**kwargs) -> "DataFrame":
|
|
305
|
+
"""Get combined DataFrame of SLURM job information"""
|
|
306
|
+
df1 = get_squeue(**kwargs)
|
|
307
|
+
df2 = get_sacct(**kwargs)
|
|
308
|
+
df = concat([df1, df2], ignore_index=True)
|
|
309
|
+
df["run_time"] = add_run_time(df)
|
|
310
|
+
df = df.sort_values(by="submit_time", ascending=True)
|
|
311
|
+
return df
|
|
466
312
|
|
|
467
|
-
return df2
|
|
468
313
|
|
|
469
|
-
|
|
314
|
+
### ADD COLUMNS
|
|
470
315
|
|
|
471
|
-
s_command = "sinfo -N --json"
|
|
472
316
|
|
|
473
|
-
|
|
474
|
-
output = x.communicate()
|
|
317
|
+
def add_run_time(df):
|
|
475
318
|
|
|
476
|
-
|
|
319
|
+
def inner(row):
|
|
320
|
+
# print(f'{row.end_time=} {type(row.end_time)=}')
|
|
477
321
|
|
|
478
|
-
|
|
322
|
+
if row.job_state == "PENDING":
|
|
323
|
+
return ""
|
|
479
324
|
|
|
480
|
-
|
|
325
|
+
if isnull(row.end_time):
|
|
326
|
+
return human_timedelta(datetime.datetime.now() - row.start_time)
|
|
327
|
+
else:
|
|
328
|
+
return human_timedelta(row.end_time - row.start_time)
|
|
481
329
|
|
|
482
|
-
|
|
330
|
+
return df.apply(inner, axis=1)
|
|
483
331
|
|
|
484
|
-
@app.command()
|
|
485
|
-
def show(user: None | str = None, long: bool = False, idle: bool = False, hist: int | None = None, hist_unit: str = "weeks"):
|
|
486
332
|
|
|
487
|
-
|
|
333
|
+
### EXTRACTORS
|
|
488
334
|
|
|
489
|
-
if hist:
|
|
490
335
|
|
|
491
|
-
|
|
336
|
+
def extract_inner(df, key, inner):
|
|
492
337
|
|
|
493
|
-
|
|
494
|
-
|
|
338
|
+
def _inner(x):
|
|
339
|
+
d = x[key]
|
|
340
|
+
if "set" in d:
|
|
341
|
+
if d["set"]:
|
|
342
|
+
return d[inner]
|
|
343
|
+
else:
|
|
344
|
+
return None
|
|
345
|
+
else:
|
|
346
|
+
return d[inner]
|
|
495
347
|
|
|
496
|
-
|
|
348
|
+
df[key] = df.apply(_inner, axis=1)
|
|
497
349
|
|
|
498
|
-
from rich.live import Live
|
|
499
|
-
import time
|
|
500
350
|
|
|
501
|
-
|
|
351
|
+
def extract_time(df, key):
|
|
352
|
+
df[key] = df.apply(
|
|
353
|
+
lambda x: datetime.datetime.fromtimestamp(x[key]["number"]), axis=1
|
|
354
|
+
)
|
|
502
355
|
|
|
503
|
-
with Live(layout, refresh_per_second=4, screen=True) as live:
|
|
504
|
-
while True:
|
|
505
356
|
|
|
506
|
-
|
|
507
|
-
|
|
357
|
+
def extract_sacct_times(df):
|
|
358
|
+
df["start_time"] = df.apply(
|
|
359
|
+
lambda x: datetime.datetime.fromtimestamp(x["time"]["start"]), axis=1
|
|
360
|
+
)
|
|
361
|
+
df["end_time"] = df.apply(
|
|
362
|
+
lambda x: datetime.datetime.fromtimestamp(x["time"]["end"]), axis=1
|
|
363
|
+
)
|
|
364
|
+
df["submit_time"] = df.apply(
|
|
365
|
+
lambda x: datetime.datetime.fromtimestamp(x["time"]["submission"]), axis=1
|
|
366
|
+
)
|
|
508
367
|
|
|
509
|
-
time.sleep(1)
|
|
510
368
|
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
369
|
+
def extract_list(df, key):
|
|
370
|
+
def inner(x):
|
|
371
|
+
if len(x[key]) == 1:
|
|
372
|
+
return x[key][0]
|
|
373
|
+
else:
|
|
374
|
+
return x[key]
|
|
516
375
|
|
|
517
|
-
|
|
518
|
-
app()
|
|
376
|
+
df[key] = df.apply(inner, axis=1)
|
|
519
377
|
|
|
520
|
-
PARSERS = {"squeue": parse_squeue_json, "sacct": parse_sacct_json}
|
|
521
378
|
|
|
522
379
|
COLUMNS = {
|
|
523
380
|
"sacct": [
|
|
@@ -559,7 +416,3 @@ COLUMNS = {
|
|
|
559
416
|
"current_working_directory",
|
|
560
417
|
],
|
|
561
418
|
}
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
if __name__ == "__main__":
|
|
565
|
-
app()
|
richqueue/table.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
from rich.table import Table
|
|
2
|
+
from .tools import human_datetime
|
|
3
|
+
from pandas import isnull
|
|
4
|
+
from .console import console
|
|
5
|
+
|
|
6
|
+
### TABLES
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def running_job_table(
|
|
10
|
+
df: "DataFrame",
|
|
11
|
+
long: bool = False,
|
|
12
|
+
limit: int | None = None,
|
|
13
|
+
hide_pending: bool | int = False,
|
|
14
|
+
user: str | None = None,
|
|
15
|
+
**kwargs,
|
|
16
|
+
):
|
|
17
|
+
|
|
18
|
+
if hide_pending:
|
|
19
|
+
pending_df = df[df["job_state"] == "PENDING"]
|
|
20
|
+
df = df[df["job_state"] == "RUNNING"]
|
|
21
|
+
|
|
22
|
+
if limit:
|
|
23
|
+
df = df[-limit:]
|
|
24
|
+
|
|
25
|
+
from .slurm import METADATA
|
|
26
|
+
|
|
27
|
+
if not user:
|
|
28
|
+
title = f"[bold]all running jobs on {METADATA['cluster_name']}"
|
|
29
|
+
else:
|
|
30
|
+
title = f"[bold]{user}'s running jobs on {METADATA['cluster_name']}"
|
|
31
|
+
|
|
32
|
+
table = Table(title=title, box=None, header_style="")
|
|
33
|
+
|
|
34
|
+
columns = RUNNING_JOB_COLUMNS[long]
|
|
35
|
+
|
|
36
|
+
for col in columns:
|
|
37
|
+
col_data = COLUMNS[col]
|
|
38
|
+
table.add_column(**col_data)
|
|
39
|
+
|
|
40
|
+
for i, row in df.iterrows():
|
|
41
|
+
row_values = []
|
|
42
|
+
for col in columns:
|
|
43
|
+
value = row[col]
|
|
44
|
+
formatter = FORMATTERS.get(col, str)
|
|
45
|
+
value = formatter(value)
|
|
46
|
+
row_values.append(value)
|
|
47
|
+
table.add_row(*row_values)
|
|
48
|
+
|
|
49
|
+
if hide_pending:
|
|
50
|
+
row = {
|
|
51
|
+
"job_id": f"(x{hide_pending})",
|
|
52
|
+
"name": "",
|
|
53
|
+
"node_count": f"{int(sum(pending_df['node_count'].values))}",
|
|
54
|
+
"cpus": f"{int(sum(pending_df['cpus'].values))}",
|
|
55
|
+
"submit_time": "",
|
|
56
|
+
"start_time": "",
|
|
57
|
+
"run_time": "",
|
|
58
|
+
"partition": "",
|
|
59
|
+
"nodes": "",
|
|
60
|
+
"job_state": "[bright_yellow bold]Pending",
|
|
61
|
+
}
|
|
62
|
+
row_values = []
|
|
63
|
+
for col in columns:
|
|
64
|
+
value = row[col]
|
|
65
|
+
row_values.append(value)
|
|
66
|
+
table.add_row(*row_values)
|
|
67
|
+
|
|
68
|
+
return table
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def history_job_table(
|
|
72
|
+
df: "DataFrame",
|
|
73
|
+
long: bool = False,
|
|
74
|
+
limit: int | None = None,
|
|
75
|
+
user: str | None = None,
|
|
76
|
+
**kwargs,
|
|
77
|
+
):
|
|
78
|
+
|
|
79
|
+
if limit:
|
|
80
|
+
df = df[-limit:]
|
|
81
|
+
|
|
82
|
+
from .slurm import METADATA
|
|
83
|
+
|
|
84
|
+
hist = METADATA["hist"]
|
|
85
|
+
hist_unit = METADATA["hist_unit"]
|
|
86
|
+
if hist == 1:
|
|
87
|
+
hist_unit = hist_unit.removesuffix("s")
|
|
88
|
+
hist_string = f"last {hist_unit}"
|
|
89
|
+
else:
|
|
90
|
+
hist_string = f"last {hist} {hist_unit}"
|
|
91
|
+
|
|
92
|
+
if not user:
|
|
93
|
+
title = f"[bold]all previous jobs on {METADATA['cluster_name']} ({hist_string})"
|
|
94
|
+
else:
|
|
95
|
+
title = f"[bold]{user}'s previous jobs on {METADATA['cluster_name']} ({hist_string})"
|
|
96
|
+
|
|
97
|
+
table = Table(title=title, box=None, header_style="")
|
|
98
|
+
|
|
99
|
+
columns = HISTORY_JOB_COLUMNS[long]
|
|
100
|
+
|
|
101
|
+
for col in columns:
|
|
102
|
+
col_data = COLUMNS[col]
|
|
103
|
+
table.add_column(**col_data)
|
|
104
|
+
|
|
105
|
+
for i, row in df.iterrows():
|
|
106
|
+
row_values = []
|
|
107
|
+
for col in columns:
|
|
108
|
+
value = row[col]
|
|
109
|
+
formatter = FORMATTERS.get(col, str)
|
|
110
|
+
value = formatter(value)
|
|
111
|
+
row_values.append(value)
|
|
112
|
+
table.add_row(*row_values)
|
|
113
|
+
|
|
114
|
+
return table
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def node_table(df, long: bool = False, **kwargs):
|
|
118
|
+
|
|
119
|
+
from .slurm import METADATA
|
|
120
|
+
|
|
121
|
+
title = f"[bold]available nodes on {METADATA['cluster_name']}"
|
|
122
|
+
|
|
123
|
+
table = Table(title=title, box=None, header_style="")
|
|
124
|
+
|
|
125
|
+
columns = NODE_COLUMNS[long]
|
|
126
|
+
|
|
127
|
+
df.sort_values(by=["partition", "node_name"], inplace=True)
|
|
128
|
+
|
|
129
|
+
if not df["reservation"].any():
|
|
130
|
+
columns = [c for c in columns if c != "reservation"]
|
|
131
|
+
|
|
132
|
+
for col in columns:
|
|
133
|
+
col_data = COLUMNS[col]
|
|
134
|
+
table.add_column(**col_data)
|
|
135
|
+
|
|
136
|
+
for i, row in df.iterrows():
|
|
137
|
+
row_values = []
|
|
138
|
+
for col in columns:
|
|
139
|
+
value = row[col]
|
|
140
|
+
formatter = FORMATTERS.get(col, str)
|
|
141
|
+
value = formatter(value)
|
|
142
|
+
row_values.append(value)
|
|
143
|
+
table.add_row(*row_values)
|
|
144
|
+
|
|
145
|
+
return table
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
### FORMATTERS
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def color_by_state(state):
|
|
152
|
+
|
|
153
|
+
if state == "RUNNING":
|
|
154
|
+
return "[bold bright_green]Running"
|
|
155
|
+
elif state == "IDLE":
|
|
156
|
+
return "[bold bright_green]Idle"
|
|
157
|
+
elif state == "PENDING":
|
|
158
|
+
return "[bright_yellow]Pending"
|
|
159
|
+
elif state == "MIXED":
|
|
160
|
+
return "[bright_yellow]Mixed"
|
|
161
|
+
elif state == "CANCELLED":
|
|
162
|
+
return "[orange3]Cancelled"
|
|
163
|
+
elif state == "FAILED":
|
|
164
|
+
return "[bold bright_red]Failed"
|
|
165
|
+
elif state == "COMPLETED":
|
|
166
|
+
return "[bold bright_green]Completed"
|
|
167
|
+
else:
|
|
168
|
+
return state
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def int_if_not_nan(value):
|
|
172
|
+
if isnull(value):
|
|
173
|
+
return ""
|
|
174
|
+
else:
|
|
175
|
+
return str(int(value))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def mem_string(mb):
|
|
179
|
+
return f"{mb/1024:.0f} GB"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
### SPECIFICATION
|
|
183
|
+
|
|
184
|
+
RUNNING_JOB_COLUMNS = {
|
|
185
|
+
True: [
|
|
186
|
+
"job_id",
|
|
187
|
+
"name",
|
|
188
|
+
"node_count",
|
|
189
|
+
"cpus",
|
|
190
|
+
"submit_time",
|
|
191
|
+
"start_time",
|
|
192
|
+
"run_time",
|
|
193
|
+
"partition",
|
|
194
|
+
"nodes",
|
|
195
|
+
"job_state",
|
|
196
|
+
],
|
|
197
|
+
False: [
|
|
198
|
+
"job_id",
|
|
199
|
+
"name",
|
|
200
|
+
"node_count",
|
|
201
|
+
"cpus",
|
|
202
|
+
"start_time",
|
|
203
|
+
"run_time",
|
|
204
|
+
"job_state",
|
|
205
|
+
],
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
HISTORY_JOB_COLUMNS = {
|
|
209
|
+
True: [
|
|
210
|
+
"job_id",
|
|
211
|
+
"name",
|
|
212
|
+
# "node_count",
|
|
213
|
+
# "cpus",
|
|
214
|
+
"submit_time",
|
|
215
|
+
"start_time",
|
|
216
|
+
"run_time",
|
|
217
|
+
"partition",
|
|
218
|
+
"nodes",
|
|
219
|
+
"job_state",
|
|
220
|
+
],
|
|
221
|
+
False: [
|
|
222
|
+
"job_id",
|
|
223
|
+
"name",
|
|
224
|
+
# "node_count",
|
|
225
|
+
# "cpus",
|
|
226
|
+
"start_time",
|
|
227
|
+
"run_time",
|
|
228
|
+
"job_state",
|
|
229
|
+
],
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
NODE_COLUMNS = {
|
|
233
|
+
True: [
|
|
234
|
+
"node_state",
|
|
235
|
+
"node_name",
|
|
236
|
+
"partition",
|
|
237
|
+
# "cpu_string",
|
|
238
|
+
# "cpus_max",
|
|
239
|
+
"cpus_idle",
|
|
240
|
+
# "memory_max",
|
|
241
|
+
"memory_free",
|
|
242
|
+
# "memory_allocated",
|
|
243
|
+
"features",
|
|
244
|
+
"reservation",
|
|
245
|
+
],
|
|
246
|
+
False: [
|
|
247
|
+
"node_state",
|
|
248
|
+
"node_name",
|
|
249
|
+
"partition",
|
|
250
|
+
# "cpu_string",
|
|
251
|
+
# "cpus_max",
|
|
252
|
+
"cpus_idle",
|
|
253
|
+
# "memory_max",
|
|
254
|
+
"memory_free",
|
|
255
|
+
# "memory_allocated",
|
|
256
|
+
"features",
|
|
257
|
+
"reservation",
|
|
258
|
+
],
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
COLUMNS = {
|
|
262
|
+
"job_id": {
|
|
263
|
+
"header": "[bold underline]Job Id",
|
|
264
|
+
"justify": "right",
|
|
265
|
+
"style": "bold",
|
|
266
|
+
"no_wrap": True,
|
|
267
|
+
},
|
|
268
|
+
"name": {
|
|
269
|
+
"header": "[underline cyan]Job Name",
|
|
270
|
+
"justify": "left",
|
|
271
|
+
"style": "cyan",
|
|
272
|
+
"no_wrap": False,
|
|
273
|
+
},
|
|
274
|
+
"node_count": {
|
|
275
|
+
"header": "[underline magenta]#N",
|
|
276
|
+
"justify": "right",
|
|
277
|
+
"style": "magenta",
|
|
278
|
+
"no_wrap": True,
|
|
279
|
+
},
|
|
280
|
+
"cpus": {
|
|
281
|
+
"header": "[underline magenta]#C",
|
|
282
|
+
"justify": "right",
|
|
283
|
+
"style": "magenta",
|
|
284
|
+
"no_wrap": True,
|
|
285
|
+
},
|
|
286
|
+
"job_state": {
|
|
287
|
+
"header": "[bold underline]State",
|
|
288
|
+
"justify": "left",
|
|
289
|
+
"style": None,
|
|
290
|
+
"no_wrap": True,
|
|
291
|
+
},
|
|
292
|
+
"submit_time": {
|
|
293
|
+
"header": "[underline dodger_blue2]Submitted",
|
|
294
|
+
"justify": "right",
|
|
295
|
+
"style": "dodger_blue2",
|
|
296
|
+
"no_wrap": True,
|
|
297
|
+
},
|
|
298
|
+
"start_time": {
|
|
299
|
+
"header": "[underline dodger_blue2]Started",
|
|
300
|
+
"justify": "right",
|
|
301
|
+
"style": "dodger_blue2",
|
|
302
|
+
"no_wrap": True,
|
|
303
|
+
},
|
|
304
|
+
"run_time": {
|
|
305
|
+
"header": "[underline dodger_blue2]Run Time",
|
|
306
|
+
"justify": "right",
|
|
307
|
+
"style": "dodger_blue2",
|
|
308
|
+
"no_wrap": True,
|
|
309
|
+
},
|
|
310
|
+
"partition": {
|
|
311
|
+
"header": "[underline green_yellow]Partition",
|
|
312
|
+
"justify": "right",
|
|
313
|
+
"style": "green_yellow",
|
|
314
|
+
"no_wrap": True,
|
|
315
|
+
},
|
|
316
|
+
"features": {
|
|
317
|
+
"header": "[underline cyan]Features",
|
|
318
|
+
"justify": "left",
|
|
319
|
+
"style": "cyan",
|
|
320
|
+
"no_wrap": False,
|
|
321
|
+
},
|
|
322
|
+
"reservation": {
|
|
323
|
+
"header": "[underline cyan]Reservation",
|
|
324
|
+
"justify": "left",
|
|
325
|
+
"style": "cyan",
|
|
326
|
+
"no_wrap": True,
|
|
327
|
+
},
|
|
328
|
+
"nodes": {
|
|
329
|
+
"header": "[underline green_yellow]Nodes",
|
|
330
|
+
"justify": "left",
|
|
331
|
+
"style": "green_yellow",
|
|
332
|
+
"no_wrap": False,
|
|
333
|
+
},
|
|
334
|
+
"node_state": {
|
|
335
|
+
"header": "[bold underline]State",
|
|
336
|
+
"justify": "left",
|
|
337
|
+
"style": None,
|
|
338
|
+
"no_wrap": True,
|
|
339
|
+
},
|
|
340
|
+
"node_name": {
|
|
341
|
+
"header": "[underline cyan]Node Name",
|
|
342
|
+
"justify": "left",
|
|
343
|
+
"style": "cyan",
|
|
344
|
+
"no_wrap": False,
|
|
345
|
+
},
|
|
346
|
+
"cpus_idle": {
|
|
347
|
+
"header": "[underline dodger_blue2]Idle #C",
|
|
348
|
+
"justify": "right",
|
|
349
|
+
"style": "dodger_blue2",
|
|
350
|
+
"no_wrap": True,
|
|
351
|
+
},
|
|
352
|
+
"memory_free": {
|
|
353
|
+
"header": "[underline magenta]RAM Free",
|
|
354
|
+
"justify": "right",
|
|
355
|
+
"style": "magenta",
|
|
356
|
+
"no_wrap": True,
|
|
357
|
+
},
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
FORMATTERS = {
|
|
361
|
+
"node_count": int_if_not_nan,
|
|
362
|
+
"cpus": int_if_not_nan,
|
|
363
|
+
"job_state": color_by_state,
|
|
364
|
+
"node_state": color_by_state,
|
|
365
|
+
"submit_time": human_datetime,
|
|
366
|
+
"start_time": human_datetime,
|
|
367
|
+
"memory_free": mem_string,
|
|
368
|
+
}
|
richqueue/tools.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: richqueue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7
|
|
4
4
|
Summary: RichQueue
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwinokan/RichQueue/issues
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwinokan/RichQueue/issues
|
|
@@ -43,3 +43,15 @@ To see more detail:
|
|
|
43
43
|
To see someone else's queue:
|
|
44
44
|
|
|
45
45
|
```rq --user USER```
|
|
46
|
+
|
|
47
|
+
To see the last `X` weeks history:
|
|
48
|
+
|
|
49
|
+
```rq --hist X```
|
|
50
|
+
|
|
51
|
+
To list available nodes on the cluster:
|
|
52
|
+
|
|
53
|
+
```rq --idle```
|
|
54
|
+
|
|
55
|
+
To show a static view:
|
|
56
|
+
|
|
57
|
+
```rq --no-loop```
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
richqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
richqueue/console.py,sha256=UpMqeJ0C8i0pkue1AHnnyyX0bFJ9zZeJ7HBR6yhuA8A,54
|
|
3
|
+
richqueue/layout.py,sha256=qD3RRNvKYQSCDZpWLRvP6eGDxIu96b5hBn1Vx-V50WM,602
|
|
4
|
+
richqueue/main.py,sha256=jiXg2CwH6xmJhLNbK47hUBSFW2lPC3deXAEs7KjlEWE,1917
|
|
5
|
+
richqueue/slurm.py,sha256=q_gmqY1zfL24u3ElsPeXH5aKN1P7kC2GU8UWcusbcJ4,10551
|
|
6
|
+
richqueue/table.py,sha256=pJ29lSUchiFHxfpLwz7b6FpSMcOsahUTSNZzBZ1Fees,8567
|
|
7
|
+
richqueue/tools.py,sha256=64sd7IOd8uYSVAqMP-UOBZLiyh22LGMP-RAr-OrIA8o,965
|
|
8
|
+
richqueue-0.7.dist-info/METADATA,sha256=O-oLQn2PtedlfK7a4zidx1GuzDJsMzd_Cqlgqdky-5Q,1171
|
|
9
|
+
richqueue-0.7.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
10
|
+
richqueue-0.7.dist-info/entry_points.txt,sha256=cm1tyoTGLAm9f_QYm0ieGnY82HE4IFiKD1KjlXRgFkU,44
|
|
11
|
+
richqueue-0.7.dist-info/licenses/LICENSE,sha256=IeuPfRcqKv1nyCnVVwJaO9Uratg-K2dnkuOROLgQ4qw,1068
|
|
12
|
+
richqueue-0.7.dist-info/RECORD,,
|
richqueue-0.6.dist-info/RECORD
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
richqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
richqueue/slurm.py,sha256=jek_F849QjuPrfVH2pd6k7ypexiLrqf1K_xl06fLnE8,13943
|
|
3
|
-
richqueue/tools.py,sha256=n2MOQWgBykaY3wnhJYqvHsJDKavnyJWx8Pq1SkSTiPU,841
|
|
4
|
-
richqueue-0.6.dist-info/METADATA,sha256=G-ptfbCSU7MWtsN3bRZH9efRLJJrbGCU5Npd7hPVLVA,1014
|
|
5
|
-
richqueue-0.6.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
6
|
-
richqueue-0.6.dist-info/entry_points.txt,sha256=cm1tyoTGLAm9f_QYm0ieGnY82HE4IFiKD1KjlXRgFkU,44
|
|
7
|
-
richqueue-0.6.dist-info/licenses/LICENSE,sha256=IeuPfRcqKv1nyCnVVwJaO9Uratg-K2dnkuOROLgQ4qw,1068
|
|
8
|
-
richqueue-0.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|