nodeocc 1.0.4__tar.gz → 1.0.6__tar.gz

{nodeocc-1.0.4 → nodeocc-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.4
+Version: 1.0.6
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.4 → nodeocc-1.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.4"
+version = "1.0.6"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/controller/controller.py

@@ -120,9 +120,9 @@ def update_data_master(instance):
     running_jobs = [j.to_nested_dict() for j in jobs if j.state == 'CG' or j.state == 'R']
     queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
     queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
-    maxlen = min(120, len(running_jobs) + len(queue_jobs))
+    maxlen = min(100, len(running_jobs) + len(queue_jobs))
 
-    msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs[:maxlen]})
+    msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs[:maxlen - len(running_jobs)]})
 
     # msg to bytes
     msg = msg.encode('utf-8')

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/model/infrastructure.py

@@ -92,4 +92,4 @@ class Infrastructure:
 
     def get_sorted_nodes(self):
 
-        return sorted(self.nodes, key=lambda x: self.prior.index(x.gpu_model) if x.gpu_model in self.prior else -1)
+        return sorted(self.nodes, key=lambda x: self.prior.index(x.gpu_model.split(',')[0]) if x.gpu_model.split(',')[0] in self.prior else -1)

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.4
+Version: 1.0.6
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/readers/slurmreader.py

@@ -41,11 +41,24 @@ def _node_from_sinfo(x):
     return n
 
 
+def _parse_gpu_models(line):
+    line = line[0].split(',')
+    if line[0] == '(null)':
+        return 'n/a'
+    else:
+        if len(line) == 1:
+            return line[0].split(':')[1]
+        else:
+            # return line[0].split(':')[1] + ',' + line[1].split(':')[1]
+            return ','.join([x.split(':')[1] for x in line])
+
+
 def _read_nodes(reservations, pending_res):
     sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c"').read()), sep=';')
     sinfo_df = _split_column(sinfo_df, 'NODELIST').drop_duplicates()
     sinfo_df['n_gpu'] = sinfo_df['GRES'].apply(lambda x: 0 if x.split('(')[0].split(':')[0] in ('(null)', '') else int(x.split('(')[0].split(':')[-1]))
-    sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(':').apply(lambda x: 'n/a' if x[0] in ('(null)', '') else x[1])
+    # sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(':').apply(lambda x: 'n/a' if x[0] in ('(null)', '') else x[1])
+    sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
     sinfo_df['reserved'] = 'no'
     if reservations is not None:
         sinfo_df.loc[sinfo_df['NODELIST'].isin(reservations.Nodes.unique()), 'reserved'] = 'yes'

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/view/curses_multiwindow.py

@@ -15,6 +15,24 @@ from view import update_views
 
 a_filter_values = [None, 'me', 'prod', 'stud', 'cvcs']
 
+BUTTON_ACTIONS = {
+    'up_1': [ord('w'), curses.KEY_UP, ord('k')],
+    'down_1': [ord('s'), curses.KEY_DOWN, ord('j')],
+    'left_1': [ord('a'), curses.KEY_LEFT, ord('h')],
+    'right_1': [ord('d'), curses.KEY_RIGHT, ord('l')],
+    'up_tab': [curses.KEY_NPAGE],
+    'down_tab': [curses.KEY_PPAGE],
+    'quit': [ord('q'), 'q'],
+    'sort_prio': [ord('S')],
+    'res_view_mode': [ord('g')],
+    'job_id_type': [ord('b')],
+    'show_starttime': [ord('z')],
+    'show_account': [ord('t')],
+    'show_prio': [ord('p')],
+    'refresh': [ord('y')],
+    'info': [ord('i')],
+}
+
 
 def try_open_socket_as_slave(instance):
     if not instance.port_file_exists():
@@ -289,8 +307,8 @@ class Buffer(object):
                     xacc += len(chunk)
         if len(self.buffer) > self.lines - 2:
             self.screen.addstr(self.lines - 2, instance.xoffset + instance.left_width // 2 - 6, ' ▼ SCROLL ▲ ', curses.color_pair(2) | curses.A_REVERSE)
-            instance.add_button(self.lines - 2, instance.xoffset + 31, 'D', ord('s'))
-            instance.add_button(self.lines - 2, instance.xoffset + 40, 'U', ord('w'))
+            instance.add_button(self.lines - 2, instance.xoffset + 31, 'D', BUTTON_ACTIONS['down_1'][0])
+            instance.add_button(self.lines - 2, instance.xoffset + 40, 'U', BUTTON_ACTIONS['up_1'][0])
 
         self.window.border()
         self.window.noutrefresh()
@@ -330,35 +348,50 @@ def handle_keys(stdscr, instance):
 
     # process input
     # RIGHT
-    if k == ord('d') or k == 261:
+    if k in BUTTON_ACTIONS['right_1']:
         instance.a_filter = (instance.a_filter + 1) % len(a_filter_values)
         instance.voff = 0
     # LEFT
-    elif k == ord('a') or k == 260:
+    elif k in BUTTON_ACTIONS['left_1']:
         instance.a_filter = (instance.a_filter + (len(a_filter_values) - 1)) % len(a_filter_values)
         instance.voff = 0
     elif valid_mouse and isinstance(k, str) and k.startswith('AF_'):
         instance.a_filter = int(k.split('AF_')[1])
     # DOWN
-    elif k == ord('s') or k == 258:
+    elif k in BUTTON_ACTIONS['down_1']:
         instance.voff += 1
     # UP
-    elif k == ord('w') or k == 259:
+    elif k in BUTTON_ACTIONS['up_1']:
         instance.voff -= 1
 
-    if k == ord('S'):
+    elif k in BUTTON_ACTIONS['sort_prio']:
         instance.sort_by_prio = not instance.sort_by_prio
-    if k == ord('g'):
+    elif k in BUTTON_ACTIONS['res_view_mode']:
         instance.view_mode = {"gpu": "ram", "ram": "cpu", "cpu": "gpu"}[instance.view_mode]
-    if k == ord('j'):
+        instance.right_width = 33 if instance.view_mode != 'info' else 39
+        stdscr.clear()
+    elif k in BUTTON_ACTIONS['job_id_type']:
         instance.job_id_type = "true" if instance.job_id_type == "agg" else "agg"
-    if k == ord('z'):
+    elif k in BUTTON_ACTIONS['show_starttime']:
         # instance.show_starttime = not instance.show_starttime
         pass
-    if k == ord('t'):
+    elif k in BUTTON_ACTIONS['show_account']:
         instance.show_account = not instance.show_account
-    if k == ord('p'):
+    elif k in BUTTON_ACTIONS['show_prio']:
         instance.show_prio = not instance.show_prio
+    elif k in BUTTON_ACTIONS['info']:
+        instance.view_mode = 'info' if instance.view_mode != 'info' else 'gpu'
+        instance.right_width = 33 if instance.view_mode != 'info' else 39
+        stdscr.clear()
+    elif k in BUTTON_ACTIONS['up_tab']:
+        # get screen size
+        height, width = stdscr.getmaxyx()
+
+        instance.voff += ((height - 4) // 2) + 1
+    elif k in BUTTON_ACTIONS['down_tab']:
+        height, width = stdscr.getmaxyx()
+
+        instance.voff -= ((height - 4) // 2) + 1
 
 
 def update_screen(stdscr, instance):
@@ -369,7 +402,7 @@ def update_screen(stdscr, instance):
     lines, columns = os.get_terminal_size().lines, os.get_terminal_size().columns
     s_lines, s_columns = stdscr.getmaxyx()
     instance.max_columns = s_columns
-    if instance.k == ord('y'):
+    if instance.k in BUTTON_ACTIONS['refresh']:
         stdscr.clear()
 
     totsize = 106
@@ -395,11 +428,12 @@ def update_screen(stdscr, instance):
 
     stdscr.refresh()
 
-    left_width = columns - 33  # 72
+    left_width = columns - instance.right_width  # 72
+    right_width = instance.right_width  # 33
     instance.left_width = left_width
     left_window = curses.newwin(lines - 1, left_width, 0, xoffset)
     left_buffer = Buffer(left_window, lines, stdscr)
-    right_window = curses.newwin(lines - 1, 31, 0, xoffset + left_width + 1)
+    right_window = curses.newwin(lines - 1, right_width - 2, 0, xoffset + left_width + 1)
     right_buffer = Buffer(right_window, lines, stdscr)
 
     left_buffer.write(instance.rens)
@@ -409,7 +443,7 @@ def update_screen(stdscr, instance):
 
     # render menu
     stdscr.addstr(lines - 1, xoffset + 1 + 0, '◀')
-    instance.add_button(lines - 1, xoffset + 1 + 0, '◀', ord('a'))
+    instance.add_button(lines - 1, xoffset + 1 + 0, '◀', BUTTON_ACTIONS['left_1'][0])
     stdscr.addstr(lines - 1, xoffset + 1 + 2, 'ALL', curses.color_pair(2) | (curses.A_REVERSE if a_filter_values[instance.a_filter] is None else 0))
 
     instance.add_button(lines - 1, xoffset + 1 + 2, 'ALL', 'AF_0')
@@ -424,37 +458,42 @@ def update_screen(stdscr, instance):
     instance.add_button(lines - 1, xoffset + 1 + 19, 'CVCS', 'AF_4')
 
     stdscr.addstr(lines - 1, xoffset + 1 + 24, '▶')
-    instance.add_button(lines - 1, xoffset + 1 + 24, '▶', ord('d'))
+    instance.add_button(lines - 1, xoffset + 1 + 24, '▶', BUTTON_ACTIONS['right_1'][0])
     stdscr.addstr(lines - 1, xoffset + 1 + 25, ' ' * (columns - 27 - xoffset))
 
     stdscr.addstr(lines - 1, left_width - 8, '[Q:QUIT]', curses.color_pair(2))
-    instance.add_button(lines - 1, left_width - 8, '[Q:QUIT]', ord('q'))  # 53
+    instance.add_button(lines - 1, left_width - 8, '[Q:QUIT]', BUTTON_ACTIONS['quit'][0])  # 53
 
     stdscr.addstr(lines - 1, left_width - 18 + 19, '[Y:REDRAW]', curses.color_pair(2))
-    instance.add_button(lines - 1, left_width - 18 + 19, '[Y:REDRAW]', ord('y'))
+    instance.add_button(lines - 1, left_width - 18 + 19, '[Y:REDRAW]', BUTTON_ACTIONS['refresh'][0])
 
+    stdscr.addstr(0, left_width + 2, '[I:', curses.color_pair(2))
+    stdscr.addstr(0, left_width + 5, 'INFO', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'info' else 0))
+    stdscr.addstr(0, left_width + 9, ']', curses.color_pair(2))
+    instance.add_button(0, left_width + 2, '[I:INFO]', BUTTON_ACTIONS['info'][0])
+    stdscr.addstr(0, left_width + 10, '─' * (columns - 15 - left_width - 3), curses.color_pair(2))
     stdscr.addstr(0, columns - 15, '[G:', curses.color_pair(2))
     stdscr.addstr(0, columns - 12, 'GPU', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'gpu' else 0))
     stdscr.addstr(0, columns - 12 + 3, 'RAM', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'ram' else 0))
     stdscr.addstr(0, columns - 12 + 6, 'CPU', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'cpu' else 0))
     stdscr.addstr(0, columns - 12 + 9, ']', curses.color_pair(2))
-    instance.add_button(0, columns - 15, '[G:GPURAMCPU]', ord('g'))
+    instance.add_button(0, columns - 15, '[G:GPURAMCPU]', BUTTON_ACTIONS['res_view_mode'][0])
 
-    stdscr.addstr(lines - 1, xoffset + 25 + 2, '[J:', curses.color_pair(2))
+    stdscr.addstr(lines - 1, xoffset + 25 + 2, '[B:', curses.color_pair(2))
     stdscr.addstr(lines - 1, xoffset + 25 + 2 + 3, 'AGG', curses.color_pair(2) | (curses.A_REVERSE if instance.job_id_type == 'agg' else 0))
     stdscr.addstr(lines - 1, xoffset + 25 + 2 + 3 + 3, 'TRUE', curses.color_pair(2) | (curses.A_REVERSE if instance.job_id_type == 'true' else 0))
     stdscr.addstr(lines - 1, xoffset + 25 + 2 + 3 + 3 + 4, ']', curses.color_pair(2))
-    instance.add_button(lines - 1, xoffset + 25 + 2, '[J:AGGTRUE]', ord('j'))
+    instance.add_button(lines - 1, xoffset + 25 + 2, '[B:AGGTRUE]', BUTTON_ACTIONS['job_id_type'][0])
 
     stdscr.addstr(lines - 1, xoffset + 37 + 2, '[P:', curses.color_pair(2))
     stdscr.addstr(lines - 1, xoffset + 37 + 2 + 3, 'PRIORITY', curses.color_pair(2) | (curses.A_REVERSE if instance.show_prio else 0))
     stdscr.addstr(lines - 1, xoffset + 37 + 2 + 3 + 8, ']', curses.color_pair(2))
-    instance.add_button(lines - 1, xoffset + 37 + 2, '[P:PRIORITY]', ord('p'))
+    instance.add_button(lines - 1, xoffset + 37 + 2, '[P:PRIORITY]', BUTTON_ACTIONS['show_prio'][0])
 
     stdscr.addstr(lines - 1, xoffset + 50 + 2, '[T:', curses.color_pair(2))
     stdscr.addstr(lines - 1, xoffset + 50 + 2 + 3, 'ACCOUNT', curses.color_pair(2) | (curses.A_REVERSE if instance.show_account else 0))
     stdscr.addstr(lines - 1, xoffset + 50 + 2 + 3 + 7, ']', curses.color_pair(2))
-    instance.add_button(lines - 1, xoffset + 50 + 2, '[T:ACCOUNT]', ord('t'))
+    instance.add_button(lines - 1, xoffset + 50 + 2, '[T:ACCOUNT]', BUTTON_ACTIONS['show_account'][0])
 
     # get slurm user partition
     stdscr.addstr(lines - 1, xoffset + 62 + 2, f'(Avg time {instance.avg_wait_time})', curses.color_pair(2))
@@ -467,7 +506,7 @@ def update_screen(stdscr, instance):
 
 
 def get_char_async(stdscr, instance):
-    while instance.k != ord('q') and instance.k != 'q':
+    while instance.k not in BUTTON_ACTIONS['quit']:  # != 'q'
         handle_keys(stdscr, instance)
         update_screen(stdscr, instance)
 
@@ -476,7 +515,7 @@ def get_char_async(stdscr, instance):
 
 
 async def update_screen_info(stdscr, instance):
-    while instance.k != ord('q') and instance.k != 'q':
+    while instance.k not in BUTTON_ACTIONS['quit']:
         await instance.fetch()
         instance.log("GOT DATA FROM MASTER")
         update_screen(stdscr, instance)
@@ -547,6 +586,7 @@ async def main(stdscr):
     # status
 
     instance.a_filter = 0
+    instance.right_width = 33
 
     stdscr.clear()
 

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/view/slurm_list.py

@@ -21,7 +21,7 @@ def _joblet_format(instance, job, width=74, jobid_type='agg'):
         if instance.show_prio:
             joblet_repr += '(' + _format_to(job.priority, len(str(job.priority))) + ')'
             joblet_repr += ' '
-            loffset += 9
+            loffset += len(str(job.priority)) + 3
         if instance.show_account:
             joblet_repr += _format_to(job.account, 9)
             joblet_repr += ' '

{nodeocc-1.0.4 → nodeocc-1.0.6}/src/view/slurm_viz.py

@@ -115,6 +115,8 @@ def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=Non
         return view_viz_ram(infrastructure, jobs, work, stylefn, current_user)
     elif mode == 'cpu':
         return view_viz_cpu(infrastructure, jobs, work, stylefn, current_user)
+    elif mode == 'info':
+        return view_viz_info(infrastructure, jobs, work, stylefn, current_user)
 
 
 def view_viz_ram(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
@@ -354,6 +356,9 @@ def get_cpu_icon(stat):
     }.get(stat, cpu_avail)
 
 
+GPU_MEM = {"K80": 12, "P100": 16, "1080": 11, "TitanXP": 12, "2080": 11, "RTX5000": 16, "RTX6000": 24, "RTXA5000": 24, "A40": 48}
+
+
 def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
     # this is for hot reload
     if not work:
@@ -480,6 +485,70 @@ def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user
     return RetScope.return_string
 
 
+def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
+    # this is for hot reload
+    if not work:
+        return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
+
+    # who is the current user?
+    if current_user is None:
+        import os
+        current_user = os.path.basename(os.environ['HOME'])
+
+    nodes = infrastructure.get_sorted_nodes()
+    infrast_down = all([x.status == 'down' for x in nodes])
+
+    class RetScope:
+        return_string = ''
+
+    def cust_print(thing, style=None):
+        RetScope.return_string += (thing if style is None else stylefn(style, thing)) + '\n'
+
+    if not infrast_down:
+        stalled_jobs = sum([j.state == 'S' for j in jobs])
+        max_gpu_len = max([len(n.gpu_model) + len(str(n.n_gpus)) if n.gpu_model is not None else 0 for n in nodes])
+        max_n_gpu_len = max([len(str(n.n_gpus)) for n in nodes])
+        max_cpu_len = max([len(str(n.cpus)) for n in nodes] + [len('CPUs')])
+        max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
+        RetScope.return_string += f'{"":{gpu_name_chars}} {"GPUs":{max_gpu_len+1}}\n'
+        for n in nodes:
+            gpu_model = n.gpu_model if n.gpu_model is not None else 'N/A'
+            # gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
+            mem = n.mem // 1024 if n.mem is not None else 'N/A'
+            n_gpus = n.n_gpus if n.n_gpus is not None else 'N/A'
+            n_cpus = n.cpus if n.cpus is not None else 'N/A'
+            # gpu_mem = GPU_MEM.get(gpu_model, 'N/A')
+            gpu_mem = ','.join([str(GPU_MEM.get(x, 'N/A')) for x in gpu_model.split(',')])
+            # gpu_str = f'{n_gpus}x{gpu_model:{max_gpu_len-(1*(len(str(n_gpus))))}}' if n_gpus != 0 else f'{"N/A":{max_gpu_len+1}}'
+            if n_gpus != 0:
+                gpu_str = f'{stylefn("RED", str(n_gpus))}'
+                gpu_str += f'x{stylefn("YELLOW", gpu_model)}'
+                gpu_str += f'@{stylefn("GREEN", str(gpu_mem))}G'
+            else:
+                gpu_str = f'{"N/A":{max_gpu_len+1}}'
+            RetScope.return_string += f'{_format_to(n.name, gpu_name_chars, "right")} {gpu_str}\n'
+            # RetScope.return_string += f'{"":{gpu_name_chars}} {"VRAM":{max_gpu_len+1}} {f"{mem}G"}\n'
+
+    onmain, waitString = maintenance_status(infrastructure)
+    if onmain:
+        cust_print('  ◀ ONGOING  MAINTENANCE ▶    ', 'BG_MAGENTA')
+    elif len(infrastructure.maintenances):
+        cust_print('  ◀ MAINTENANCE  in %4s ▶    ' % waitString, 'BG_MAGENTA')
+    elif len(jobs) == 0:
+        cust_print('         ◀ NO  JOBS ▶         ', 'BG_GREEN')
+    elif stalled_jobs / len(jobs) > 0.5:
+        cust_print('       ◀ JOBS ON HOLD ▶       ', 'BG_YELLOW')
+    else:
+        cust_print('')
+
+    # print summary
+    # cust_print(''.join([' ['+ ram_occ + 'run', paused + 'hld', drain + 'drn', down + 'dwn', '()res]']))
+    # cust_print(''.join(['[' + ram_occ + f'{cpu_size}{cpu_unit}', cpu_paused + 'hld', cpu_drain + 'drn', cpu_pendr + 'pnd', cpu_down + 'dwn', '()res]']))
+    # gpuc = 'GREEN'
+
+    return RetScope.return_string
+
+
 if __name__ == '__main__':
     import sys