nodeocc 1.0.7__tar.gz → 1.0.9__tar.gz

{nodeocc-1.0.7 → nodeocc-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.7
+Version: 1.0.9
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.7 → nodeocc-1.0.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.7"
+version = "1.0.9"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/controller/controller.py

@@ -66,7 +66,7 @@ END_DELIM_ENCODED = "!}$".encode('utf-8')
 EXTRA_MSG_BEGIN_DELIM = "!{#"
 MAX_BUF = 65535
 MAX_MSG_LEN = 1024 * 1024 * 1024
-JOB_LIMIT_PER_MSG = 100
+JOB_LIMIT_PER_MSG = 60
 
 parser = argparse.ArgumentParser(description='Visualize slurm jobs')
 parser.add_argument('--debug', action='store_true', help='Enable logging')

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/model/infrastructure.py

@@ -10,6 +10,7 @@ class Node:
     def __init__(self, name, n_gpus, gpu_model, reserved, status, cpus, mem) -> None:
         self.name = name
         self.n_gpus = n_gpus
+        # self.feature = feature
         self.gpu_model = gpu_model
         self.reserved = reserved
         assert reserved in ('no', 'yes', 'pending')

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.7
+Version: 1.0.9
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/readers/slurmreader.py

@@ -31,7 +31,7 @@ def _split_column(df, column):
 
 
 def _node_from_sinfo(x):
-    n = Node(_node_preproc(x['NODELIST']), x.n_gpu, x.m_gpu, x.reserved, None, x.CPUS, x.MEMORY)
+    n = Node(_node_preproc(x['NODELIST']), x.n_gpu, x.m_gpu, x.reserved, None, x.CPUS, x.MEMORY)#, feature=x.AVAIL_FEATURES)
     if x['STATE'] == 'drng' or 'drain' in x['STATE']:
         n.status = 'drain'
     elif x['STATE'].upper() in ('MAINT', 'DOWN', 'DOWN*', 'FAIL', 'FAILING', 'FAIL*'):
@@ -53,18 +53,20 @@ def _parse_gpu_models(line):
             return ','.join([x.split(':')[1] for x in line])
 
 
-def _read_nodes(reservations, pending_res):
-    sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c"').read()), sep=';')
+def _read_nodes(reservations=None, pending_res=None, return_raw=False):
+    sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c;%f"').read()), sep=';')
     sinfo_df = _split_column(sinfo_df, 'NODELIST').drop_duplicates()
-    sinfo_df['n_gpu'] = sinfo_df['GRES'].apply(lambda x: 0 if x.split('(')[0].split(':')[0] in ('(null)', '') else int(x.split('(')[0].split(':')[-1]))
+    sinfo_df.loc[:,'n_gpu'] = sinfo_df['GRES'].apply(lambda x: 0 if x.split('(')[0].split(':')[0] in ('(null)', '') else int(x.split('(')[0].split(':')[-1]))
     # sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(':').apply(lambda x: 'n/a' if x[0] in ('(null)', '') else x[1])
-    sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
+    sinfo_df.loc[:,'m_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
     sinfo_df['reserved'] = 'no'
     if reservations is not None:
         sinfo_df.loc[sinfo_df['NODELIST'].isin(reservations.Nodes.unique()), 'reserved'] = 'yes'
     if pending_res is not None:
         sinfo_df.loc[sinfo_df['NODELIST'].isin(pending_res.Nodes.unique()), 'reserved'] = 'pending'
-
+    if return_raw:
+        return sinfo_df
+    
     return sinfo_df.apply(_node_from_sinfo, axis=1).tolist()
 
 
@@ -83,8 +85,10 @@ def _read_maintenances():
     if not len(reservations):
         return [], None, None
     reservations = pd.DataFrame.from_records(reservations)
-    reservations['StartTime'] = str(pd.to_datetime(reservations['StartTime']))
-    reservations['EndTime'] = str(pd.to_datetime(reservations['EndTime']))
+    reservations['StartTime'] = reservations['StartTime'].apply(lambda x:str(pd.to_datetime(x)))
+    reservations['EndTime'] = reservations['EndTime'].apply(lambda x:str(pd.to_datetime(x)))
+    # reservations['StartTime'] = str(pd.to_datetime(reservations['StartTime']))
+    # reservations['EndTime'] = str(pd.to_datetime(reservations['EndTime']))
     reservations = _split_column(reservations, 'Nodes')
     maint_flag = reservations['Flags'].str.contains('MAINT') & \
         reservations['Flags'].str.contains('SPEC_NODES') & \

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/view/__init__.py

@@ -1,12 +1,16 @@
 import curses
 from . import slurm_list, slurm_viz, styles
 
+
 def update_views(stdscr, instance, filter):
     if instance.jobs is None or instance.inf is None:
         return
 
-    instance.rens = slurm_list.view_list(instance, instance.jobs, filter, stylefn=styles.crsstyler, width=instance.left_width if hasattr(instance, 'left_width') else 72, jit=instance.job_id_type if hasattr(instance, 'job_id_type') else 'agg')
+    instance.rens = slurm_list.view_list(
+        instance, instance.jobs, filter, stylefn=styles.crsstyler, width=instance.left_width if hasattr(
+            instance, 'left_width') else 72, jit=instance.job_id_type if hasattr(
+            instance, 'job_id_type') else 'agg')
     instance.timeme(f"- print job list")
 
     instance.nocc = slurm_viz.view_viz(instance.inf, instance.jobs, stylefn=styles.crsstyler, mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu')
-    instance.timeme(f"- view nodeocc")
+    instance.timeme(f"- view nodeocc")

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/view/slurm_viz.py

@@ -1,6 +1,7 @@
 import random
 
 import pandas as pd
+import numpy as np
 
 from view.styles import _format_to, cmdstyle
 from view.utils import is_cvcs_viz, is_student_viz, maintenance_status, to_font
@@ -99,6 +100,8 @@ cpu_pendr = '⧖'
 cpu_paused = '▚'
 cpu_down = '⨯'
 
+GLOBAL_NODES = None
+
 
 def get_ram_block(megs):
     return int(round(megs / 1024 / mem_size))
@@ -356,9 +359,6 @@ def get_cpu_icon(stat):
     }.get(stat, cpu_avail)
 
 
-GPU_MEM = {"K80": 12, "P100": 16, "1080": 11, "TitanXP": 12, "2080": 11, "RTX5000": 16, "RTX6000": 24, "RTXA5000": 24, "A40": 48}
-
-
 def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
     # this is for hot reload
     if not work:
@@ -490,13 +490,22 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
     if not work:
         return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
 
+    global GLOBAL_NODES
+    if GLOBAL_NODES is None:
+        from readers.slurmreader import _read_nodes, _node_preproc
+        GLOBAL_NODES = _read_nodes(return_raw=True)
+        GLOBAL_NODES.loc[:, 'nodename'] = GLOBAL_NODES.NODELIST.apply(_node_preproc)
+        prior = ['n/a', None, 'A40', 'RTXA5000', 'RTX6000', 'RTX5000', '2080', 'V100', '1080', 'P100', 'K80']
+        reord_idx = sorted(zip(GLOBAL_NODES['m_gpu'], GLOBAL_NODES.index), key=lambda g: prior.index(g[0].split(',')[0]) if g[0].split(',')[0] in prior else -1)
+        GLOBAL_NODES = GLOBAL_NODES.loc[np.array([i for x, i in reord_idx])].reset_index(drop=True)
+
     # who is the current user?
     if current_user is None:
         import os
         current_user = os.path.basename(os.environ['HOME'])
 
-    nodes = infrastructure.get_sorted_nodes()
-    infrast_down = all([x.status == 'down' for x in nodes])
+    # nodes = infrastructure.get_sorted_nodes()
+    infrast_down = all([x.status == 'down' for x in infrastructure.get_sorted_nodes()])
 
     class RetScope:
         return_string = ''
@@ -506,27 +515,34 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
 
     if not infrast_down:
         stalled_jobs = sum([j.state == 'S' for j in jobs])
-        max_gpu_len = max([len(n.gpu_model) + len(str(n.n_gpus)) if n.gpu_model is not None else 0 for n in nodes])
-        max_n_gpu_len = max([len(str(n.n_gpus)) for n in nodes])
-        max_cpu_len = max([len(str(n.cpus)) for n in nodes] + [len('CPUs')])
-        max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
-        RetScope.return_string += f'{"":{gpu_name_chars}} {"GPUs":{max_gpu_len+1}}\n'
-        for n in nodes:
-            gpu_model = n.gpu_model if n.gpu_model is not None else 'N/A'
+        # max_gpu_len = max([len(n.gpu_model) + len(str(n.n_gpus)) if n.gpu_model is not None else 0 for n in nodes])
+        # max_n_gpu_len = max([len(str(n.n_gpus)) for n in nodes])
+        # max_cpu_len = max([len(str(n.cpus)) for n in nodes] + [len('CPUs')])
+        # max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
+        feat_str = stylefn("GREEN", "FEATURES")
+        RetScope.return_string += f'{"":{gpu_name_chars}} GPU {feat_str}\n'
+        for i, n in GLOBAL_NODES.iterrows():
+            gpu_model = n.m_gpu  # if n.m_gpu is not None else 'N/A'
             # gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
-            mem = n.mem // 1024 if n.mem is not None else 'N/A'
-            n_gpus = n.n_gpus if n.n_gpus is not None else 'N/A'
-            n_cpus = n.cpus if n.cpus is not None else 'N/A'
+            mem = n.MEMORY // 1024 if n.MEMORY is not None else 'N/A'
+            n_gpus = n.n_gpu if n.n_gpu is not None else 'N/A'
+            n_cpus = n.CPUS  # if n.CPUS is not None else 'N/A'
             # gpu_mem = GPU_MEM.get(gpu_model, 'N/A')
-            gpu_mem = ','.join([str(GPU_MEM.get(x, 'N/A')) for x in gpu_model.split(',')])
+
+            gpu_feature = n.AVAIL_FEATURES  # if n.feature is not None else '(null)'
+
+            # gpu_mem = ','.join([str(GPU_MEM.get(x, 'N/A')) for x in gpu_model.split(',')])
             # gpu_str = f'{n_gpus}x{gpu_model:{max_gpu_len-(1*(len(str(n_gpus))))}}' if n_gpus != 0 else f'{"N/A":{max_gpu_len+1}}'
             if n_gpus != 0:
                 gpu_str = f'{stylefn("RED", str(n_gpus))}'
-                gpu_str += f'x{stylefn("YELLOW", gpu_model)}'
-                gpu_str += f'@{stylefn("GREEN", str(gpu_mem))}G'
+                if gpu_feature != '(null)':
+                    gpu_str += f'x{stylefn("GREEN", gpu_feature)}'
+                else:
+                    gpu_str += f'@({stylefn("YELLOW", gpu_model)})'
             else:
-                gpu_str = f'{"N/A":{max_gpu_len+1}}'
-            RetScope.return_string += f'{_format_to(n.name, gpu_name_chars, "right")} {gpu_str}\n'
+                gpu_str = "N/A"  # f'{"N/A":{max_gpu_len+1}}'
+
+            RetScope.return_string += f'{_format_to(n.nodename, gpu_name_chars, "right")} {gpu_str}\n'
             # RetScope.return_string += f'{"":{gpu_name_chars}} {"VRAM":{max_gpu_len+1}} {f"{mem}G"}\n'
 
     onmain, waitString = maintenance_status(infrastructure)

{nodeocc-1.0.7 → nodeocc-1.0.9}/src/view/utils.py

@@ -22,8 +22,12 @@ def to_datetime(time_str):
     return np.datetime64(re.match(r'\d+\s+(.*)\n', str(time_str)).group(1))
 
 
-def to_datetime(time_str):
-    return np.datetime64(re.match(r'\d+\s+(.*)\n', str(time_str)).group(1))
+def to_datetime(time_str): 
+    try:
+        dt = np.datetime64(re.match(r'\d+\s+(.*)\n', str(time_str)).group(1))
+    except:
+        dt = np.datetime64(str(time_str))
+    return dt
 
 
 def maintenance_status(infrastructure):