nodeocc 1.0.9__tar.gz → 1.0.11__tar.gz

{nodeocc-1.0.9 → nodeocc-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.9
+Version: 1.0.11
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -14,6 +14,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Requires-Dist: numpy
+Requires-Dist: psutil
 
 # Slurm NodeOCC
 ![[screenshot](https://github.com/apanariello4/slurm_nodeocc/blob/master/sshot.jpg?raw=true)](sshot.jpg)

{nodeocc-1.0.9 → nodeocc-1.0.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.9"
+version = "1.0.11"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"
@@ -27,7 +27,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 
-dependencies = ["pandas", "numpy"]
+dependencies = ["pandas", "numpy", "psutil"]
 
 [project.urls]
 Repository = "https://github.com/apanariello4/slurm_nodeocc"

{nodeocc-1.0.9 → nodeocc-1.0.11}/src/controller/controller.py

@@ -60,7 +60,6 @@ except Exception as e:
     newest_version = '?.?.?'
     time.sleep(5)
 
-
 BEGIN_DELIM = "!{$"
 END_DELIM_ENCODED = "!}$".encode('utf-8')
 EXTRA_MSG_BEGIN_DELIM = "!{#"
@@ -70,8 +69,7 @@ JOB_LIMIT_PER_MSG = 60
 
 parser = argparse.ArgumentParser(description='Visualize slurm jobs')
 parser.add_argument('--debug', action='store_true', help='Enable logging')
-parser.add_argument('--master', action='store_true', help='Start master process')
-parser.add_argument('--daemon_only', action='store_true', help='Disable all prints - only run in background')
+parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
 parser.add_argument('--force_override', action='store_true', help='Force override of port file, kill previous instance')
 parser.add_argument('--basepath', type=str, default='/nas/softechict-nas-2/mboschini/cool_scripts/new_nodeocc/', help='Base path for nodeocc. Must be readable by all users')
 args = parser.parse_args()
@@ -80,15 +78,19 @@ args = parser.parse_args()
 assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
 
 # export args
-Singleton.getInstance(args)
+orig_instance = Singleton.getInstance(args)
 
-last_update = os.path.getmtime(conf_path + "/view/slurm_viz.py")
-if os.path.getmtime(conf_path + "/view/slurm_list.py") > last_update:
-    last_update = os.path.getmtime(conf_path + "/view/slurm_list.py")
-if os.path.getmtime(conf_path + "/view/styles.py") > last_update:
-    last_update = os.path.getmtime(conf_path + "/view/styles.py")
-if os.path.getmtime(conf_path + "/readers/slurmreader.py") > last_update:
-    last_update = os.path.getmtime(conf_path + "/readers/slurmreader.py")
+try:
+    last_update = os.path.getmtime(conf_path + "/view/slurm_viz.py")
+    if os.path.getmtime(conf_path + "/view/slurm_list.py") > last_update:
+        last_update = os.path.getmtime(conf_path + "/view/slurm_list.py")
+    if os.path.getmtime(conf_path + "/view/styles.py") > last_update:
+        last_update = os.path.getmtime(conf_path + "/view/styles.py")
+    if os.path.getmtime(conf_path + "/readers/slurmreader.py") > last_update:
+        last_update = os.path.getmtime(conf_path + "/readers/slurmreader.py")
+except Exception as e:
+    orig_instance.err(f"Could not get last update time: {e}")
+    last_update = time.time()
 
 
 def get_avg_wait_time(instance: Singleton):
@@ -104,7 +106,8 @@ def get_avg_wait_time(instance: Singleton):
         instance.err(traceback.format_exc())
         return 'err', 'err'
 
-def send_msg(instance, msg, n_extra_msgs=0, is_extra=False):
+
+def send_msg(instance: Singleton, msg, n_extra_msgs=0, is_extra=False):
     # msg to bytes
     msg = msg.encode('utf-8')
     begin_delim = EXTRA_MSG_BEGIN_DELIM if is_extra else BEGIN_DELIM
@@ -116,11 +119,11 @@ def send_msg(instance, msg, n_extra_msgs=0, is_extra=False):
 
     instance.sock.sendto(msg, ('<broadcast>', instance.port))
 
-def update_data_master(instance):
-    if not instance.check_port_file_master():
-        instance.log(f"Port file dead, killing current instance")
-        instance.sock.close()
-        del instance
+
+def update_data_master(instance: Singleton):
+    if instance.check_existing_master_running():
+        instance.log(f"Master already running, killing current instance")
+        instance.cleanup()
         exit(0)
 
     inf = readers.slurmreader.read_infrastructure()
@@ -134,12 +137,11 @@ def update_data_master(instance):
     queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
     queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
     maxlen = min(JOB_LIMIT_PER_MSG, len(running_jobs) + len(queue_jobs))
-    
+
     queue_jobs, extra_queue_jobs = queue_jobs[:maxlen - len(running_jobs)], queue_jobs[maxlen - len(running_jobs):]
     cur_timestamp = time.time()
     msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs, 'ts': str(cur_timestamp)})
 
-
     N_EXTRA_MSGS = (len(extra_queue_jobs) // JOB_LIMIT_PER_MSG) + 1 if len(extra_queue_jobs) > 0 else 0
     send_msg(instance, msg, n_extra_msgs=N_EXTRA_MSGS)
     instance.timeme("- main broadcast")
@@ -154,6 +156,7 @@ def update_data_master(instance):
 
     return inf, jobs
 
+
 def decode_msg_slave(data, delim=BEGIN_DELIM):
     data = data.split(END_DELIM_ENCODED)[0]
     data = data.decode('utf-8')
@@ -165,8 +168,9 @@ def decode_msg_slave(data, delim=BEGIN_DELIM):
     avg_wait_time = "N/A"  # get_avg_wait_time(instance)
     return inf, jobs, avg_wait_time, msg
 
+
 async def get_data_slave(instance):
-    inf, jobs = None, None
+    inf, jobs, avg_wait_time = None, None, None
     try:
         # listen for data from master
         # instance.sock.settimeout(6.5)
@@ -182,24 +186,24 @@ async def get_data_slave(instance):
             orig_timestamp = float(decoded_msg['ts'])
 
             total_msg_n = n_msgs_remaining
-            msg_store = {i:[] for i in range(total_msg_n)}
-            while n_msgs_remaining>0:
+            msg_store = {i: [] for i in range(total_msg_n)}
+            while n_msgs_remaining > 0:
                 data, _ = instance.sock.recvfrom(MAX_BUF)
                 back_msg_idx = int(data.decode('utf-8').split(EXTRA_MSG_BEGIN_DELIM)[0])
-                msg_idx = total_msg_n - back_msg_idx - 1 
+                msg_idx = total_msg_n - back_msg_idx - 1
                 _, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
 
                 # check if timestamp is the same
                 if float(new_ts['ts']) != orig_timestamp:
                     instance.timeme(f"Timestamp mismatch, skipping")
-                    return inf, jobs, avg_wait_time # return what we have
-                
+                    return inf, jobs, avg_wait_time  # return what we have
+
                 msg_store[msg_idx] += new_jobs
 
                 n_msgs_remaining -= 1
                 instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
 
-            # sort and merge 
+            # sort and merge
             for msg_idx in range(total_msg_n):
                 if len(msg_store[msg_idx]) == 0:
                     instance.log(f"Empty message {msg_idx}")
@@ -226,8 +230,7 @@ async def get_all():
     updt = max(updt, os.path.getmtime(conf_path + "/view/styles.py"))
     updt = max(updt, os.path.getmtime(conf_path + "/readers/slurmreader.py"))
 
-    instance = Singleton.getInstance()
-    instance.timeme(f"Starting update")
+    orig_instance.timeme(f"Starting update")
 
     if updt > last_update:
         reload(readers.slurmreader)
@@ -236,21 +239,23 @@ async def get_all():
         reload(view.styles)
         last_update = updt
 
-    instance.timeme(f"- reload")
+    orig_instance.timeme(f"- reload")
 
     inf, jobs, avg_wait_time = None, None, None
     try:
-        if args.master:
-            inf, jobs = update_data_master(instance)
+        if orig_instance.is_master:
+            inf, jobs = update_data_master(orig_instance)
+
+            time.sleep(5)
         else:
             # loop = asyncio.get_event_loop()
-            instance.timeme(f"- listening for data")
+            orig_instance.timeme(f"- listening for data")
 
             # wait for data from master but async update the view
-            inf, jobs, avg_wait_time = await get_data_slave(instance)
+            inf, jobs, avg_wait_time = await get_data_slave(orig_instance)
     except Exception as e:
-        instance.err(f"Exception: {e}")
-        instance.err(traceback.format_exc())
+        orig_instance.err(f"Exception: {e}")
+        orig_instance.err(traceback.format_exc())
         # instance.rens = 'Something went wrong'
         # instance.nocc = ':('
 
@@ -262,43 +267,44 @@ def display_main(stdscr):
 
 
 def _main():
-    if args.daemon_only:
-        assert args.master, "Daemon mode only available for master"
-        instance = Singleton.getInstance()
+    if args.master_only:
+        assert orig_instance.is_master, "Instance is not `master`, run with `force_override` to override"
 
-        instance.log(f"Starting master daemon")
+        orig_instance.log(f"Starting master daemon")
         # register atexit
         import atexit
 
         def exit_handler():
-            instance.log(f"Exiting...")
-            instance.sock.close()
+            orig_instance.log(f"Exiting...")
+            orig_instance.sock.close()
             # remove .port file
-            if instance.port_file_exists():
-                Path(instance.get_port_file_name()[1]).unlink()
+            if orig_instance.port_file_exists():
+                Path(orig_instance.get_port_file_name()[1]).unlink()
         atexit.register(exit_handler)
 
         while True:
-            instance.timeme(f"Updating...")
+            orig_instance.timeme(f"Updating...")
             try:
-                update_data_master(instance)
+                update_data_master(orig_instance)
             except Exception as e:
-                instance.err(f"Exception: {e}")
-                instance.err(traceback.format_exc())
+                orig_instance.err(f"Exception: {e}")
+                orig_instance.err(traceback.format_exc())
 
             time.sleep(5)
 
     else:
         # configure singleton
 
-        Singleton.getInstance().signature = f"{program_name} v{version_number}"
-        Singleton.getInstance().version = version_number
-        Singleton.getInstance().updated = updated
-        Singleton.getInstance().newest_version = newest_version
-        Singleton.getInstance().fetch_fn = get_all
+        orig_instance.signature = f"{program_name} v{version_number}"
+        orig_instance.version = version_number
+        orig_instance.updated = updated
+        orig_instance.newest_version = newest_version
+        orig_instance.fetch_fn = get_all
 
         wrapper(display_main)
 
 
-if __name__ == '__main__':
-    _main()
+_main()
+
+orig_instance.cleanup()
+exit(0)

{nodeocc-1.0.9 → nodeocc-1.0.11}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.9
+Version: 1.0.11
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -14,6 +14,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Requires-Dist: numpy
+Requires-Dist: psutil
 
 # Slurm NodeOCC
 ![[screenshot](https://github.com/apanariello4/slurm_nodeocc/blob/master/sshot.jpg?raw=true)](sshot.jpg)

{nodeocc-1.0.9 → nodeocc-1.0.11}/src/nodeocc.egg-info/requires.txt

@@ -1,2 +1,3 @@
 pandas
 numpy
+psutil

{nodeocc-1.0.9 → nodeocc-1.0.11}/src/view/curses_multiwindow.py

@@ -1,5 +1,7 @@
 import asyncio
 import curses
+import random
+import psutil
 import grp
 import logging
 import os
@@ -10,7 +12,6 @@ import time
 import traceback
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
-
 from view import update_views
 
 a_filter_values = [None, 'me', 'prod', 'stud', 'cvcs']
@@ -56,19 +57,40 @@ class Singleton:
     __instance = None
 
     @staticmethod
-    def getInstance(args=None):
+    def getInstance(args=None, create=True):
         """ Static access method. """
         if Singleton.__instance is None:
+            if not create:
+                return None
             Singleton(args)
         if args is not None:
             Singleton.__instance.args = args
         return Singleton.__instance
 
     # destructor
-    def __del__(self):
-        if self.args.master:
+    def cleanup(self):
+        if self.is_master:
             if self.port_file_exists():
-                Path(self.get_port_file_name()[1]).unlink()
+                os.system(f"rm {self.get_port_file_name()[1]}")
+        if hasattr(self, 'sock') and self.sock is not None:
+            self.sock.close()
+
+    def setup_logging(self):
+        if self.args.master_only:
+            handler = RotatingFileHandler(os.path.join(self.basepath, '.master_log.txt'), maxBytes=5 * 1024 * 1024, backupCount=2, mode='w')
+            logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
+                                handlers=[handler])
+        # set logging file
+        elif self.args.debug:
+            # create rotating file handler
+            handler = RotatingFileHandler(os.path.expanduser('~') + '/.nodeocc_log.txt', maxBytes=5 * 1024 * 1024, backupCount=2, mode='w')
+            logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
+                                handlers=[handler])
+
+    def clean_port_files(self):
+        for f in os.listdir(self.basepath):
+            if f.endswith('.port'):
+                os.system(f"rm {os.path.join(self.basepath, f)}")
 
     def __init__(self, args=None):
         """ Virtually private constructor. """
@@ -76,6 +98,10 @@ class Singleton:
             raise Exception("This class is a singleton!")
         else:
             Singleton.__instance = self
+
+        self.args = args
+        self.setup_logging()
+
         self.try_open_counter = 0
         self.voff = 0
         self.mouse_state = {}
@@ -87,7 +113,6 @@ class Singleton:
         self.fetch_fn = None
         self.view_mode = 'gpu'
         self.job_id_type = 'agg'
-        self.args = args
 
         self.show_account = False
         self.show_prio = False
@@ -106,36 +131,27 @@ class Singleton:
 
         self.basepath = self.args.basepath
 
-        if self.args.daemon_only:
-            handler = RotatingFileHandler(os.path.join(self.basepath, '.master_log.txt'), maxBytes=5 * 1024 * 1024, backupCount=2, mode='w')
-            logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
-                                handlers=[handler])
-        # set logging file
-        elif self.args.debug:
-            # create rotating file handler
-            handler = RotatingFileHandler(os.path.expanduser('~') + '/.nodeocc_log.txt', maxBytes=5 * 1024 * 1024, backupCount=2, mode='w')
-            logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
-                                handlers=[handler])
+        # check if any .port file exists
+        if args.force_override and self.port_file_exists():
+            _, port_filepath = self.get_port_file_name()
 
-        if self.args.master:
-            # check if any .port file exists
-            if self.port_file_exists():
-                if self.args.force_override:
-                    _, port_filepath = self.get_port_file_name()
+            # read pid from file
+            pid = Path(port_filepath).read_text()
+            # kill process
+            try:
+                os.kill(int(pid), signal.SIGTERM)
+            except Exception as e:
+                self.log("Could not kill process, deleting port file")
+                self.log(e)
 
-                    # read pid from file
-                    pid = Path(port_filepath).read_text()
-                    # kill process
-                    try:
-                        os.kill(int(pid), signal.SIGTERM)
-                    except Exception as e:
-                        self.log("Could not kill process, deleting port file")
-                        self.log(e)
+            # delete port file
+            Path(port_filepath).unlink()
 
-                    # delete port file
-                    Path(port_filepath).unlink()
-                else:
-                    raise Exception("Master already running")
+        self.is_master = not self.check_existing_master_running()
+
+        if self.is_master:
+            # clean up old port files
+            self.clean_port_files()
             self.port = self.create_socket_as_master()
         else:
             try_open_socket_as_slave(self)
@@ -146,11 +162,12 @@ class Singleton:
             return fname, os.path.join(self.basepath, fname)
         return None
 
-    def check_port_file_master(self):
+    def check_existing_master_running(self):
         if self.port_file_exists():
-            prev_port = int(self.get_port_file_name()[0].split('.')[0])
-            if hasattr(self, 'port'):
-                return self.port == prev_port
+            # check if process is still running
+            pid = Path(self.get_port_file_name()[1]).read_text()
+            if pid and int(pid) != os.getpid():
+                return psutil.pid_exists(int(pid))
         return False
 
     def port_file_exists(self):
@@ -213,11 +230,11 @@ class Singleton:
             self._ctime = _ntime
 
     def err(self, msg):
-        if self.args.debug or self.args.daemon_only:
+        if self.args.debug or self.args.master_only:
             logging.error(msg)
 
     def log(self, msg):
-        if self.args.debug or self.args.daemon_only:
+        if self.args.debug or self.args.master_only:
             logging.info(msg)
 
     async def fetch(self):
@@ -225,12 +242,14 @@ class Singleton:
 
         if self.fetch_fn is not None:
             inf, jobs, avg_wait_time = await self.fetch_fn()  # a_filter_values[self.a_filter])
+            master_dead = inf is None
             self.inf = inf if inf is not None else self.inf
             self.jobs = jobs if jobs is not None else self.jobs
             self.avg_wait_time = avg_wait_time if avg_wait_time is not None else self.avg_wait_time
 
         _delta_t = time.time() - _ctime
         self.log(f"Fetch took {_delta_t:.2f} seconds")
+        return master_dead
 
     def add_button(self, y, x, width, action):
         if y not in self.mouse_state:
@@ -514,10 +533,25 @@ def get_char_async(stdscr, instance):
     os.kill(os.getpid(), signal.SIGINT)
 
 
-async def update_screen_info(stdscr, instance):
+async def update_screen_info(stdscr, instance: Singleton):
     while instance.k not in BUTTON_ACTIONS['quit']:
-        await instance.fetch()
-        instance.log("GOT DATA FROM MASTER")
+        master_dead = await instance.fetch()
+        if not master_dead:
+            instance.log("GOT DATA FROM MASTER")
+        else:
+            if not instance.check_existing_master_running():
+                random_wait_time_ms = random.randint(100, 10000)
+                instance.log("MASTER DEAD, WAITING " + str(random_wait_time_ms) + "ms")
+
+                # WARNING: Possible race-condition here, if master dies and slave tries to become master
+                await asyncio.sleep(random_wait_time_ms / 1000)
+                if not instance.check_existing_master_running():
+                    instance.log("MASTER STILL DEAD, TRYING TO BECOME MASTER")
+                    instance.setup_logging()
+                    instance.clean_port_files()
+                    instance.port = instance.create_socket_as_master()
+                    instance.is_master = True
+
         update_screen(stdscr, instance)
 
 
@@ -592,7 +626,7 @@ async def main(stdscr):
 
     def exit_handler(sig, frame):
         instance.log(f"FORCED EXIT...")
-        instance.sock.close()
+        instance.cleanup()
         exit(0)
     signal.signal(signal.SIGINT, exit_handler)