nodeocc 1.0.12__tar.gz → 1.0.14__tar.gz

{nodeocc-1.0.12 → nodeocc-1.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.12
+Version: 1.0.14
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.12 → nodeocc-1.0.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.12"
+version = "1.0.14"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.12 → nodeocc-1.0.14}/src/controller/controller.py

@@ -38,15 +38,15 @@ try:
     import requests
     newest_version = json.loads(requests.get(url).text)['info']['version']
     print(BBLUE, end='')
-    if newest_version.split('.')[0] > version_number.split('.')[0]:
+    if int(newest_version.split('.')[0]) > int(version_number.split('.')[0]):
         print(f"New major version available: {newest_version} (current: {version_number})")
         print(f"Please update with 'pip install {program_name} --upgrade'")
         time.sleep(7)
-    elif newest_version.split('.')[1] > version_number.split('.')[1]:
+    elif int(newest_version.split('.')[1]) > int(version_number.split('.')[1]):
         print(f"New minor version available: {newest_version} (current: {version_number})")
         print(f"Please update with 'pip install {program_name} --upgrade'")
         time.sleep(7)
-    elif newest_version.split('.')[2] > version_number.split('.')[2]:
+    elif int(newest_version.split('.')[2]) > int(version_number.split('.')[2]):
         print(f"New patch version available: {newest_version} (current: {version_number})")
         print(f"Please update with 'pip install {program_name} --upgrade'")
         time.sleep(7)
@@ -70,7 +70,6 @@ JOB_LIMIT_PER_MSG = 60
 parser = argparse.ArgumentParser(description='Visualize slurm jobs')
 parser.add_argument('--debug', action='store_true', help='Enable logging')
 parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
-parser.add_argument('--force_override', action='store_true', help='Force override of port file, kill previous instance')
 parser.add_argument('--basepath', type=str, default='/nas/softechict-nas-2/mboschini/cool_scripts/new_nodeocc/', help='Base path for nodeocc. Must be readable by all users')
 args = parser.parse_args()
 
@@ -123,7 +122,6 @@ def send_msg(instance: Singleton, msg, n_extra_msgs=0, is_extra=False):
 def update_data_master(instance: Singleton):
     if instance.check_existing_master_running():
         instance.log(f"Master already running, killing current instance")
-        instance.cleanup()
         exit(0)
 
     inf = readers.slurmreader.read_infrastructure()
@@ -138,6 +136,7 @@ def update_data_master(instance: Singleton):
     queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
     maxlen = min(JOB_LIMIT_PER_MSG, len(running_jobs) + len(queue_jobs))
 
+
     queue_jobs, extra_queue_jobs = queue_jobs[:maxlen - len(running_jobs)], queue_jobs[maxlen - len(running_jobs):]
     cur_timestamp = time.time()
     msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs, 'ts': str(cur_timestamp)})
@@ -196,7 +195,7 @@ async def get_data_slave(instance):
                 # check if timestamp is the same
                 if float(new_ts['ts']) != orig_timestamp:
                     instance.timeme(f"Timestamp mismatch, skipping")
-                    return inf, jobs, avg_wait_time  # return what we have
+                    return inf, jobs, avg_wait_time # return what we have
 
                 msg_store[msg_idx] += new_jobs
 
@@ -271,16 +270,6 @@ def _main():
         assert orig_instance.is_master, "Instance is not `master`, run with `force_override` to override"
 
         orig_instance.log(f"Starting master daemon")
-        # register atexit
-        import atexit
-
-        def exit_handler():
-            orig_instance.log(f"Exiting...")
-            orig_instance.sock.close()
-            # remove .port file
-            if orig_instance.port_file_exists():
-                Path(orig_instance.get_port_file_name()[1]).unlink()
-        atexit.register(exit_handler)
 
         while True:
             orig_instance.timeme(f"Updating...")
@@ -306,5 +295,4 @@ def _main():
 
 _main()
 
-orig_instance.cleanup()
 exit(0)

{nodeocc-1.0.12 → nodeocc-1.0.14}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nodeocc
-Version: 1.0.12
+Version: 1.0.14
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.12 → nodeocc-1.0.14}/src/view/curses_multiwindow.py

@@ -45,10 +45,12 @@ def is_file_readable_byall(file_path):
 
 
 def try_open_socket_as_slave(instance):
-    if not instance.port_file_exists():
+    if len(instance.get_port_files()) == 0:
         raise Exception("No master running")
 
-    cur_port = int(instance.get_port_file_name()[0].split('.')[0])
+    cur_port = [int(f.split('.')[0].split('master_')[1]) for f in instance.get_port_files()
+                if (pid := Path(f).read_text()) and psutil.pid_exists(int(pid))][0]
+
     if hasattr(instance, 'port'):
         # check if port file has same name
         if instance.port != cur_port:
@@ -76,14 +78,6 @@ class Singleton:
             Singleton.__instance.args = args
         return Singleton.__instance
 
-    # destructor
-    def cleanup(self):
-        if self.is_master:
-            if self.port_file_exists():
-                os.system(f"rm {self.get_port_file_name()[1]}")
-        if hasattr(self, 'sock') and self.sock is not None:
-            self.sock.close()
-
     def setup_logging(self):
         if self.args.master_only:
             handler = RotatingFileHandler(os.path.join(self.basepath, '.master_log.txt'), maxBytes=5 * 1024 * 1024, backupCount=2, mode='w')
@@ -97,9 +91,18 @@ class Singleton:
                                 handlers=[handler])
 
     def clean_port_files(self):
+        nodename = socket.getfqdn().split('.')[0]
         for f in os.listdir(self.basepath):
-            if f.endswith('.port') and is_file_writable_byall(os.path.join(self.basepath, f)):
+            if nodename in f and 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(os.path.join(self.basepath, f)):
+                # read pid from file
+                pid = Path(os.path.join(self.basepath, f)).read_text()
+                # assert no process is running
+                if psutil.pid_exists(int(pid)):
+                    self.log(f"Process {pid} is still running, killing self")
+                    return False
+
                 os.system(f"rm {os.path.join(self.basepath, f)}")
+        return True
 
     def __init__(self, args=None):
         """ Virtually private constructor. """
@@ -140,60 +143,36 @@ class Singleton:
 
         self.basepath = self.args.basepath
 
-        # check if any .port file exists
-        if args.force_override and self.port_file_exists():
-            _, port_filepath = self.get_port_file_name()
-
-            # read pid from file
-            pid = Path(port_filepath).read_text()
-            # kill process
-            try:
-                os.kill(int(pid), signal.SIGTERM)
-            except Exception as e:
-                self.log("Could not kill process, deleting port file")
-                self.log(e)
-
-            # delete port file
-            Path(port_filepath).unlink()
-
         self.is_master = not self.check_existing_master_running()
 
         if self.is_master:
-            # clean up old port files
-            self.clean_port_files()
             self.port = self.create_socket_as_master()
         else:
             try_open_socket_as_slave(self)
 
-    def get_port_file_name(self):
-        if self.port_file_exists():
-            portfiles = [f for f in os.listdir(self.basepath) if f.endswith('.port')]
-            portfiles = [(f, os.path.join(self.basepath, f)) for f in portfiles]
-            portfiles = [(fname, fpath) for fname, fpath in portfiles if is_file_readable_byall(fpath) and is_file_writable_byall(fpath)][0]
-
-            return portfiles[0], portfiles[1]
-        return None
-
     def check_existing_master_running(self):
-        if self.port_file_exists():
-            # check if process is still running
-            pid = Path(self.get_port_file_name()[1]).read_text()
-            if pid and int(pid) != os.getpid():
-                return psutil.pid_exists(int(pid))
+        portfiles = self.get_port_files()
+        if len(portfiles) > 0:
+            for f in portfiles:
+                # check if process is still running
+                pid = Path(f).read_text()
+                if pid and int(pid) != os.getpid():
+                    return psutil.pid_exists(int(pid))
         return False
 
-    def port_file_exists(self):
+    def get_port_files(self):
         """
         Check if a port file exists in the basepath and the file has 666 permissions
         """
-        portfiles = [f for f in os.listdir(self.basepath) if f.endswith('.port')]
-        if len(portfiles) == 0:
-            return False
-        portfiles = [os.path.join(self.basepath, f) for f in portfiles]
+        nodename = socket.getfqdn().split('.')[0]
+        portfiles = [os.path.join(self.basepath, f) for f in os.listdir(self.basepath)
+                     if f.endswith('.portfile') and 'master_' in f and nodename in f]
         portfiles = [f for f in portfiles if is_file_readable_byall(f) and is_file_writable_byall(f)]
-        return len(portfiles) > 0
+        return portfiles
 
     def create_socket_as_master(self):
+        nodename = socket.getfqdn().split('.')[0]
+
         # create udp socket for broadcasting
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
 
@@ -211,10 +190,26 @@ class Singleton:
         # get pid of current process
         self.pid = os.getpid()
 
-        # create file to store port with 666 permissions to file
-        filepath = Path(self.basepath, f'{self.port}.port')
-        filepath.write_text(str(self.pid))
-        os.chmod(filepath, 0o666)
+        # clean up old port files
+        kill_ok = self.clean_port_files()
+        if not kill_ok:
+            return None
+
+        try:
+            # create file to store port with 666 permissions to file
+            filepath = Path(self.basepath, f'{nodename}_master_{self.port}.portfile')
+            filepath.write_text(str(self.pid))
+            os.chmod(filepath, 0o666)
+
+            self.port_filepath = filepath
+        except Exception as e:
+            self.err(f"Could not create port file: {e}")
+            self.err(traceback.format_exc())
+            # check if file exists
+            if filepath.exists():
+                # delete file
+                os.system(f"rm {filepath}")
+            return None
 
         return self.port
 
@@ -570,9 +565,12 @@ async def update_screen_info(stdscr, instance: Singleton):
                 if not instance.check_existing_master_running():
                     instance.log("MASTER STILL DEAD, TRYING TO BECOME MASTER")
                     instance.setup_logging()
-                    instance.clean_port_files()
                     instance.port = instance.create_socket_as_master()
                     instance.is_master = True
+                else:
+                    try_open_socket_as_slave(instance)
+            else:
+                try_open_socket_as_slave(instance)
 
         update_screen(stdscr, instance)
 
@@ -648,7 +646,13 @@ async def main(stdscr):
 
     def exit_handler(sig, frame):
         instance.log(f"FORCED EXIT...")
-        instance.cleanup()
+        # instance.cleanup()
+        if hasattr(instance, 'port_filepath'):
+            os.system(f"rm {instance.port_filepath}")
+
+        if hasattr(instance, 'sock'):
+            instance.sock.close()
+
         exit(0)
     signal.signal(signal.SIGINT, exit_handler)