nodeocc 1.0.23__tar.gz → 1.0.24__tar.gz

{nodeocc-1.0.23 → nodeocc-1.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nodeocc
-Version: 1.0.23
+Version: 1.0.24
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.23 → nodeocc-1.0.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.23"
+version = "1.0.24"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.23 → nodeocc-1.0.24}/src/controller/controller.py

@@ -1,3 +1,4 @@
+import random
 import sys
 import os
 from pathlib import Path
@@ -64,11 +65,13 @@ BEGIN_DELIM = "!{$"
 END_DELIM_ENCODED = "!}$".encode('utf-8')
 EXTRA_MSG_BEGIN_DELIM = "!{#"
 MAX_BUF = 65535
-MAX_MSG_LEN = 1024 * 1024 * 1024
-JOB_LIMIT_PER_MSG = 60
+JOB_LIMIT_PER_MSG = 40
+TIMEOUT_MAX = 3
+n_timeout = 0
 
 parser = argparse.ArgumentParser(description='Visualize slurm jobs')
 parser.add_argument('--debug', action='store_true', help='Enable logging')
+parser.add_argument('--override', action='store_true', help='Enable logging')
 parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
 parser.add_argument('--basepath', type=str, default='/tmp', help='Base path for nodeocc. Must be readable by all users')
 args = parser.parse_args()
@@ -168,6 +171,7 @@ def decode_msg_slave(data, delim=BEGIN_DELIM):
 
 
 async def get_data_slave(instance):
+    global n_timeout
     inf, jobs, avg_wait_time = None, None, None
     try:
         # listen for data from master
@@ -210,12 +214,18 @@ async def get_data_slave(instance):
             instance.timeme(f"- no data")
             return None, None, None, None
 
+        n_timeout = 0
+
     except BlockingIOError as e:
         pass
 
     except TimeoutError as e:
-        instance.log(f"TIMEOUT")
-        try_open_socket_as_slave(instance)
+        n_timeout += 1
+        instance.log(f"TIMEOUT {n_timeout}/{TIMEOUT_MAX}")
+        if n_timeout > TIMEOUT_MAX:
+            try_open_socket_as_slave(instance, force=True)
+
+        time.sleep(random.randint(100, 10000) / 1000)
 
     return inf, jobs, avg_wait_time
 

{nodeocc-1.0.23 → nodeocc-1.0.24}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nodeocc
-Version: 1.0.23
+Version: 1.0.24
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.23 → nodeocc-1.0.24}/src/view/curses_multiwindow.py

@@ -44,7 +44,7 @@ def is_file_readable_byall(file_path):
     return os.stat(file_path).st_mode & stat.S_IROTH
 
 
-def try_open_socket_as_slave(instance):
+def try_open_socket_as_slave(instance, force=False):
     if len(instance.get_port_files()) == 0:
         raise Exception("No master running")
 
@@ -57,7 +57,7 @@ def try_open_socket_as_slave(instance):
             instance.log(f"- MASTER HAS CHANGED PORT: {instance.port} vs {cur_port}")
 
     instance.port = cur_port
-    instance._open_socket_as_slave(instance.port)
+    instance._open_socket_as_slave(instance.port, force=force)
 
     if instance.try_open_counter > 5:
         instance.err(f"Could not open socket as slave on port {instance.port}")
@@ -90,7 +90,7 @@ class Singleton:
             logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
                                 handlers=[handler])
 
-    def clean_port_files(self):
+    def clean_port_files(self, force=False):
         nodename = socket.getfqdn().split('.')[0]
         bp = os.path.join(self.basepath, nodename)
         if not os.path.exists(bp):
@@ -100,7 +100,7 @@ class Singleton:
                 # read pid from file
                 pid = Path(os.path.join(bp, f)).read_text()
                 # assert no process is running
-                if psutil.pid_exists(int(pid)):
+                if psutil.pid_exists(int(pid)) and not force:
                     self.log(f"Process {pid} is still running, killing self")
                     return False
 
@@ -148,8 +148,8 @@ class Singleton:
 
         self.is_master = not self.check_existing_master_running()
 
-        if self.is_master:
-            self.port = self.create_socket_as_master()
+        if self.is_master or args.override:
+            self.port = self.create_socket_as_master(force=args.override)
         else:
             try_open_socket_as_slave(self)
 
@@ -176,28 +176,14 @@ class Singleton:
         portfiles = [f for f in portfiles if is_file_readable_byall(f) and is_file_writable_byall(f)]
         return portfiles
 
-    def create_socket_as_master(self):
+    def update_port_file(self, force=False):
         nodename = socket.getfqdn().split('.')[0]
 
-        # create udp socket for broadcasting
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
-
-        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-
-        # Enable broadcasting mode
-        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
-
-        self.sock.bind(('', 0))
-
-        self.port = self.sock.getsockname()[1]
-        self.sock.settimeout(2)
-        self.log(f"Socket created as master on port {self.port}")
-
         # get pid of current process
         self.pid = os.getpid()
 
         # clean up old port files
-        kill_ok = self.clean_port_files()
+        kill_ok = self.clean_port_files(force=force)
         if not kill_ok:
             return None
 
@@ -228,9 +214,26 @@ class Singleton:
                 os.system(f"rm {filepath}")
             return None
 
+    def create_socket_as_master(self, force=False):
+        # create udp socket for broadcasting
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
+
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        # Enable broadcasting mode
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+
+        self.sock.bind(('', 0))
+
+        self.port = self.sock.getsockname()[1]
+        self.sock.settimeout(2)
+        self.log(f"Socket created as master on port {self.port}")
+
+        self.update_port_file(force=force)
+
         return self.port
 
-    def _open_socket_as_slave(self, port):
+    def _open_socket_as_slave(self, port, force=False):
         try:
             # create udp socket for broadcasting
             self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
@@ -245,6 +248,8 @@ class Singleton:
             self.sock.settimeout(6.5)
             self.log(f"Socket opened on port {self.port}")
 
+            self.update_port_file(force=force)
+
             self.try_open_counter = 0
             return self.port
         except Exception as e: