nodeocc 1.0.1__py3-none-any.whl

__init__.py

@@ -0,0 +1 @@
+name = 'nodeocc'

controller/controller.py

@@ -0,0 +1,255 @@
+import sys
+import os
+from pathlib import Path
+
+
+import numpy as np
+
+conf_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+print(conf_path)
+sys.path.append(conf_path)
+from curses import wrapper
+from view.curses_multiwindow import main, Singleton, try_open_socket_as_slave
+
+from readers.compute_wait_time import get_wait_time
+from importlib import reload
+import readers.slurmreader
+import view.slurm_list
+import view.slurm_viz
+import view.styles
+
+import time
+import json
+import argparse
+import traceback
+from model.infrastructure import Infrastructure
+from model.job import Job
+import asyncio
+import importlib.metadata
+
+program_name = 'nodeocc'
+version_number = importlib.metadata.version(program_name)
+
+# get newest version from pip
+try:
+    url = "https://pypi.python.org/pypi/nodeocc/json"
+    import requests
+    newest_version = json.loads(requests.get(url).text)['info']['version']
+    if newest_version.split('.')[0] > version_number.split('.')[0]:
+        print(f"New major version available: {newest_version} (current: {version_number})")
+        for _ in range(5):
+            print('.', end='')
+            time.sleep(0.5)
+    elif newest_version.split('.')[1] > version_number.split('.')[1]:
+        print(f"New minor version available: {newest_version} (current: {version_number})")
+        for _ in range(3):
+            print('.', end='')
+            time.sleep(0.5)
+    elif newest_version.split('.')[2] > version_number.split('.')[2]:
+        print(f"New patch version available: {newest_version} (current: {version_number})")
+        time.sleep(0.5)
+except Exception as e:
+    print(f"Could not get newest version from pip, check if {program_name} is up to date")
+    time.sleep(1)
+
+
+BEGIN_DELIM = "!{$"
+END_DELIM_ENCODED = "!}$".encode('utf-8')
+MAX_BUF = 65535
+MAX_MSG_LEN = 1024 * 1024 * 1024
+
+parser = argparse.ArgumentParser(description='Visualize slurm jobs')
+parser.add_argument('--debug', action='store_true', help='Enable logging')
+parser.add_argument('--master', action='store_true', help='Start master process')
+parser.add_argument('--daemon_only', action='store_true', help='Disable all prints - only run in background')
+parser.add_argument('--force_override', action='store_true', help='Force override of port file, kill previous instance')
+parser.add_argument('--basepath', type=str, default='/nas/softechict-nas-2/mboschini/cool_scripts/new_nodeocc/', help='Base path for nodeocc. Must be readable by all users')
+args = parser.parse_args()
+
+# check basepath is readable
+assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
+
+# export args
+Singleton.getInstance(args)
+
+last_update = os.path.getmtime(conf_path + "/view/slurm_viz.py")
+if os.path.getmtime(conf_path + "/view/slurm_list.py") > last_update:
+    last_update = os.path.getmtime(conf_path + "/view/slurm_list.py")
+if os.path.getmtime(conf_path + "/view/styles.py") > last_update:
+    last_update = os.path.getmtime(conf_path + "/view/styles.py")
+if os.path.getmtime(conf_path + "/readers/slurmreader.py") > last_update:
+    last_update = os.path.getmtime(conf_path + "/readers/slurmreader.py")
+
+
+def get_avg_wait_time(instance: Singleton):
+    try:
+        raise Exception("Not implemented")
+        avg_wait_time = get_wait_time('prod' if instance.cur_partition == 'prod' else 'students-prod')
+        h = avg_wait_time // 3600
+        m = (avg_wait_time % 3600) // 60
+        time_str = f"{int(h)}h{int(m)}m"
+        return time_str
+    except Exception as e:
+        instance.err(f"Could not get wait times")
+        instance.err(traceback.format_exc())
+        return 'err', 'err'
+
+
+def update_data_master(instance):
+    if not instance.check_port_file_master():
+        instance.log(f"Port file dead, killing current instance")
+        instance.sock.close()
+        del instance
+        exit(0)
+
+    inf = readers.slurmreader.read_infrastructure()
+    instance.timeme(f"- infrastructure load")
+
+    jobs, _ = readers.slurmreader.read_jobs()
+    instance.timeme(f"- jobs list read")
+
+    # send data to clients (if any)
+    running_jobs = [j.to_nested_dict() for j in jobs if j.state == 'CG' or j.state == 'R']
+    queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
+    queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
+    maxlen = min(120, len(running_jobs) + len(queue_jobs))
+
+    msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs[:maxlen]})
+
+    # msg to bytes
+    msg = msg.encode('utf-8')
+    msg_len = len(msg)
+    msg = (str(msg_len) + BEGIN_DELIM).encode('utf-8') + msg + END_DELIM_ENCODED
+    instance.timeme(f"- msg encoded with len {msg_len}")
+
+    instance.sock.sendto(msg, ('<broadcast>', instance.port))
+    instance.timeme(f"- broadcast")
+
+    return inf, jobs
+
+
+async def get_data_slave(instance):
+    inf, jobs = None, None
+    try:
+        # listen for data from master
+        # instance.sock.settimeout(6.5)
+        data, addr = instance.sock.recvfrom(MAX_BUF)
+
+        if BEGIN_DELIM in data.decode('utf-8'):
+            msg_len = int(data.decode('utf-8').split(BEGIN_DELIM)[0])
+            msg_len += len(str(msg_len)) + 2
+            instance.timeme(f"about to receive {msg_len} bytes")
+            while END_DELIM_ENCODED not in data:
+                buf_len = min(MAX_BUF, msg_len - len(data))
+                instance.timeme(f"bytes remaining {msg_len - len(data)} - receiving")
+                data += instance.sock.recvfrom(buf_len)[0]
+
+                if len(data) > MAX_MSG_LEN:
+                    instance.err(f"Message too long, aborting")
+                    return
+
+            data = data.split(END_DELIM_ENCODED)[0]
+            # data = data[:msg_len]
+            data = data.decode('utf-8')
+            data = data.split(BEGIN_DELIM)[1]
+
+            msg = json.loads(data)
+            inf = Infrastructure.from_dict(msg['inf'])
+            jobs = [Job.from_dict(j) for j in msg['jobs']]
+            avg_wait_time = "N/A"  # get_avg_wait_time(instance)
+            instance.timeme(f"- receive")
+        else:
+            instance.timeme(f"- no data")
+            return None, None, None, None
+
+    except BlockingIOError as e:
+        pass
+
+    except TimeoutError as e:
+        instance.log(f"TIMEOUT")
+        try_open_socket_as_slave(instance)
+
+    return inf, jobs, avg_wait_time
+
+
+async def get_all():
+    global last_update
+    # watch for reload
+    updt = os.path.getmtime(conf_path + "/view/slurm_viz.py")
+    updt = max(updt, os.path.getmtime(conf_path + "/view/slurm_list.py"))
+    updt = max(updt, os.path.getmtime(conf_path + "/view/styles.py"))
+    updt = max(updt, os.path.getmtime(conf_path + "/readers/slurmreader.py"))
+
+    instance = Singleton.getInstance()
+    instance.timeme(f"Starting update")
+
+    if updt > last_update:
+        reload(readers.slurmreader)
+        reload(view.slurm_list)
+        reload(view.slurm_viz)
+        reload(view.styles)
+        last_update = updt
+
+    instance.timeme(f"- reload")
+
+    inf, jobs, avg_wait_time = None, None, None
+    try:
+        if args.master:
+            inf, jobs = update_data_master(instance)
+        else:
+            # loop = asyncio.get_event_loop()
+            instance.timeme(f"- listening for data")
+
+            # wait for data from master but async update the view
+            inf, jobs, avg_wait_time = await get_data_slave(instance)
+    except Exception as e:
+        instance.err(f"Exception: {e}")
+        instance.err(traceback.format_exc())
+        # instance.rens = 'Something went wrong'
+        # instance.nocc = ':('
+
+    return inf, jobs, avg_wait_time
+
+
+def display_main(stdscr):
+    return asyncio.run(main(stdscr))
+
+
+def _main():
+    if args.daemon_only:
+        assert args.master, "Daemon mode only available for master"
+        instance = Singleton.getInstance()
+
+        instance.log(f"Starting master daemon")
+        # register atexit
+        import atexit
+
+        def exit_handler():
+            instance.log(f"Exiting...")
+            instance.sock.close()
+            # remove .port file
+            if instance.port_file_exists():
+                Path(instance.get_port_file_name()[1]).unlink()
+        atexit.register(exit_handler)
+
+        while True:
+            instance.timeme(f"Updating...")
+            try:
+                update_data_master(instance)
+            except Exception as e:
+                instance.err(f"Exception: {e}")
+                instance.err(traceback.format_exc())
+
+            time.sleep(5)
+
+    else:
+        # configure singleton
+
+        Singleton.getInstance().signature = f"{program_name} v{version_number}"
+        Singleton.getInstance().fetch_fn = get_all
+
+        wrapper(display_main)
+
+
+if __name__ == '__main__':
+    _main()

model/infrastructure.py

@@ -0,0 +1,95 @@
+from typing import List
+
+
+class Node:
+    """
+    A simple class modelling a node within
+    the SLURM infrastructure
+    """
+
+    def __init__(self, name, n_gpus, gpu_model, reserved, status, cpus, mem) -> None:
+        self.name = name
+        self.n_gpus = n_gpus
+        self.gpu_model = gpu_model
+        self.reserved = reserved
+        assert reserved in ('no', 'yes', 'pending')
+        self.status = status
+        self.cpus = cpus
+        self.mem = mem
+
+    def __repr__(self):
+        return f'NODE {self.name} - {self.n_gpus}x{self.gpu_model} [{self.cpus}/{self.mem/1000}] - {self.status} ({self.reserved} reserv)'
+
+
+class Maintenance:
+    """
+    A simple class modelling a pending
+    SLURM maintenance
+    """
+
+    def __init__(self, nodes, start_time, end_time) -> None:
+        self.nodes = nodes
+        self.start_time = start_time
+        self.end_time = end_time
+
+    def __repr__(self):
+        return f'RESERVATION {self.start_time} - {self.end_time} on {",".join(self.nodes)}'
+
+    @staticmethod
+    def from_dict(d):
+        m = Maintenance(None, None, None)
+        for k, v in d.items():
+            setattr(m, k, v)
+        return m
+
+    def to_nested_dict(self):
+        return {
+            k: v for k, v in self.__dict__.items()
+        }
+
+
+class Infrastructure:
+    def __init__(self, maintenances: List[Maintenance], nodes, gpu_lims, ram_lims, cpu_lims) -> None:
+        self.maintenances = maintenances
+        self.nodes = nodes
+        gpu_limit_pu, gpu_limit_grp, gpu_limit_stu, gpu_limit_stugrp = gpu_lims
+        ram_limit_pu, ram_limit_grp, ram_limit_stu, ram_limit_stugrp = ram_lims
+        cpu_limit_pu, cpu_limit_grp, cpu_limit_stu, cpu_limit_stugrp = cpu_lims
+        self.gpu_limit_pu = gpu_limit_pu
+        self.gpu_limit_grp = gpu_limit_grp
+        self.gpu_limit_stu = gpu_limit_stu
+        self.gpu_limit_stugrp = gpu_limit_stugrp
+        self.ram_limit_pu = ram_limit_pu
+        self.ram_limit_grp = ram_limit_grp
+        self.ram_limit_stu = ram_limit_stu
+        self.ram_limit_stugrp = ram_limit_stugrp
+        self.cpu_limit_pu = cpu_limit_pu
+        self.cpu_limit_grp = cpu_limit_grp
+        self.cpu_limit_stu = cpu_limit_stu
+        self.cpu_limit_stugrp = cpu_limit_stugrp
+        self.prior = ['RTX6000', '2080', 'V100', 'RTXA5000', 'RTX5000', '1080', 'P100', 'n/a', None, 'K80']
+
+    @staticmethod
+    def from_dict(d):
+        i = Infrastructure(None, None, [None] * 4, [None] * 4, [None] * 4)
+        for k, v in d.items():
+            if k == 'nodes':
+                setattr(i, k, [Node(**n) for n in v])
+            else:
+                setattr(i, k, v)
+        return i
+
+    def to_nested_dict(self):
+        return {
+            k: ([n.__dict__ for n in self.nodes] if k == 'nodes' else
+                ([n.to_nested_dict() for n in self.maintenances] if k == 'maintenances' else v))
+            for k, v in self.__dict__.items()
+        }
+
+    def __repr__(self):
+        return "INFRASTRUCTURE\n" + "\n".join([str(x) for x in self.maintenances]) + "\n\n" + "\n".join([str(x) for x in self.nodes]) + \
+            f"\n\nlimits:({self.gpu_limit_pu}:{self.gpu_limit_grp}),({self.gpu_limit_stu}:{self.gpu_limit_stugrp})"
+
+    def get_sorted_nodes(self):
+
+        return sorted(self.nodes, key=lambda x: self.prior.index(x.gpu_model) if x.gpu_model in self.prior else -1)

model/job.py

@@ -0,0 +1,75 @@
+import grp
+import os
+from pwd import getpwnam
+
+from view.curses_multiwindow import Singleton
+
+
+class Job:
+    """
+    A simple class modelling a SLURM job
+    """
+
+    def __init__(self, jobid, true_jobid, name, user, partition, state, runtime, reason, account, priority, tres, start_time, nice) -> None:
+        self.jobid = jobid
+        self.true_jobid = str(true_jobid)
+        self.name = name
+        self.user = user
+        self.partition = partition
+        self.state = state
+        self.runtime = runtime
+        self.joblets = []
+        self.reason = reason
+        self.account = account
+        self.priority = priority
+        self.tres = tres
+        self.starttime = start_time
+        self.nice = nice
+        self.user_group = grp.getgrgid(getpwnam(self.user)[3]).gr_name if self.user is not None else None
+
+    def is_student(self):
+        return self.user_group in ['studenti', 'tesisti']
+
+    @staticmethod
+    def from_dict(d):
+        j = Job(None, None, None, None, None, None, None, None, None, None, None, None, None)
+        j.joblets = []
+        for k, v in d.items():
+            if k == 'joblets':
+                setattr(j, k, [Joblet(**jblet) for jblet in v])
+            else:
+                setattr(j, k, v)
+        return j
+
+    def to_nested_dict(self):
+        return {
+            k: (v if k != 'joblets' else [n.__dict__ for n in self.joblets]) for k, v in self.__dict__.items()
+        }
+
+    def __repr__(self):
+        return f"JOB: {self.jobid} - {self.name} ({self.user}) [{self.reason}]\n" + \
+            '\n'.join(['\t' + str(x) for x in self.joblets])
+
+
+class Joblet:
+    """
+    A simple class modelling the portion
+    of a SLURM job running on a given node
+    """
+
+    def __init__(self, jobid, true_jobid, node, n_gpus, cpus, mem) -> None:
+        self.jobid = jobid
+        self.true_jobid = true_jobid
+        self.node = node.replace('ailb-login-02', '0💀') if node is not None else None
+        self.n_gpus = int(n_gpus)
+        self.mem = mem
+        self.cpus = cpus
+
+    def __repr__(self):
+        try:
+            if isinstance(self.mem, str):
+                retstr = f"JOBLET: {self.jobid} on {self.node} ({self.n_gpus} gpus, {self.cpus} cpus, {self.mem} Mmem)"
+            retstr = f"JOBLET: {self.jobid} on {self.node} ({self.n_gpus} gpus, {self.cpus} cpus, {self.mem/1024} Gmem)"
+        except Exception as e:
+            Singleton.getInstance().err(f"Exception in joblet repr: {e}")
+        return retstr

nodeocc-1.0.1.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: nodeocc
+Version: 1.0.1
+Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
+Author: Matteo Boschini
+Maintainer: Lorenzo Bonicelli, Aniello Panariello
+License: MIT
+Project-URL: Repository, https://github.com/mbosc/slurm_nodeocc
+Keywords: Slurm,GPU,TUI,CLI,HPC,cluster,allocations,nodeocc
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console :: Curses
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: numpy
+
+# Slurm NodeOCC
+![[screenshot](https://github.com/apanariello4/slurm_nodeocc/blob/master/sshot.jpg?raw=true)](sshot.jpg)
+
+TUI application for viewing the status of GPU allocations on a Slurm cluster
+
+Contents:
+
++ `model/`: folder containing the fundamental classes definitions to model the Slurm cluster, its jobs and their portions;
++ `readers/`: folder containing the functions to parse Slurm infrastructure and jobs status from standard unix commands;
++ `view/slurm_list.py`: a simple list-based viewer of queued and allocated jobs;
++ `view/slurm_viz.py`: a graphic-based viewer of allocated jobs across Slurm topology;
++ `controller/controller.py`: the main entrypoint for the curses-based interactive TUI application.
+
+Run it with: `python controller/controller.py`
+
+Requirements: pandas, pycurses.

nodeocc-1.0.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+__init__.py,sha256=LlgHaFKDsLYKxyTe8BOiLhyt4Id6f6TG9KDiSrpMU5c,17
+controller/controller.py,sha256=kewivhyKPoPfwWY3-qj_fmFyev6UrOHe0GFPwRNkyq4,8746
+model/infrastructure.py,sha256=65iMo7ajIC42S-3nhVfR7_eEbXsRSqfUz-dMdNquwWk,3339
+model/job.py,sha256=xBJd1W1ngnUZkZezEtsVSQfKbEI91e5v2kd9MLp99h0,2482
+readers/compute_wait_time.py,sha256=sUH-VKceMninrj1keQ0vSE4z_qHhDMlurdtQIdZ4bDU,804
+readers/slurmreader.py,sha256=Jc-XFylsCKIdIavmnTnKLyys6Syowl6NR328h4O5Ylw,12485
+view/__init__.py,sha256=Q-1NQu1KuD4mHNnyMxMTlhj3O7z7d8vGu7-uD_XlYsU,660
+view/curses_multiwindow.py,sha256=d_YK-W33EsRdrG1avuJUvgEgx77-aFnEiR4NeTu5kYs,21257
+view/slurm_list.py,sha256=f_sGMtIpMp5h10KO7MTyrd48qh2Abukr8I0rYFez7Iw,5657
+view/slurm_viz.py,sha256=E_FzP8D3u6fNQLOisXEg9eEfxFrjBWtD6DFHaMHFhZE,24809
+view/styles.py,sha256=nYRtfVQXwowGk5AdIFP3_zD1NT1fBMQ8_GM_j0SdxaQ,2812
+view/utils.py,sha256=Ze8WVi7OWjTJ1FTHy66Zg5eKnlBE72Pxirv5vC8aXeI,1834
+nodeocc-1.0.1.dist-info/METADATA,sha256=LFW8a-XlJegL9KxJw-1TObO5bAg2YLseSPLy23JNyUY,1395
+nodeocc-1.0.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+nodeocc-1.0.1.dist-info/entry_points.txt,sha256=7dTNBQ0rS2Weve3sFUIWaQksCsIJxHVX3ey2M7mCCwg,56
+nodeocc-1.0.1.dist-info/top_level.txt,sha256=Kth-SFY-5rHDYYus3pnUwfB9CoHeREptSPhVLJLvRaY,39
+nodeocc-1.0.1.dist-info/RECORD,,

nodeocc-1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.42.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nodeocc-1.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nodeocc = controller.controller:_main

nodeocc-1.0.1.dist-info/top_level.txt

@@ -0,0 +1,5 @@
+__init__
+controller
+model
+readers
+view

readers/compute_wait_time.py

@@ -0,0 +1,28 @@
+import os
+import numpy as np
+
+def get_sec(time_str):
+    days_chks = time_str.split('-')
+    if len(days_chks) > 1:
+        days = days_chks[0]
+        time_str = days_chks[1]
+    else:
+        days = 0
+    h, m, s = time_str.split(':')
+    return int(days) * 86400 + int(h) * 3600 + int(m) * 60 + int(s)
+
+def get_wait_time(partition):
+    assert partition in ['prod', 'students-prod']
+    cmd = 'sacct --noheader -X -a --partition=%s --start=now-1weeks --format="Reserved"' % partition
+
+    output = os.popen(cmd).readlines()
+    wait_times = []
+
+    for line in output:
+        if ':' not in line:
+            continue
+        wait_times.append(get_sec(line.strip()))
+
+    wait_times = np.asarray(wait_times)
+    avg_wait_time = np.mean(wait_times) if len(wait_times)>0 else -1
+    return avg_wait_time