hpc-nodes 1.8.3__tar.gz

hpc_nodes-1.8.3/MANIFEST.in

@@ -0,0 +1,8 @@
+include nodes.spec
+include nodes.conf
+include nodes.conf.sample
+include MANIFEST.in
+include nodes.1
+include nodes.conf.5
+include README
+include VERSION.file

hpc_nodes-1.8.3/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: hpc-nodes
+Version: 1.8.3
+Summary: Handle hierarchical groups of nodes
+Author-email: Kent Engström <kent@nsc.liu.se>
+License-Expression: GPL-2.0
+Project-URL: homepage, http://www.nsc.liu.se/~kent/nodes/
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Systems Administration
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Requires-Dist: python-hostlist

hpc_nodes-1.8.3/README

@@ -0,0 +1 @@
+Handle hierarchical groups of nodes in high performance computing cluster, such as enclosures and racks.

hpc_nodes-1.8.3/VERSION.file

@@ -0,0 +1 @@
+1.8.3

hpc_nodes-1.8.3/hpc_nodes.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: hpc-nodes
+Version: 1.8.3
+Summary: Handle hierarchical groups of nodes
+Author-email: Kent Engström <kent@nsc.liu.se>
+License-Expression: GPL-2.0
+Project-URL: homepage, http://www.nsc.liu.se/~kent/nodes/
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Systems Administration
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Requires-Dist: python-hostlist

hpc_nodes-1.8.3/hpc_nodes.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+MANIFEST.in
+README
+VERSION.file
+nodes
+nodes.1
+nodes.conf
+nodes.conf.5
+nodes.conf.sample
+nodes.py
+nodes.spec
+pyproject.toml
+setup.py
+hpc_nodes.egg-info/PKG-INFO
+hpc_nodes.egg-info/SOURCES.txt
+hpc_nodes.egg-info/dependency_links.txt
+hpc_nodes.egg-info/requires.txt
+hpc_nodes.egg-info/top_level.txt
+test/test_nodes.py

hpc_nodes-1.8.3/hpc_nodes.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hpc_nodes-1.8.3/hpc_nodes.egg-info/requires.txt

@@ -0,0 +1 @@
+python-hostlist

hpc_nodes-1.8.3/hpc_nodes.egg-info/top_level.txt

@@ -0,0 +1 @@
+nodes

hpc_nodes-1.8.3/nodes

@@ -0,0 +1,134 @@
+#!/usr/bin/python3
+# Version 1.8.3
+import sys
+import optparse
+from nodes import Hierarchy, NodesException, numerically_sorted_entities, collect_name_hostlist
+
+# MAIN
+
+op = optparse.OptionParser(usage="usage: %prog [OPTION]... NODESPEC")
+op.add_option("--to-nodes",
+              action="store_true",
+              help="convert NODESPEC to nodes (default)")
+op.add_option("-u", "--up",
+              action="store",
+              metavar="LEVEL",
+              help="list groups where any node in NODESPEC is a member")
+op.add_option("-f", "--fill",
+              action="store",
+              metavar="LEVEL",
+              help="as --up, but then convert to nodes")
+op.add_option("-r", "--representative",
+              action="store",
+              metavar="LEVEL",
+              help="as --up, but then convert to a representative node per group")
+op.add_option("-g", "--gather",
+              action="store",
+              metavar="LEVEL",
+              help="list groups whose members are all in NODESPEC plus the individual nodes that are left over")
+op.add_option("-i", "--index",
+              action="store",
+              metavar="LEVEL",
+              help="list the nodes together with their index in their group at the specified LEVEL (--expand is implied)")
+op.add_option("-I", "--index-separator",
+              action="store", type="string",  default=": ",
+              metavar="SEPARATOR",
+              help="separator to use between node and index in --index (default is ': ')")
+op.add_option("--convert-to-clustershell",
+              action="store_true",
+              help="convert to ClusterShell YAML")
+op.add_option("-m", "--missing",
+              action="store",
+              metavar="GROUP",
+              help="when doing --up, pretend that nodes not in any group belong to GROUP")
+op.add_option("-e", "--expand",
+              action="store_true",
+              help="expand to list of names instead of collecting a hostlist")
+op.add_option("--separator",
+              action="store", type="string",  default="\n",
+              help="separator to use between nodes when outputting an expanded list (default is newline)")
+op.add_option("--prepend",
+              action="store", type="string",  default="",
+              help="string to prepend to each node when outputting an expanded list")
+op.add_option("--append",
+              action="store", type="string",  default="",
+              help="string to append to each node when outputting an expanded list")
+op.add_option("-n", "--count",
+              action="store_true",
+              help="output the number of names instead of the hostlist or expanded list")
+op.add_option("-s", "--slurm",
+              action="store_true",
+              help="create dynamic groups for SLURM jobs and users")
+op.add_option("--config-file",
+              action="store",
+              default="/etc/nodes.conf",
+              metavar="FILE",
+              help="use FILE instead of /etc/nodes.conf")
+
+(opts, args) = op.parse_args()
+
+h = Hierarchy()
+try:
+    h.parse_file(opts.config_file)
+except NodesException as e:
+    print()
+    sys.stderr.write("Config file error: %s\n" % e.msg)
+    sys.exit(1)
+
+if opts.slurm:
+    try:
+        h.parse_slurm()
+    except NodesException as e:
+        print()
+        sys.stderr.write("SLURM error: %s\n" % e.msg)
+        sys.exit(1)
+
+
+if not opts.convert_to_clustershell:
+    if len(args) != 1:
+        sys.exit("Error: You need to provide a single node specifier!\n")
+        sys.exit(1)
+    else:
+        arg = args[0]
+
+modes_chosen = int(opts.to_nodes is not None) + \
+    int(opts.up is not None) + \
+    int(opts.fill is not None) + \
+    int(opts.representative is not None) + \
+    int(opts.gather is not None) + \
+    int(opts.index is not None) + \
+    int(opts.convert_to_clustershell is not None)
+
+if modes_chosen > 1:
+    sys.exit("Error: You cannot choose more than one mode at a time!\n")
+    sys.exit(1)
+elif modes_chosen == 0 or opts.to_nodes:
+    func = lambda a: h.to_nodes(a)
+else:
+    if opts.up:
+        func = lambda arg: h.up(opts.up, arg, missing_group = opts.missing)
+    elif opts.fill:
+        func = lambda arg: h.fill(opts.fill, arg)
+    elif opts.representative:
+        func = lambda arg: h.representative(opts.representative, arg)
+    elif opts.gather:
+        func = lambda arg: h.gather(opts.gather, arg)
+    elif opts.index:
+        func = lambda arg: h.index(opts.index, arg, separator = opts.index_separator)
+    elif opts.convert_to_clustershell:
+        h.convert_to_clustershell()
+        sys.exit(0)
+
+try:
+    res = func(arg)
+    if opts.count:
+        print(len(res))
+    elif opts.expand or opts.index:
+        print(opts.separator.join([opts.prepend + x.name + opts.append for x in numerically_sorted_entities(res)]))
+    else:
+        print(collect_name_hostlist(res))
+
+except NodesException as e:
+    print()
+    sys.stderr.write("Error: %s\n" % e.msg)
+    sys.exit(1)

hpc_nodes-1.8.3/nodes.1

@@ -0,0 +1,201 @@
+.TH nodes 1 "Version 1.8.3"
+.SH NAME
+nodes \- handle hierarchical groups of nodes
+.SH SYNOPSIS
+.B nodes
+.RI [ OPTION "]... " NODESPEC
+.SH DESCRIPTION
+Expand
+.I NODESPEC
+to a set of nodes and then, if specified by the options,
+collect them into groups.
+
+A nodespec follows the same syntax as a LLNL hostlist, but may contain
+names of groups that refer to several nodes. For example, if nodes
+n[1-4] make up enclosure e1, the group name e1 may refer to all
+four nodes n[1-4]. The group hierarchy is defined in the
+.B /etc/nodes.conf
+configuration file.
+
+A group is defined to be at a specific level, which has a name like
+"enclosure", "rack" etc. In the options below where a level name is required,
+any unambiguous prefix of the level name may be used. 
+
+.SH OPTIONS
+.TP
+.B --to-nodes
+Expand
+.I NODESPEC
+to a set of nodes and collect that into a hostlist that will only contain
+node names, not group names. This is the default operation.
+.TP
+.BI "-u " LEVEL ", --up=" LEVEL
+Expand
+.I NODESPEC
+to a set of nodes and then collect the nodes into groups at the
+specified
+.I LEVEL
+and return the resulting nodespec. A group is included in the result
+if any of its members is present in the initial nodespec.
+.TP
+.BI "-f " LEVEL ", --fill=" LEVEL
+Expand
+.I NODESPEC
+to a set of nodes and then collect the nodes into groups at the
+specified
+.I LEVEL,
+just like for 
+.B -u/--up
+above. Then expand the resulting nodespec again to a hostlist
+that will only contain node names.
+.TP
+.BI "-r " LEVEL ", --representative=" LEVEL
+Expand
+.I NODESPEC
+to a set of nodes and then collect the nodes into groups at the
+specified
+.I LEVEL,
+just like for 
+.B -u/--up
+above. Then expand the resulting nodespec again to a hostlist that
+contains one representative node (the first in lexicographic
+order) from each group.
+.TP
+.BI "-g " LEVEL ", --gather=" LEVEL
+Expand
+.I NODESPEC
+to a set of nodes and then collect the nodes into groups at the
+specified
+.I LEVEL
+and individual nodes, so that the resulting nodespec will 
+refer to exactly the same set of nodes as the initial one.
+Groups are only present in the resulting nodespec when all of
+their members are present in the initial nodespec.
+.TP
+.BI "-i " LEVEL ", --index=" LEVEL
+Expand
+.I NODESPEC
+to a set of nodes and then for each node add ": " and the index
+of the node in its group at 
+.IR LEVEL .
+The index is 0 for the first node in the group (in lexicographic order),
+1 for the next one, etc. This option implies
+.BR --expand .
+.TP
+.BI "-I " SEPARATOR ", --index-separator=" SEPARATOR
+Use
+.I SEPARATOR
+as the separator between node and index in the
+.B index
+list. The default is ": ".
+.TP
+.BI "-m " GROUP ", --missing=" GROUP
+When using
+.BR "-u/--up" ,
+pretend that nodes that do not belong to any group at the given LEVEL
+do belong to GROUP. If this option is not used, an error will be
+signaled in that situation.
+.TP
+.B -e, --expand
+Instead of collecting the names (nodes and/or groups) to a hostlist,
+output them as an expanded list of names.
+.TP
+.BI "--separator=" SEPARATOR
+Use
+.I SEPARATOR
+as the separator between the nodes in the expanded list.
+The default is a newline.
+.TP
+.BI "--prepend=" PREPEND
+Output
+.I PREPEND
+before each node in the expanded list.
+The default is to prepend nothing.
+.TP
+.BI "--append=" APPEND
+Output
+.I APPEND
+after each node in the expanded list.
+The default is to append nothing.
+.TP
+.B -n, --count
+Output the number of names (nodes and/or groups) in the result instead of the list itself.
+.TP
+.B -s, --slurm
+Invoke the 
+.B /usr/bin/squeue
+command of the SLURM resource manager and use the output to build
+dynamic groups for users and jobs. The user groups use level
+.B user
+and have group names equal to the user names . The job groups use level
+.B job
+and have the job numbers prefixed by "job" as group names.
+.TP
+.BI --config-file= FILE
+Read configuration from
+.I FILE
+instead of
+.B /etc/nodes.conf.
+.TP
+.B --convert-to-clustershell
+Output configuration suitable for a YAML file in
+.B /etc/clustershell/groups.d
+when using ClusterShell 1.7 or later.
+.TP
+.B -h, --help
+Show brief usage information and exit.
+.SH EXAMPLES
+The examples below assume a configuration where each enclosure
+contains four nodes, such as the the example in the
+.I nodes.conf(5)
+man page.
+.TP
+What nodes belong to enclosures e2 and e3?
+.B % nodes e[2-3]
+.br
+n[5-12]
+.TP
+What enclosures contain the nodes n[8-10]?
+.B % nodes --up enclosure n[8-10]
+.br
+e[2-3]
+.TP
+What nodes are in the same enclosures as n[8-10]?
+.B % nodes --fill enclosure n[8-10]
+.br
+n[5-12]
+.TP
+This is to much to type! Can I abbreviate?
+.B % nodes -fe n[8-10]
+.br
+n[5-12]
+.TP
+Can I have a hostlist with one representative node from each enclosure that has nodes from the set n[8-10]?
+.B % nodes --representative enclosure n[8-10]
+.br
+n[5,9]
+.TP
+Express n[1-10] as full enclosures and remaining nodes:
+.B % nodes --gather enclosure n[1-10]
+.br
+e[1-2],n[9-10]
+.SH AUTHOR
+Written by Kent Engström <kent@nsc.liu.se>.
+
+The program is published at http://www.nsc.liu.se/~kent/nodes/
+.SH SEE ALSO
+.I nodes.conf(5), hostlist(1)
+
+The hostlist expression syntax is used by several programs developed at 
+.B LLNL
+(https://computing.llnl.gov/linux/), for example
+.B SLURM
+(https://computing.llnl.gov/linux/slurm/) and 
+.B Pdsh
+(https://computing.llnl.gov/linux/pdsh.html).
+
+See the
+.B HOSTLIST EXPRESSIONS
+section of the
+.BR pdsh (1)
+manual page for a short introduction to the hostlist syntax.

hpc_nodes-1.8.3/nodes.conf

@@ -0,0 +1,28 @@
+# Sample config file
+
+# We have to define nodes first, so that misspelled groups won't be
+# treaded as node names:
+
+nodes: n[1-64]
+
+# Now, we define groups
+
+enclosure e1: n[1-4]
+enclosure e2: n[5-8]
+enclosure e3: n[9-12]
+enclosure e4: n[13-16]
+enclosure e5: n[17-20]
+enclosure e6: n[21-24]
+enclosure e7: n[25-28]
+enclosure e8: n[29-32]
+enclosure e9: n[33-36]
+enclosure e10: n[37-40]
+enclosure e11: n[41-44]
+enclosure e12: n[45-48]
+enclosure e13: n[49-52]
+enclosure e14: n[53-56]
+enclosure e15: n[57-60]
+enclosure e16: n[61-64]
+
+rack rack1: e[1-8]
+rack rack2: e[9-16]

hpc_nodes-1.8.3/nodes.conf.5

@@ -0,0 +1,51 @@
+.TH nodes.conf 5 "Version 1.8.3"
+.SH NAME
+nodes.conf \- definition of nodes and groups for the
+.I nodes(1)
+command
+.SH SYNOPSIS
+.B /etc/nodes.conf
+.SH DESCRIPTION
+This file defines the nodes and groups known to the
+.I nodes(1)
+command.
+
+Definitions have to start at the beginning of a line, with no
+prepended whitespace.  An indented line following a definition is
+assumed to contain a nodespec. The nodes it refers to are added to
+the group being defined.
+
+Blank lines are ignored. A # sign starts a comment that
+causes the rest of the line to be ignored.
+
+Before groups are defined, valid node names have to be defined
+with a line that starts with "nodes:" and then contains a hostlist
+listing all valid node names.
+
+A group is defined by a line containing the level name, whitespace,
+the group name, colon, whitespace and then a nodespec listing the
+members of the group.
+
+A group name has to be unique. The same group name cannot be used at
+several levels. A nodespec can only refer to nodes that have already
+been defined.
+.SH EXAMPLES
+The example below defines sixteen nodes n[1-16] belonging
+to four enclosures e[1-4]. Nodes n[1-8] belong to rack r1,
+while nodes n[9-12] belong to rack r2. We define enclosure e4
+in an odd way just to show how multiple nodespecs can be used.
+.nf
+\fB
+nodes: n[1-16]
+enclosure e1: n[1-4]
+enclosure e2: n[5-8]
+enclosure e3: n[9-12]
+enclosure e4: n13
+	n14
+	n15,n16
+rack r1: e[1-2]
+rack r2: e[3-4]
+\fR
+.fi
+.SH SEE ALSO
+nodes(1)

hpc_nodes-1.8.3/nodes.conf.sample

@@ -0,0 +1,28 @@
+# Sample config file
+
+# We have to define nodes first, so that misspelled groups won't be
+# treaded as node names:
+
+nodes: n[1-64]
+
+# Now, we define groups
+
+enclosure e1: n[1-4]
+enclosure e2: n[5-8]
+enclosure e3: n[9-12]
+enclosure e4: n[13-16]
+enclosure e5: n[17-20]
+enclosure e6: n[21-24]
+enclosure e7: n[25-28]
+enclosure e8: n[29-32]
+enclosure e9: n[33-36]
+enclosure e10: n[37-40]
+enclosure e11: n[41-44]
+enclosure e12: n[45-48]
+enclosure e13: n[49-52]
+enclosure e14: n[53-56]
+enclosure e15: n[57-60]
+enclosure e16: n[61-64]
+
+rack rack1: e[1-8]
+rack rack2: e[9-16]

hpc_nodes-1.8.3/nodes.py

@@ -0,0 +1,376 @@
+# Version 1.8.3
+
+import re
+import sys
+from hostlist import expand_hostlist, collect_hostlist
+import subprocess
+
+# Exceptions
+
+class NodesException(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+class UnknownName(NodesException): pass
+class DuplicatedGroup(NodesException): pass
+class BadConfigSyntax(NodesException): pass
+class MissingGroup(NodesException): pass
+class DuplicateMembership(NodesException): pass
+class UnknownLevel(NodesException): pass
+class AmbiguousLevel(NodesException): pass
+class FailedSLURM(NodesException): pass
+
+# Helper functions
+
+def collect_name_hostlist(node_iterable):
+    return collect_hostlist([node.name for node in node_iterable])
+
+# Sort a list of entities numerically
+
+def numerically_sorted_entities (l):
+    """Sort a list of entities numerically.
+
+    E.g. sorted order should be n1, n2, n10; not n1, n10, n2.
+    """
+
+    return sorted(l, key=entity_numeric_sort_key)
+
+nsk_re = re.compile("([0-9]+)|([^0-9]+)")
+def entity_numeric_sort_key(x):
+    return [handle_int_nonint(i_ni) for i_ni in nsk_re.findall(x.name)]
+
+def handle_int_nonint(int_nonint_tuple):
+    if int_nonint_tuple[0]:
+        return int(int_nonint_tuple[0])
+    else:
+        return int_nonint_tuple[1]
+
+
+# Classes
+
+class Entity:
+    pass
+
+class Node(Entity):
+    def __init__(self, name):
+        self.name = name
+        self.part_of = {} # level_name -> Group
+
+    def __repr__(self):
+        return "<Node: %s>" % (self.name)
+
+    def expand_to_node_set(self):
+        return {self}
+
+    def add_to_group(self, group):
+        if group.level in self.part_of:
+            raise DuplicateMembership("%s cannot belong to %s %s and %s at the same time" % \
+                                          (self.name,
+                                           group.level,
+                                           self.part_of[group.level].name,
+                                           group.name))
+
+        self.part_of[group.level] = group
+
+    def get_group(self, level):
+        return self.part_of.get(level)
+
+    def get_clustershell_name(self):
+        return self.name
+
+class Group(Entity):
+    def __init__(self, hierarchy, level, name, all_nodes_group = False):
+        self.hierarchy = hierarchy
+        self.level = level
+        self.name = name
+        self.node_set = set() # Node set
+        self.representative = None
+        self.sorted_nodes = None
+        self.all_nodes_group = all_nodes_group
+        self.nodespecs = [] # Raw list of all nodespecs added
+
+    def __repr__(self):
+        return f"<Group {self.level}: {self.name}>"
+
+    def expand_to_node_set(self):
+        return self.node_set
+
+    def get_sorted_nodes(self):
+        if self.sorted_nodes is None:
+            self.sorted_nodes = numerically_sorted_entities(self.node_set)
+        return self.sorted_nodes
+
+    def get_representative(self):
+        if self.representative is None:
+            self.representative = self.get_sorted_nodes()[0]
+        return self.representative
+
+    def get_index_of(self, node):
+        return self.get_sorted_nodes().index(node)
+
+    def add_nodespec(self, nodespec):
+        #print "  NODES", self.name, "-->", nodespec
+        self.nodespecs.append(nodespec)
+        node_set = set()
+        for name in expand_hostlist(nodespec):
+            entity = self.hierarchy.get_entity(name, auto_add_node = self.all_nodes_group)
+            nodes = entity.expand_to_node_set()
+            node_set |= nodes
+        #print "  NODE_SET", node_set
+        self.node_set |= node_set
+        for node in node_set:
+            node.add_to_group(self)
+        self.representative = None
+        return self
+
+    def get_clustershell_name(self):
+        return f"@{self.level}:{self.name}"
+
+    def get_clustershell_definition(self):
+        res = []
+        for ns in self.nodespecs:
+            converted_set = set()
+            for e in set(expand_hostlist(ns)):
+                entity = self.hierarchy.get_entity(e)
+                converted_set.add(entity.get_clustershell_name())
+            part = collect_hostlist(converted_set)
+
+            res.append(part)
+        return ",".join(res)
+
+class FakedNode(Entity):
+    """Use to fake nodes with indexes for --index"""
+    def __init__(self, nodename, index, separator):
+        self.name = "%s%s%d" %(nodename, separator, index)
+
+class Hierarchy:
+    def __init__(self):
+        self.names = {} # name -> Node or Group
+        self.level_names = set() # level names used for abbreviation lookup
+
+        self.level_names_list = [] # level names in the order mentioned in the file
+        self.level_groups = {} # level name -> list of Groups
+
+    def create_group(self, level, name, all_nodes_group = False):
+        #print "GROUP", level, name
+        if not level in self.level_names:
+            self.level_names_list.append(level)
+            self.level_groups[level] = []
+        self.level_names.add(level)
+
+        if name in self.names:
+            raise DuplicatedGroup("group %s seen twice" % name)
+        group = Group(self, level, name, all_nodes_group)
+        self.names[name] = group
+        self.level_groups[level].append(group)
+        return group
+
+    def create_or_get_group(self, level, name):
+        if name in self.names:
+            return self.names[name]
+        else:
+            return self.create_group(level, name)
+
+    def get_entity(self, name, auto_add_node = False):
+        if name not in self.names:
+            if auto_add_node:
+                self.names[name] = Node(name)
+            else:
+                raise UnknownName("unknown name " + name)
+        return self.names[name]
+
+    def parse_file(self, file_or_filename):
+        if isinstance(file_or_filename, str):
+            f = open(file_or_filename)
+        else:
+            f = file_or_filename
+        current_group = None
+        for line in f:
+            line = re.sub(r' *#.*', '', line)
+            line = line.rstrip()
+            if line.strip() == "": continue
+
+            # nodes: <nodes>
+            m = re.match(r'^nodes:\s*(.*)', line)
+            if m:
+                rest = m.group(1)
+                current_group = self.create_group("all", "nodes", all_nodes_group = True)
+                if rest:
+                    current_group.add_nodespec(rest)
+                continue
+
+            # <level> <name>: <parts>
+            m = re.match(r'^([a-z]+)\s+([a-z0-9]+):\s*(.*)', line)
+            if m:
+                (level, name, rest) = m.group(1, 2, 3)
+                current_group = self.create_group(level, name)
+                if rest:
+                    current_group.add_nodespec(rest)
+                continue
+
+            # <parts> (indented)
+            if current_group is not None:
+                m = re.match(r'^\s+(.*)', line)
+                if m:
+                    rest = m.group(1)
+                    current_group.add_nodespec(rest)
+                    continue
+
+            # Fail
+            raise BadConfigSyntax(line)
+        return self
+
+    def parse_slurm(self):
+        rc, slurm_data = subprocess.getstatusoutput('/usr/bin/squeue -aho "%i %u %N"')
+        if rc != 0:
+            raise FailedSLURM("Failed to get data from SLURM")
+
+        for line in slurm_data.split("\n"):
+            fields = line.split()
+            if len(fields) != 3:
+                continue
+            job, user, nodes = fields
+
+            gj = self.create_group("job", "job" + job)
+            gj.add_nodespec(nodes)
+
+            gu = self.create_or_get_group("user", user)
+            gu.add_nodespec(nodes)
+        return self
+
+    def expand_abbreviated_level(self, level):
+        matching = [l for l in self.level_names if l.startswith(level)]
+        if len(matching) == 1:
+            return matching[0]
+        elif len(matching) == 0:
+            raise UnknownLevel("unknown level %s" % level)
+        else:
+            raise AmbiguousLevel("ambiguous level {} matching {}".format(level,
+                                                                     ", ".join(matching)))
+
+    def expand_to_node_set(self, nodespec):
+        res = set()
+        for name in expand_hostlist(nodespec):
+            entity = self.get_entity(name)
+            res |= entity.expand_to_node_set()
+        return res
+
+    # This is the method invoked by "nodes --to-nodes" (default)
+    def to_nodes(self, nodespec):
+        return self.expand_to_node_set(nodespec)
+
+    # This is the central logic behind up, fill and gather
+    # Returns a tuple of sets (groups, leftovers, missing):
+    #   groups:    groups that nodes in nodespec belong to
+    #   leftovers: nodes that did not fill whole groups (or empty if fill is True)
+    #   missing:   nodes that did not belong to any group
+
+    def up_set(self, level, nodespec, fill = True, missing_group = None):
+        groups = set()
+        leftovers = set()
+        missing = set()
+        node_set = self.expand_to_node_set(nodespec)
+        for node in node_set:
+            group = node.get_group(level)
+            if group is None:
+                if missing_group is None:
+                    missing.add(node)
+                else:
+                    groups.add(self.create_or_get_group(level, missing_group))
+            else:
+                if fill or group.expand_to_node_set() <= node_set:
+                    groups.add(group)
+                else:
+                    leftovers.add(node)
+
+        return groups, leftovers, missing
+
+    # This is the method invoked by "nodes --up"
+    def up(self, level, nodespec, missing_group = None):
+        level = self.expand_abbreviated_level(level)
+        groups, leftovers, missing = self.up_set(level, nodespec,
+                                                 missing_group = missing_group)
+
+        assert len(leftovers) == 0
+        if missing:
+            raise MissingGroup("missing %s for %s" % \
+                                   (level,
+                                    collect_name_hostlist(missing)))
+
+        return groups
+
+
+    # This is the method invoked by "nodes --fill"
+    def fill(self, level, nodespec):
+        level = self.expand_abbreviated_level(level)
+        groups, leftovers, missing = self.up_set(level, nodespec)
+
+        assert len(leftovers) == 0
+        if missing:
+            sys.stderr.write("Warning: missing %s for %s\n" % \
+                                 (level,
+                                  collect_name_hostlist(missing)))
+
+        # FIXME: Check if it makes more sense to signal error above?
+        res = missing
+
+        for group in groups:
+            res |= group.expand_to_node_set()
+        return res
+
+    # This is the method invoked by "nodes --representative"
+    def representative(self, level, nodespec):
+        level = self.expand_abbreviated_level(level)
+        groups, leftovers, missing = self.up_set(level, nodespec)
+
+        assert len(leftovers) == 0
+        if missing:
+            sys.stderr.write("Warning: missing %s for %s\n" % \
+                                 (level,
+                                  collect_name_hostlist(missing)))
+
+        # FIXME: Check if it makes more sense to signal error above?
+        res = missing
+
+        for group in groups:
+            res.add(group.get_representative())
+        return res
+
+    # This is the method invoked by "nodes --gather"
+    def gather(self, level, nodespec):
+        level = self.expand_abbreviated_level(level)
+        groups, leftovers, missing = self.up_set(level, nodespec, fill=False)
+
+        if missing:
+            sys.stderr.write("Warning: missing %s for %s\n" % \
+                                 (level,
+                                  collect_name_hostlist(missing)))
+
+        return groups|leftovers|missing
+
+    # This is the method invoked by "nodes --index"
+    def index(self, level, nodespec, separator = ": "):
+        node_set = self.expand_to_node_set(nodespec)
+        level = self.expand_abbreviated_level(level)
+        res = []
+        missing = set()
+        for node in node_set:
+            group = node.get_group(level)
+            if group is None:
+                missing.add(node)
+            else:
+                index = group.get_index_of(node)
+                res.append(FakedNode(node.name, index, separator))
+        if missing:
+            sys.stderr.write("Warning: missing %s for %s\n" % \
+                                 (level,
+                                  collect_name_hostlist(missing)))
+        return res
+
+    # Convert to ClusterShell YAML definition
+    def convert_to_clustershell(self):
+        for level in self.level_names_list:
+            if level == "all": continue
+            print("%s:" % level)
+            for group in self.level_groups[level]:
+                print(f"   {group.name}: '{group.get_clustershell_definition()}'")
+            print()

hpc_nodes-1.8.3/nodes.spec

@@ -0,0 +1,99 @@
+Name:           nodes
+Version:        %(cat VERSION.file)
+Release:        1%{?dist}
+Summary:        Handle hierarchical groups of nodes
+Vendor:         NSC
+
+Group:          Development/Languages
+License:        GPLv2+
+URL:            http://www.nsc.liu.se/~kent/nodes/
+Source0:        http://www.nsc.liu.se/~kent/nodes/%{name}-%{version}.tar.gz
+BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
+
+BuildArch:      noarch
+BuildRequires:  python3-devel
+Requires:       python3-hostlist
+
+
+# The default value is '-s' (a subset of -I).
+%global py3_shbang_opts -I
+
+
+%description
+Handle hierarchical groups of nodes, such as enclosures and racks.
+
+
+%prep
+%setup -q
+
+
+%build
+%py3_build
+
+
+%install
+%py3_install -- --prefix /usr
+
+
+%files
+%defattr(-,root,root,-)
+%{python3_sitelib}/nodes.py
+%{python3_sitelib}/nodes-*.egg-info
+%{python3_sitelib}/__pycache__/*
+/usr/bin/nodes
+%config(noreplace) /etc/nodes.conf
+/usr/share/man/man1/nodes.1.gz
+/usr/share/man/man5/nodes.conf.5.gz
+%doc nodes.conf.sample
+
+
+%changelog
+* Tue Mar 04 2026 Christian Luckey <rovanion@nsc.liu.se> - 1.8.3-1
+- Move project declaration to pyproject.toml
+- setup.py: Remove indicators of Python 2 support
+- Remove #RELEASE# from nodes and nodes.py
+
+* Fri Feb 27 2026 Christian Luckey <rovanion@nsc.liu.se> - 1.8.2-1
+- Return self from self-modifying methods of Group and Hierarchy
+
+* Tue Nov 15 2022 Torbjörn Lönnemark <ketl@nsc.liu.se> - 1.8.1-1
+- Let RPM handle Python interpreter line flags
+
+* Thu Oct 27 2022 Torbjörn Lönnemark <ketl@nsc.liu.se> - 1.8.0-1
+- Port to Python 3
+
+* Wed Jun  2 2021 Torbjörn Lönnemark <ketl@nsc.liu.se> - 1.7-1
+- Specify Python version explicitly in shebang
+- Explicitly use Python 2 in Makefile
+- Fix spec file for el8
+- Remove unneeded python_sitelib fallback
+- Update dependency hostlist to use new package name
+- Include dist tag in Release in RPMs
+- Add --convert-to-clustershell option.
+
+* Fri Nov  2 2012 Kent Engström <kent@nsc.liu.se> - 1.6-1
+- Add --count, --index and --index-separator.
+
+* Mon Aug 23 2010 Kent Engström <kent@nsc.liu.se> - 1.5-1
+- Add --expand together with --separator, --prepend and --append.
+
+* Tue Mar  9 2010 Kent Engström <kent@nsc.liu.se> - 1.4-1
+- Add --representative operation.
+- Add --missing option for the --up operation.
+
+* Sun Feb 21 2010 Kent Engström <kent@nsc.liu.se> - 1.3-1
+- Add man pages nodes(1) and nodes.conf(5).
+- Change gather behaviour when nodes do not belong to a group.
+- Refactor and rename internal methods.
+- Add unit tests.
+
+* Tue Feb 09 2010 Kent Engström <kent@nsc.liu.se> - 1.2-1
+- Add dynamic SLURM groups for users and jobs.
+
+* Mon Nov 02 2009 Kent Engström <kent@nsc.liu.se> - 1.1-1
+- Add one-letter options for --up, --fill and --gather.
+- Add --config-file option.
+- Allow abbreviations of level names as long as they are not ambiguous.
+
+* Fri Oct 09 2009 Kent Engström <kent@nsc.liu.se> - 1.0-1
+- Package as RPM.

hpc_nodes-1.8.3/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name            = "hpc-nodes"
+description     = "Handle hierarchical groups of nodes"
+authors         = [
+    {name = "Kent Engström", email = "kent@nsc.liu.se"},
+  ]
+requires-python = ">=3.9"
+license = "GPL-2.0"
+classifiers     = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: System Administrators",
+    "Topic :: System :: Clustering",
+    "Topic :: System :: Systems Administration",
+    "Programming Language :: Python :: 3",
+  ]
+dynamic        = [ "version" ]
+dependencies   = [
+    "python-hostlist",
+  ]
+
+[project.urls]
+homepage       = "http://www.nsc.liu.se/~kent/nodes/"
+
+[tool.setuptools.dynamic]
+version        = { file = "VERSION.file" }
+
+[dependency-groups]
+dev = [
+    "ruff",
+    "twine",
+]
+
+[tool.ruff.lint]
+ignore = [ "E731", "E701" ]

hpc_nodes-1.8.3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hpc_nodes-1.8.3/setup.py

@@ -0,0 +1,10 @@
+from setuptools import setup
+
+setup(name         = "hpc-nodes",
+      py_modules   = ["nodes"],
+      scripts      = ["nodes"],
+      data_files   = [("/etc", ["nodes.conf"]),
+                      ("share/man/man1", ["nodes.1"]),
+                      ("share/man/man5", ["nodes.conf.5"]),
+                      ],
+      )

hpc_nodes-1.8.3/test/test_nodes.py

@@ -0,0 +1,152 @@
+import nodes
+import unittest
+import io
+
+class Test1(unittest.TestCase):
+
+    def setUp(self):
+        self.h = nodes.Hierarchy()
+        f = io.StringIO("""
+nodes: n[1-99]
+enclosure e1: n[1-4]
+enclosure e2: n[5-8]
+enclosure e3: n[9-12]
+enclosure e4: n[13-16]
+rack r1: e[1-2]
+rack r2: e[3-4]
+oddball o1: n[6-10]
+""")
+        self.h.parse_file(f)
+        f.close()
+
+    def expands_to(self, nodespec, hostlist):
+        self.assertEqual(nodes.collect_name_hostlist(self.h.to_nodes(nodespec)), hostlist)
+        
+    def expands_bad(self, nodespec):
+        self.assertRaises(nodes.NodesException,
+                          self.h.to_nodes,  nodespec)
+
+    def test_expand(self):
+        self.expands_to("",       "")
+        self.expands_to("n1",     "n1")
+        self.expands_to("n99",    "n99")
+        self.expands_to("e1",     "n[1-4]")
+        self.expands_to("e1,e1",  "n[1-4]")
+        self.expands_to("e[1-2]", "n[1-8]")
+        self.expands_to("e[1,3]", "n[1-4,9-12]")
+        self.expands_to("e1,n1",  "n[1-4]")
+        self.expands_to("e1,n5",  "n[1-5]")
+        self.expands_to("n6,e1",  "n[1-4,6]")
+        self.expands_to("r1",     "n[1-8]")
+        self.expands_to("r1,e3",  "n[1-12]")
+        self.expands_to("r1,e4",  "n[1-8,13-16]")
+        self.expands_to("o1",     "n[6-10]")
+        self.expands_to("o1,e1",  "n[1-4,6-10]")
+        self.expands_to("o1,e2",  "n[5-10]")
+        self.expands_to("o1,e3",  "n[6-12]")
+        
+        self.expands_bad("n0")
+        self.expands_bad("n100")
+
+    def ufrg(self, level, nodespec, up, fill, representative, gather):
+        if up is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.up(level, nodespec)), up)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.up, level, nodespec)
+
+        if fill is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.fill(level, nodespec)), fill)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.fill, level, nodespec)
+
+        if representative is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.representative(level, nodespec)), representative)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.representative, level, nodespec)
+
+        if gather is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.gather(level, nodespec)), gather)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.gather, level, nodespec)
+
+    def ufrg_e(self, nodespec, up, fill, representative, gather):
+        self.ufrg("enclosure", nodespec, up, fill, representative, gather)
+
+    def ufrg_r(self, nodespec, up, fill, representative, gather):
+        self.ufrg("rack", nodespec, up, fill, representative, gather)
+
+    def test_up_fill_representative_gather(self):
+        #           NODESPEC     UP          FILL            REPRESENTATIVE  GATHER
+        # -----------------------------------------------------------------------------------
+        self.ufrg_e("",          "",         "",             "",             "")
+        self.ufrg_e("e1",        "e1",       "n[1-4]",       "n1",           "e1")  
+        self.ufrg_e("n1",        "e1",       "n[1-4]",       "n1",           "n1")
+        self.ufrg_e("n2",        "e1",       "n[1-4]",       "n1",           "n2")
+        self.ufrg_e("n3",        "e1",       "n[1-4]",       "n1",           "n3")
+        self.ufrg_e("n4",        "e1",       "n[1-4]",       "n1",           "n4")
+        self.ufrg_e("n5",        "e2",       "n[5-8]",       "n5",           "n5")
+        self.ufrg_e("n[1-2]",    "e1",       "n[1-4]",       "n1",           "n[1-2]")
+        self.ufrg_e("n[4-5]",    "e[1-2]",   "n[1-8]",       "n[1,5]",       "n[4-5]")
+        self.ufrg_e("n[1-5,14]", "e[1-2,4]", "n[1-8,13-16]", "n[1,5,13]",    "e1,n[5,14]")
+        self.ufrg_e("n[1-8,14]", "e[1-2,4]", "n[1-8,13-16]", "n[1,5,13]",    "e[1-2],n14")
+        self.ufrg_e("n[1-9,14]", "e[1-4]",   "n[1-16]",      "n[1,5,9,13]",  "e[1-2],n[9,14]")
+                                                                            
+        self.ufrg_e("r1",        "e[1-2]",   "n[1-8]",       "n[1,5]",       "e[1-2]")
+        self.ufrg_e("r1,e3",     "e[1-3]",   "n[1-12]",      "n[1,5,9]",     "e[1-3]")
+        self.ufrg_e("r1,e4",     "e[1-2,4]", "n[1-8,13-16]", "n[1,5,13]",    "e[1-2,4]")
+        self.ufrg_e("r1,n16",    "e[1-2,4]", "n[1-8,13-16]", "n[1,5,13]",    "e[1-2],n16")
+        self.ufrg_e("r[1-2]",    "e[1-4]",   "n[1-16]",      "n[1,5,9,13]",  "e[1-4]")
+                                                                            
+        self.ufrg_e("o1",        "e[2-3]",   "n[5-12]",      "n[5,9]",       "n[6-10]")
+                                                                            
+        self.ufrg_e("e5",        None,       None,           None,           None)
+        self.ufrg_e("n17",       None,       "n17",          "n17",          "n17")
+        self.ufrg_e("n[16-17]",  None,       "n[13-17]",     "n[13,17]",     "n[16-17]") # representative?
+                                                                            
+        self.ufrg_r("r1",        "r1",       "n[1-8]",       "n1",           "r1")
+        self.ufrg_r("e1",        "r1",       "n[1-8]",       "n1",           "n[1-4]")
+        self.ufrg_r("n1",        "r1",       "n[1-8]",       "n1",           "n1")
+        self.ufrg_r("e[1-2]",    "r1",       "n[1-8]",       "n1",           "r1")
+        self.ufrg_r("e[1-3]",    "r[1-2]",   "n[1-16]",      "n[1,9]",       "n[9-12],r1")
+        self.ufrg_r("e[1-4]",    "r[1-2]",   "n[1-16]",      "n[1,9]",       "r[1-2]")
+        self.ufrg_r("r1,e4",     "r[1-2]",   "n[1-16]",      "n[1,9]",       "n[13-16],r1")
+        self.ufrg_r("r1,n16",    "r[1-2]",   "n[1-16]",      "n[1,9]",       "n16,r1")
+                                                                            
+        self.ufrg_r("o1",        "r[1-2]",   "n[1-16]",      "n[1,9]",       "n[6-10]")
+                                                                            
+        self.ufrg_r("r3",        None,       None,           None,           None)
+        self.ufrg_r("n17",       None,       "n17",          "n17",          "n17")
+        self.ufrg_r("n[16-17]",  None,       "n[9-17]",      "n[9,17]",      "n[16-17]") # representative?
+
+    def um(self, level, nodespec, up, up_missing):
+        if up is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.up(level, nodespec)), up)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.up, level, nodespec)
+        if up_missing is not None:
+            self.assertEqual(nodes.collect_name_hostlist(self.h.up(level, nodespec, "M")), up_missing)
+        else:
+            self.assertRaises(nodes.NodesException,
+                              self.h.up, level, nodespec, "M")
+
+    def um_e(self, nodespec, up, up_missing):
+        self.um("enclosure", nodespec, up, up_missing)
+
+    def test_up_missing(self):
+        #         NODESPEC     UP          UP_MISSING
+        # ---------------------------------------------
+        self.um_e("n1",        "e1",       "e1")
+        self.um_e("n17",       None,       "M")
+        self.um_e("n[16-17]",  None,       "M,e4")
+        self.um_e("n[1-20]",   None,       "M,e[1-4]")
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+