ClusterShell 1.9.2__tar.gz → 1.10__tar.gz

{ClusterShell-1.9.2 → clustershell-1.10}/ChangeLog

@@ -1,3 +1,92 @@
+2026-07-04 S. Thiell <sthiell@stanford.edu>
+
+	* Version 1.10 released. The main changes are listed below.
+
+	* NodeUtils: add the 'mapall' group source upcall to retrieve all group
+	mappings in a single call, greatly speeding up group resolution on slow
+	sources (#612, #613)
+
+	* NodeUtils: do not pass stdin to group source upcalls (#581)
+
+	* NodeSet, RangeSet: add a list-like index() method returning the position
+	of a node or element, the inverse of [i] (#631, #633)
+
+	* cluset, nodeset: add a --index command to output a node's position in the
+	set, the reverse of -I/--slice (#637)
+
+	* groups.conf: add mapall upcalls to the Slurm group bindings (#623)
+
+	* groups.conf: add an Ansible inventory group source example (#610)
+
+	* clush, clubak: add --axis to fold nD node sets along selected axis only,
+	as already supported by nodeset and cluset (#356)
+
+	* Display: fix line buffering on Python 3.6 (#597)
+
+	* Tree: implement TreeWorker.abort() and per-gateway abort, with gateway
+	failover fixes (#229, #566)
+
+	* Tree: fire ev_pickup only once the command can no longer be rerouted and
+	improve reroute event reporting (#594)
+
+	* Tree: fix rcopy when a remote node closes (#545)
+
+	* Tree: improve handling of MessageProcessingError (#596)
+
+	* Tree: keep tar/scp errors on stderr instead of stdout during forward
+	copy (#622)
+
+	* Tree: pass filter='fully_trusted' to tarfile.extractall() (PEP 706) (#617)
+
+	* EngineClient: properly honor the abort flag on close (#591)
+
+	* Engine: fix set_events debug logging to identify the client and stream (#630)
+
+	* Task: only terminate a task once when killed, fixing a double termination
+	(#110, #640)
+
+	* NodeUtils, Topology: emit a DeprecationWarning for set_verbosity() and the
+	topology.conf [Main] section (#226, #639)
+
+	* Gateway: fix a Python 2 syntax error (#625)
+
+	* RangeSet: accept unicode strings on Python 2 (#635)
+
+	* completion: complete copy paths and command, and short options without
+	spaces (#585, #586)
+
+	* completion: treat -b/-B and clush's other no-arg options as flags (#598, #619)
+
+	* completion: fix cluset no-arg option lists (#620)
+
+	* doc: clarify the '*' all-nodes wildcard versus the literal 'all' group
+	source (#552)
+
+	* doc: HPSF onboarding (charter, DCO, governance) and misc doc fixes
+	(#577, #604)
+
+	* setup: clean up wheel packaging metadata (#634)
+
+	* CI: add Python 3.14 to the test matrix; run the test suite with the
+	standard library unittest instead of the abandoned nosetests runner
+	(#603, #618)
+
+2025-01-23 S. Thiell <sthiell@stanford.edu>
+
+	* Version 1.9.3 released. The main changes are listed below.
+
+	* Added bash completions (#563)
+
+	* Communication: sax parser: add flush() after feed() (#556)
+
+	* Additional Slurm binding options (#558 and #561)
+
+	* clush: use set instead of NodeSet for runtime progress info (#562)
+
+	* Tree: Tree: use set instead of NodeSet for gwtargets tracking (#562)
+
+	* CLI/Nodeset: omit @source: prefix for cluset -s source -L (#563) (#570)
+
 2023-09-29 S. Thiell <sthiell@stanford.edu>
 
 	* Version 1.9.2 released. The main changes are listed below.
@@ -138,8 +227,8 @@
 
 	* EventHandler: reinstate ev_error and ev_timeout as deprecated (#377)
 
-	* nodeset/cluset CLI: allow litteral new line in -S, so both -S "\n"
-	and -S $'\n' will work
+	* nodeset/cluset CLI: allow literal new line in -S, so both -S "\n" and
+	-S $'\n' will work
 
 	* nodeset/cluset CLI: handle multiline shell arguments in options (#394)
 
@@ -565,7 +654,7 @@
 	CLUSTERSHELL_GW_LOG_DIR and CLUSTERSHELL_GW_LOG_LEVEL environment
 	variables from the root node.
 
-	* Communication.py: Messages are now transfered in xml payload instead of
+	* Communication.py: Messages are now transferred in xml payload instead of
 	'output' attribute for improved handling of multi-lines messages in
 	StdOutMessage and StdErrMessage.
 
@@ -1389,7 +1478,7 @@
 2009-12-09 A. Degremont  <aurelien.degremont@cea.fr>
 
 	* scripts/nodeset.py: Protect --separator from code injection and
-	handle gracefully incorrect separtor.
+	handle gracefully incorrect separator.
 
 2009-12-09 S. Thiell <stephane.thiell@cea.fr>
 

{ClusterShell-1.9.2 → clustershell-1.10}/MANIFEST.in

@@ -1,6 +1,7 @@
 include ChangeLog
 include README.md
 include COPYING.LGPLv2.1
+include bash_completion.d/*
 include conf/*.conf
 include conf/*.example
 include conf/clush.conf.d/README
@@ -31,3 +32,4 @@ include doc/examples/*.py
 include doc/examples/defaults.conf-rsh
 include doc/epydoc/*.conf
 include tests/*.py
+include tests/bin/*

clustershell-1.10/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: ClusterShell
+Version: 1.10
+Summary: ClusterShell library and tools
+Home-page: https://clustershell.readthedocs.io/
+Download-URL: https://github.com/clustershell/clustershell/archive/refs/tags/v1.10.tar.gz
+Author: Stephane Thiell
+Author-email: sthiell@stanford.edu
+License: LGPLv2+
+Keywords: clustershell,clush,clubak,nodeset
+Platform: GNU/Linux
+Platform: BSD
+Platform: MacOSX
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: BSD
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Distributed Computing
+Description-Content-Type: text/x-rst
+License-File: COPYING.LGPLv2.1
+Requires-Dist: PyYAML
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: platform
+Dynamic: requires-dist
+Dynamic: summary
+
+ClusterShell is an event-driven open source Python framework, designed to run
+local or distant commands in parallel on server farms or on large Linux
+clusters. It will take care of common issues encountered on HPC clusters, such
+as operating on groups of nodes, running distributed commands using optimized
+execution algorithms, as well as gathering results and merging identical
+outputs, or retrieving return codes. ClusterShell takes advantage of existing
+remote shell facilities already installed on your systems, like SSH.
+
+ClusterShell provides clush, clubak and cluset/nodeset, convenient command-line
+tools that allow traditional shell scripts to benefit from some of the
+library's features:
+
+- **clush**: issue commands to cluster nodes and format output
+
+Example of use:
+
+::
+
+  $ clush -abL uname -r
+  node[32-49,51-71,80,82-150,156-159]: 2.6.18-164.11.1.el5
+  node[3-7,72-79]: 2.6.18-164.11.1.el5_lustre1.10.0.36
+  node[2,151-155]: 2.6.31.6-145.fc11.2.x86_64
+
+See *man clush* for more details.
+
+- **clubak**: improved dshbak to gather and sort dsh-like outputs
+
+See *man clubak* for more details.
+
+- **nodeset** (or **cluset**): compute advanced nodeset/nodegroup operations
+
+Examples of use:
+
+::
+
+  $ echo node160 node161 node162 node163 | nodeset -f
+  node[160-163]
+
+  $ nodeset -f node[0-7,32-159] node[160-163]
+  node[0-7,32-163]
+
+  $ nodeset -e node[160-163]
+  node160 node161 node162 node163
+
+  $ nodeset -f node[32-159] -x node33
+  node[32,34-159]
+
+  $ nodeset -f node[32-159] -i node[0-7,20-21,32,156-159]
+  node[32,156-159]
+
+  $ nodeset -f node[33-159] --xor node[32-33,156-159]
+  node[32,34-155]
+
+  $ nodeset -l
+  @oss
+  @mds
+  @io
+  @compute
+
+  $ nodeset -e @mds
+  node6 node7
+
+See *man nodeset* (or *man cluset*) for more details.
+
+Please visit the ClusterShell website_.
+
+.. _website: https://clustershell.readthedocs.io/
+

{ClusterShell-1.9.2 → clustershell-1.10}/README.md

@@ -1,13 +1,19 @@
 ClusterShell Python Library and Tools
 =====================================
 
-ClusterShell is an event-driven open source Python library, designed to run
+[![HPSF Established](https://raw.githubusercontent.com/hpsfoundation/tac/main/badges/HPSF_Project_Badge_Established.png)](https://hpsf.io)
+
+This project adheres to a [Technical Charter](https://clustershell.readthedocs.io/en/latest/CHARTER.html), which defines its governance model, decision-making process, and long-term vision.
+
+ClusterShell is an event-driven open source Python library designed to run
 local or distant commands in parallel on server farms or on large Linux
-clusters. It will take care of common issues encountered on HPC clusters, such
-as operating on groups of nodes, running distributed commands using optimized
-execution algorithms, as well as gathering results and merging identical
-outputs, or retrieving return codes. ClusterShell takes advantage of existing
-remote shell facilities already installed on your systems, like SSH.
+clusters. It is an [HPSF Established](https://hpsf.io) project under
+[LF Europe](https://linuxfoundation.eu/) stewardship.
+
+ClusterShell handles typical HPC cluster administration tasks: operating on
+groups of nodes, executing distributed commands with optimized algorithms,
+gathering and merging output, and collecting return codes. It leverages remote
+shell facilities already present on your systems, such as SSH.
 
 ClusterShell's primary goal is to improve the administration of high-
 performance clusters by providing a lightweight but scalable Python API for
@@ -34,7 +40,7 @@ Documentation
 
 Online documentation is available here:
 
-    http://clustershell.readthedocs.org/
+    https://clustershell.readthedocs.io/
 
 The Sphinx documentation source is available under the doc/sphinx directory.
 Type 'make' to see all available formats (you need Sphinx installed and
@@ -54,11 +60,13 @@ The following man pages are also provided:
 Test Suite
 ----------
 
-Regression testing scripts are available in the 'tests' directory:
+Regression testing scripts are available in the 'tests' directory. To run the
+whole test suite from the source tree root:
 
-    $ cd tests
-    $ nosetests -sv <Test.py>
-    $ nosetests -sv --all-modules
+    $ python -m unittest discover -v -s tests -p '*Test.py' -t .
+
+To run a single test module, narrow the discovery pattern, for example
+`-p 'NodeSetTest.py'`.
 
 You have to allow 'ssh localhost' and 'ssh $HOSTNAME' without any warnings for
 "remote" tests to run as expected. $HOSTNAME should not be 127.0.0.1 nor ::1.
@@ -84,32 +92,22 @@ linux[4-6] 2.6.32-71.el6.x86_64
 Links
 -----
 
-Web site:
-
-    http://cea-hpc.github.com/clustershell/
-
 Online documentation:
 
-    http://clustershell.readthedocs.org/
+    https://clustershell.readthedocs.io/
 
 Github source repository:
 
-    https://github.com/cea-hpc/clustershell
-
-Github Wiki:
-
-    https://github.com/cea-hpc/clustershell/wiki
+    https://github.com/clustershell/clustershell
 
 Github Issue tracking system:
 
-    https://github.com/cea-hpc/clustershell/issues
+    https://github.com/clustershell/clustershell/issues
 
 Python Package Index (PyPI) links:
 
     https://pypi.org/project/ClusterShell/
 
-    http://pypi.python.org/pypi/ClusterShell
-
 ClusterShell was born along with Shine, a scalable Lustre FS admin tool:
 
     https://github.com/cea-hpc/shine
@@ -119,7 +117,16 @@ Core developers/reviewers
 
 * Stephane Thiell
 * Aurelien Degremont
-* Henri Doreau
 * Dominique Martinet
 
-CEA/DAM 2010, 2011, 2012, 2013, 2014, 2015 - http://www-hpc.cea.fr
+Acknowledgments
+---------------
+
+We would like to express our gratitude to [CEA](https://www.cea.fr/english)
+for the resources and funding provided to the project over the years.
+
+We would also like to thank [Stanford University](https://srcc.stanford.edu)
+(SRC) for their long-term support and resources provided to the project.
+
+See the full [Acknowledgments](https://clustershell.readthedocs.io/en/latest/acknowledgments.html)
+page for more details.

clustershell-1.10/bash_completion.d/cluset

@@ -0,0 +1,80 @@
+# cluset bash completion
+#
+# to install in /usr/share/bash-completion/completions/ or ~/.local/share/bash-completion/completions/
+_cluset()
+{
+	# shellcheck disable=SC2034 # set/used by _init_completion
+	local cur prev words cword split
+	local word options="" skip=argv0 groupsource="" cleangroup=""
+
+	_init_completion -s -n : || return
+
+	# stop parsing if there had been any non-option before (or --)
+	for word in "${words[@]}"; do
+		case "$skip" in
+		"") ;;
+		groupsource)
+			groupsource="$word"
+			;& # fallthrough
+		*)
+			skip=""
+			continue
+			;;
+		esac
+		case "$word" in
+		"") ;;
+		--) return;;
+		# no-arg options
+		--version|-h|--help|-l|--list|-L|--list-all|-r|--regroup|\
+		--list-sources|--groupsources|-d|--debug|-q|--quiet|\
+		-R|--rangeset|-G|--groupbase|--contiguous|-a|--all|--completion) ;;
+		# get source separately...
+		--groupsource=*) groupsource="${word#*=}";;
+		-s|--groupsource) skip=groupsource;;
+		# assume all the rest as options...
+		# options with = included in word
+		--*=*) ;;
+		-*) skip=any;;
+		*) return;; # was non-option
+		esac
+	done
+
+	case "$prev" in
+	-c|--count|-e|--expand|-f|--fold|--index|\
+	-x|--exclude|-i|--intersection|-X|--xor)
+		case "$cur" in
+		*:*)
+			groupsource="${cur%%:*}"
+			groupsource="${groupsource#@}"
+			;;
+		*)
+			if [ -n "$groupsource" ]; then
+				cleangroup=1
+			fi
+			;;
+		esac
+		options="$(cluset ${groupsource:+-s "$groupsource"} --completion @*)"
+		if [ -n "$cleangroup" ]; then
+			options=${options//@"$groupsource":/@}
+		fi
+		;;
+	-s|--groupsource)
+		options=$(cluset --groupsources --quiet)
+		;;
+	# no-arg options
+	--version|-h|--help|-l|--list|-L|--list-all|-r|--regroup|\
+	--list-sources|--groupsources|-d|--debug|-q|--quiet|\
+	-R|--rangeset|-G|--groupbase|--contiguous|-a|--all|--completion) ;;
+	# any other option: just ignore.
+	-*)
+		return;;
+	esac
+	# get all options from help text... not 100% accurate but good enough.
+	[ -n "$options" ] || options="$(cluset --help | grep -oP -- '(?<=[ \t])(-[a-z]|--[^= \t]*)')"
+
+	# append space for everything that doesn't end in `:` (likely a groupsource)
+	mapfile -t COMPREPLY < <(compgen -W "$options" -- "$cur" | sed -e 's/[^:]$/& /')
+	# remove the prefix from COMPREPLY if $cur contains colons and
+	# COMP_WORDBREAKS splits on colons...
+	__ltrim_colon_completions "$cur"
+} && complete -o nospace -F _cluset ${BASH_SOURCE##*/}

clustershell-1.10/bash_completion.d/clush

@@ -0,0 +1,159 @@
+# clush bash completion
+#
+# to install in /usr/share/bash-completion/completions/ or ~/.local/share/bash-completion/completions/
+_clush_command_or_file() {
+	# undo our nospace setting...
+	compopt +o nospace
+
+	# skip if shortopt is set -- these helpers cannot "restore" shortopt at start of
+	# the completions they provide easily
+	[ -n "$shortopt" ] && return
+
+	# complete either files (copy mode) or commands (if target set)
+	case "$target_set,$mode" in
+	*,copy)
+		# available since bash-completion 2.12
+		if declare -F _comp_compgen_filedir >/dev/null; then
+			_comp_compgen_filedir
+		else
+			_filedir
+		fi
+		;;
+	1,command)
+		# available since bash-completion 2.12
+		if declare -F _comp_command_offset >/dev/null; then
+			_comp_command_offset "$i"
+		else
+			_command_offset "$i"
+		fi
+		;;
+	esac
+}
+
+_clush()
+{
+	# shellcheck disable=SC2034 # set/used by _init_completion
+	local cur prev words cword split
+	local i word options="" compopts="" skip=argv0 groupsource="" cleangroup=""
+	local mode=command target_set=""
+	local shortopt=""
+
+	_init_completion -s -n : || return
+
+	# stop parsing if there had been any non-option before (or --)
+	for i in "${!words[@]}"; do
+		word="${words[i]}"
+		case "$skip" in
+		"") ;;
+		groupsource)
+			groupsource="$word"
+			;& # fallthrough
+		*)
+			skip=""
+			continue
+			;;
+		esac
+		case "$word" in
+		"") ;;
+		--)
+			i=$((i+1))  # command from next word!
+			_clush_command_or_file
+			return
+			;;
+		-c|--copy|--rcopy) mode=copy;;
+		-w|-g|--group) target_set=1; skip=any;;
+		# no-arg options
+		--version|-h|--help|-n|--nostdin|-a|--all|-q|--quiet|\
+		-b|-B|--dshbak|-G|--groupbase|-L|-N|-P|--progress|\
+		-r|--regroup|-S|--maxrc|--diff|-p|-v|--verbose|-d|--debug) ;;
+		# get source separately...
+		--groupsource=*) groupsource="${word#*=}";;
+		-s|--groupsource) skip=groupsource;;
+		# assume all the rest as options...
+		# options with = included in word
+		--*=*) ;;
+		-*) skip=any;;
+		*)
+			# likely non-option, in copy mode options like -w can come
+			# later so just skip, otherwise likely start of command
+			[ "$mode" = copy ] && continue
+			_clush_command_or_file
+			return;;
+		esac
+	done
+
+	# split short opts without space...
+	case "$cur" in
+	-[a-z]*)
+		shortopt="${cur:0:2}"
+		prev="$shortopt"
+		cur="${cur:2}"
+		;;
+	esac
+
+	case "$prev" in
+	-w|-x|-g|--group|-X)
+		case "$cur" in
+		*:*)
+			groupsource="${cur%%:*}"
+			groupsource="${groupsource#@}"
+			;;
+		*)
+			if [ -n "$groupsource" ]; then
+				cleangroup=1
+			fi
+			;;
+		esac
+		if [ "$prev" = "-w" ]; then
+			compopts="@*"  # include all nodes
+		fi
+		# shellcheck disable=SC2086 ## $compopts expanded on purpose
+		options="$(cluset ${groupsource:+-s "$groupsource"} --completion $compopts)"
+		if [ -n "$cleangroup" ]; then
+			options=${options//@"$groupsource":/@}
+		fi
+		case "$prev" in
+		-g|--group|-X)
+			options=${options//@/}
+			;;
+		esac
+		;;
+	-s|--groupsource)
+		options=$(cluset --groupsources --quiet)
+		;;
+	--color)
+		options="never always auto"
+		;;
+	-R|--worker)
+		options="ssh exec rsh"
+		;;
+	# no-arg options
+	--version|-h|--help|-n|--nostdin|-a|--all|-q|--quiet|\
+	-b|-B|--dshbak|-G|--groupbase|-L|-N|-P|--progress|\
+	-r|--regroup|-S|--maxrc|--diff|-p|-v|--verbose|-d|--debug|-c|--copy|--rcopy) ;;
+	# any other option: ignore next word (likely argument)
+	-*)
+		return;;
+	esac
+	# new option or no option:
+	if [ -z "$options" ]; then
+		case "$cur" in
+		-*)
+			# starts with dash - get all options from help text...
+			options="$(clush --help | grep -oP -- '(?<=[ \t])(-[a-z]|--[^= \t]*)')"
+			;;
+		*)
+			# otherwise complete command or file if appropriate and stop here
+			_clush_command_or_file
+			return
+		esac
+	fi
+
+	# append space for everything that doesn't end in `:` (likely a groupsource)
+	mapfile -t COMPREPLY \
+		< <(compgen -W "$options" -- "$cur" \
+			| sed -e 's/[^:]$/& /' -e "s/^/$shortopt/")
+	# remove the prefix from COMPREPLY if $cur contains colons and
+	# COMP_WORDBREAKS splits on colons...
+	__ltrim_colon_completions "$cur"
+} && complete -o nospace -F _clush "${BASH_SOURCE##*/}"

{ClusterShell-1.9.2 → clustershell-1.10}/conf/groups.conf

@@ -1,7 +1,7 @@
 # ClusterShell node groups main configuration file
 #
 # Please see `man 5 groups.conf` and
-# http://clustershell.readthedocs.org/en/latest/config.html#node-groups
+# https://clustershell.readthedocs.io/en/latest/config.html#node-groups
 # for further details.
 #
 # NOTE: This is a simple group configuration example file, not a

clustershell-1.10/conf/groups.conf.d/ansible.conf.example

@@ -0,0 +1,25 @@
+# Additional ClusterShell group source config file
+#
+# Please see `man 5 groups.conf` for further details.
+#
+
+# Ansible inventory bindings
+#
+# Requirements: ansible-core (provides ansible-inventory), jq
+#
+# Replace /path/to/inventory with your inventory path or directory,
+# or set ANSIBLE_INVENTORY in your environment to override it.
+#
+[ansible]
+
+# Resolve group members, recursing into child groups
+map: ANSIBLE_INVENTORY="${ANSIBLE_INVENTORY:-/path/to/inventory}" ansible-inventory --list 2>/dev/null | jq -r --arg g "$GROUP" 'def r(d;g): (d[g].hosts//[])[], ((d[g].children//[])[] as $c | r(d;$c)); r(.;$g)'
+
+# Resolve all groups in a single call, recursing into child groups
+mapall: ANSIBLE_INVENTORY="${ANSIBLE_INVENTORY:-/path/to/inventory}" ansible-inventory --list 2>/dev/null | jq -r 'def r(d;g): (d[g].hosts//[])[], ((d[g].children//[])[] as $c | r(d;$c)); . as $d | keys[] | select(. != "_meta" and . != "all" and (test("[[:space:]:]") | not)) | "\(.): \([r($d;.)] | join(" "))"'
+
+# All inventory hosts
+all: ANSIBLE_INVENTORY="${ANSIBLE_INVENTORY:-/path/to/inventory}" ansible-inventory --list 2>/dev/null | jq -r '.[].hosts // empty | .[]'
+
+# All groups
+list: ANSIBLE_INVENTORY="${ANSIBLE_INVENTORY:-/path/to/inventory}" ansible-inventory --list 2>/dev/null | jq -r 'keys[] | select(. != "_meta" and . != "all")'