multiSSH3 5.39__tar.gz → 5.41__tar.gz

{multissh3-5.39 → multissh3-5.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: multiSSH3
-Version: 5.39
+Version: 5.41
 Summary: Run commands on multiple hosts via SSH
 Home-page: https://github.com/yufei-pan/multiSSH3
 Author: Yufei Pan

{multissh3-5.39 → multissh3-5.41}/multiSSH3.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: multiSSH3
-Version: 5.39
+Version: 5.41
 Summary: Run commands on multiple hosts via SSH
 Home-page: https://github.com/yufei-pan/multiSSH3
 Author: Yufei Pan

{multissh3-5.39 → multissh3-5.41}/multiSSH3.py

@@ -1,13 +1,21 @@
 #!/usr/bin/env python3
 __curses_available = False
+__resource_lib_available = False
 try:
 	import curses
 	__curses_available = True
 except ImportError:
 	pass
+try:
+	import resource
+	__resource_lib_available = True
+except ImportError:
+	pass
+
 import subprocess
 import threading
-import time,os
+import time
+import os
 import argparse
 from itertools import product
 import re
@@ -37,7 +45,7 @@ except AttributeError:
 		# If neither is available, use a dummy decorator
 		def cache_decorator(func):
 			return func
-version = '5.39'
+version = '5.41'
 VERSION = version
 
 CONFIG_FILE = '/etc/multiSSH3.config.json'	
@@ -352,26 +360,35 @@ if True:
 	__curses_current_color_pair_index = 2  # Start from 1, as 0 is the default color pair
 	__curses_color_table = {}
 	__curses_current_color_index = 10
+	__max_connections_nofile_limit_supported = 0
+	__thread_start_delay = 0
+	if __resource_lib_available:
+		# Get the current limits
+		_, __system_nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+		# Set the soft limit to the hard limit
+		resource.setrlimit(resource.RLIMIT_NOFILE, (__system_nofile_limit, __system_nofile_limit))
+		__max_connections_nofile_limit_supported = int((__system_nofile_limit - 10) / 3)
 
 # Mapping of ANSI 4-bit colors to curses colors
-ANSI_TO_CURSES_COLOR = {
-	30: curses.COLOR_BLACK,
-	31: curses.COLOR_RED,
-	32: curses.COLOR_GREEN,
-	33: curses.COLOR_YELLOW,
-	34: curses.COLOR_BLUE,
-	35: curses.COLOR_MAGENTA,
-	36: curses.COLOR_CYAN,
-	37: curses.COLOR_WHITE,
-	90: curses.COLOR_BLACK,   # Bright Black (usually gray)
-	91: curses.COLOR_RED,     # Bright Red
-	92: curses.COLOR_GREEN,   # Bright Green
-	93: curses.COLOR_YELLOW,  # Bright Yellow
-	94: curses.COLOR_BLUE,    # Bright Blue
-	95: curses.COLOR_MAGENTA, # Bright Magenta
-	96: curses.COLOR_CYAN,    # Bright Cyan
-	97: curses.COLOR_WHITE    # Bright White
-}
+if __curses_available:
+	ANSI_TO_CURSES_COLOR = {
+		30: curses.COLOR_BLACK,
+		31: curses.COLOR_RED,
+		32: curses.COLOR_GREEN,
+		33: curses.COLOR_YELLOW,
+		34: curses.COLOR_BLUE,
+		35: curses.COLOR_MAGENTA,
+		36: curses.COLOR_CYAN,
+		37: curses.COLOR_WHITE,
+		90: curses.COLOR_BLACK,   # Bright Black (usually gray)
+		91: curses.COLOR_RED,     # Bright Red
+		92: curses.COLOR_GREEN,   # Bright Green
+		93: curses.COLOR_YELLOW,  # Bright Yellow
+		94: curses.COLOR_BLUE,    # Bright Blue
+		95: curses.COLOR_MAGENTA, # Bright Magenta
+		96: curses.COLOR_CYAN,    # Bright Cyan
+		97: curses.COLOR_WHITE    # Bright White
+	}
 # ------------ Exportable Help Functions ----------------
 # check if command sshpass is available
 _binPaths = {}
@@ -1150,7 +1167,7 @@ def __handle_writing_stream(stream,stop_event,host):
 	#     host.stdout.append(' $ ' + ''.join(__keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
 	return sentInput
 	
-def run_command(host, sem, timeout=60,passwds=None):
+def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
 	'''
 	Run the command on the host. Will format the commands accordingly. Main execution function.
 
@@ -1168,6 +1185,11 @@ def run_command(host, sem, timeout=60,passwds=None):
 	global __ipmiiInterfaceIPPrefix
 	global _binPaths
 	global __DEBUG_MODE
+	if retry_limit < 0:
+		host.output.append('Error: Retry limit reached!')
+		host.stderr.append('Error: Retry limit reached!')
+		host.returncode = 1
+		return
 	try:
 		localExtraArgs = []
 		
@@ -1244,7 +1266,7 @@ def run_command(host, sem, timeout=60,passwds=None):
 					host.command = 'ipmitool power status'
 				else:
 					host.command = 'ipmitool '+host.command if not host.command.startswith('ipmitool ') else host.command
-				run_command(host,sem,timeout,passwds)
+				run_command(host,sem,timeout,passwds,retry_limit=retry_limit - 1)
 				return
 			else:
 				host.output.append('Ipmitool not found on the local machine! Please install ipmitool to use ipmi mode.')
@@ -1263,7 +1285,7 @@ def run_command(host, sem, timeout=60,passwds=None):
 					host.stderr.append('shell not found on the local machine! Using ssh localhost instead...')
 				host.shell = False
 				host.name = 'localhost'
-				run_command(host,sem,timeout,passwds)
+				run_command(host,sem,timeout,passwds,retry_limit=retry_limit - 1)
 		else:
 			if host.files:
 				if host.scp:
@@ -1404,6 +1426,16 @@ def run_command(host, sem, timeout=60,passwds=None):
 			if host.stderr:
 				# filter out the error messages that we want to ignore
 				host.stderr = [line for line in host.stderr if not __ERROR_MESSAGES_TO_IGNORE_REGEX.search(line)]
+		# except os error too many open files
+		except OSError as e:
+			if e.errno == 24:  # Errno 24 corresponds to "Too many open files"
+				host.output.append("Warning: Too many open files. retrying...")
+				# Handle the error, e.g., clean up, retry logic, or exit
+				time.sleep(0.1)
+				run_command(host,sem,timeout,passwds,retry_limit=retry_limit - 1)
+			else:
+				# Re-raise the exception if it's not the specific one
+				raise
 		except Exception as e:
 			import traceback
 			host.stderr.extend(str(e).split('\n'))
@@ -1420,7 +1452,7 @@ def run_command(host, sem, timeout=60,passwds=None):
 		host.ipmi = False
 		host.interface_ip_prefix = None
 		host.command = 'ipmitool '+host.command if not host.command.startswith('ipmitool ') else host.command
-		run_command(host,sem,timeout,passwds)
+		run_command(host,sem,timeout,passwds,retry_limit=retry_limit - 1)
 	# If transfering files, we will try again using scp if rsync connection is not successful
 	if host.files and not host.scp and not useScp and host.returncode != 0 and host.stderr:
 		host.stderr = []
@@ -1429,7 +1461,7 @@ def run_command(host, sem, timeout=60,passwds=None):
 		if __DEBUG_MODE:
 			host.stderr.append('Rsync connection failed! Trying SCP connection...')
 		host.scp = True
-		run_command(host,sem,timeout,passwds)
+		run_command(host,sem,timeout,passwds,retry_limit=retry_limit - 1)
 
 # ------------ Start Threading Block ----------------
 def start_run_on_hosts(hosts, timeout=60,password=None,max_connections=4 * os.cpu_count()):
@@ -1445,12 +1477,14 @@ def start_run_on_hosts(hosts, timeout=60,password=None,max_connections=4 * os.cp
 	Returns:
 		list: A list of threads that get started
 	'''
+	global __thread_start_delay
 	if len(hosts) == 0:
 		return []
 	sem = threading.Semaphore(max_connections)  # Limit concurrent SSH sessions
 	threads = [threading.Thread(target=run_command, args=(host, sem,timeout,password), daemon=True) for host in hosts]
 	for thread in threads:
 		thread.start()
+		time.sleep(__thread_start_delay)
 	return threads
 
 # ------------ Display Block ----------------
@@ -2405,6 +2439,8 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
 	global _no_env
 	global _emo
 	global __DEBUG_MODE
+	global __thread_start_delay
+	global __max_connections_nofile_limit_supported
 	_emo = False
 	_no_env = no_env
 	if os.path.exists(os.path.join(tempfile.gettempdir(),'__multiSSH3_UNAVAILABLE_HOSTS.csv')):
@@ -2437,9 +2473,16 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
 	if not max_connections:
 		max_connections = 4 * os.cpu_count()
 	elif max_connections == 0:
-		max_connections = 1048576
+		max_connections = __max_connections_nofile_limit_supported
 	elif max_connections < 0:
 		max_connections = (-max_connections) * os.cpu_count()
+	if __max_connections_nofile_limit_supported > 0:
+		if max_connections > __max_connections_nofile_limit_supported:
+			eprint(f"Warning: The number of maximum connections {max_connections} is larger than estimated limit {__max_connections_nofile_limit_supported} from ulimit nofile limit {__system_nofile_limit}, setting the maximum connections to {__max_connections_nofile_limit_supported}.")
+			max_connections = __max_connections_nofile_limit_supported
+		if max_connections > __max_connections_nofile_limit_supported * 2:
+			# we need to throttle thread start to avoid hitting the nofile limit
+			__thread_start_delay = 0.001
 	if not commands:
 		commands = []
 	else:

{multissh3-5.39 → multissh3-5.41}/setup.py

@@ -1,8 +1,9 @@
 from setuptools import setup
+from multiSSH3 import version
 
 setup(
     name='multiSSH3',
-    version='5.39',
+    version=version,
     description='Run commands on multiple hosts via SSH',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',