multiSSH3 4.75__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of multiSSH3 might be problematic. Click here for more details.

multiSSH3.py ADDED
@@ -0,0 +1,1509 @@
1
+ #!/usr/bin/env python3
2
+ import curses
3
+ import subprocess
4
+ import threading
5
+ import time,os
6
+ import argparse
7
+ from itertools import product
8
+ import re
9
+ import string
10
+ import ipaddress
11
+ import sys
12
+ import json
13
+ import socket
14
+ import io
15
+ import signal
16
+ import functools
17
+ import glob
18
+ try:
19
+ # Check if functiools.cache is available
20
+ cache_decorator = functools.cache
21
+ except AttributeError:
22
+ try:
23
+ # Check if functools.lru_cache is available
24
+ cache_decorator = functools.lru_cache(maxsize=None)
25
+ except AttributeError:
26
+ # If neither is available, use a dummy decorator
27
+ def cache_decorator(func):
28
+ return func
29
+
30
+ version = '4.75'
31
+ VERSION = version
32
+
33
+ DEFAULT_ENV_FILE = '/etc/profile.d/hosts.sh'
34
+ DEFAULT_USERNAME = None
35
+ DEFAULT_EXTRA_ARGS = None
36
+ DEFAULT_PASSWORD = ''
37
+ DEFAULT_ONE_ON_ONE = False
38
+ DEFAULT_FILE_SYNC = False
39
+ DEFAULT_SCP = False
40
+ DEFAULT_TIMEOUT = 50
41
+ DEFAULT_REPEAT = 1
42
+ DEFAULT_INTERVAL = 0
43
+ DEFAULT_IPMI = False
44
+ DEFAULT_INTERFACE_IP_PREFIX = None
45
+ DEFAULT_IPMI_INTERFACE_IP_PREFIX = None
46
+ DEFAULT_QUIET = False
47
+ DEFAULT_ERROR_ONLY = False
48
+ DEFAULT_NO_OUTPUT = False
49
+ DEFAULT_NO_ENV = False
50
+ DEFAULT_MAX_CONNECTIONS = 4 * os.cpu_count()
51
+ DEFAULT_JSON_MODE = False
52
+ DEFAULT_PRINT_SUCCESS_HOSTS = False
53
+ DEFAULT_GREPPABLE_MODE = False
54
+ DEFAULT_NO_WATCH = False
55
+ DEFAULT_SKIP_UNREACHABLE = False
56
+ DEFAULT_SKIP_HOSTS = ''
57
+ DEFAULT_CURSES_MINIMUM_CHAR_LEN = 40
58
+ DEFAULT_CURSES_MINIMUM_LINE_LEN = 1
59
+ DEFAULT_SINGLE_WINDOW = False
60
+
61
+ DEFAULT_CALLED = True
62
+ DEFAULT_RETURN_UNFINISHED = False
63
+ DEFAULT_UPDATE_UNREACHABLE_HOSTS = True
64
+ DEFAULT_NO_START = False
65
+
66
+ global_suppress_printout = True
67
+
68
+ mainReturnCode = 0
69
+ failedHosts = set()
70
+ class Host:
71
+ def __init__(self, name, command, files = None,ipmi = False,interface_ip_prefix = None,scp=False,extraargs=None):
72
+ self.name = name # the name of the host (hostname or IP address)
73
+ self.command = command # the command to run on the host
74
+ self.returncode = None # the return code of the command
75
+ self.output = [] # the output of the command for curses
76
+ self.stdout = [] # the stdout of the command
77
+ self.stderr = [] # the stderr of the command
78
+ self.printedLines = -1 # the number of lines printed on the screen
79
+ self.files = files # the files to be copied to the host
80
+ self.ipmi = ipmi # whether to use ipmi to connect to the host
81
+ self.interface_ip_prefix = interface_ip_prefix # the prefix of the ip address of the interface to be used to connect to the host
82
+ self.scp = scp # whether to use scp to copy files to the host
83
+ self.extraargs = extraargs # extra arguments to be passed to ssh
84
+ self.resolvedName = None # the resolved IP address of the host
85
+ def __iter__(self):
86
+ return zip(['name', 'command', 'returncode', 'stdout', 'stderr'], [self.name, self.command, self.returncode, self.stdout, self.stderr])
87
+ def __repr__(self):
88
+ # return the complete data structure
89
+ return f"Host(name={self.name}, command={self.command}, returncode={self.returncode}, stdout={self.stdout}, stderr={self.stderr}, output={self.output}, printedLines={self.printedLines}, files={self.files}, ipmi={self.ipmi}, interface_ip_prefix={self.interface_ip_prefix}, scp={self.scp}, extraargs={self.extraargs}, resolvedName={self.resolvedName})"
90
+ def __str__(self):
91
+ return f"Host(name={self.name}, command={self.command}, returncode={self.returncode}, stdout={self.stdout}, stderr={self.stderr})"
92
+
93
+ wildCharacters = ['*','?','x']
94
+
95
+ gloablUnavailableHosts = set()
96
+
97
+ ipmiiInterfaceIPPrefix = DEFAULT_IPMI_INTERFACE_IP_PREFIX
98
+
99
+ keyPressesIn = [[]]
100
+
101
+ emo = False
102
+
103
+ etc_hosts = {}
104
+
105
+ env_file = DEFAULT_ENV_FILE
106
+
107
+ # check if command sshpass is available
108
+ sshpassAvailable = False
109
+ try:
110
+ subprocess.run(['which', 'sshpass'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
111
+ sshpassAvailable = True
112
+ except:
113
+ pass
114
+
115
+
116
+ @cache_decorator
117
+ def expandIPv4Address(hosts):
118
+ '''
119
+ Expand the IP address range in the hosts list
120
+
121
+ Args:
122
+ hosts (list): A list of IP addresses or IP address ranges
123
+
124
+ Returns:
125
+ list: A list of expanded IP addresses
126
+ '''
127
+ expandedHosts = []
128
+ expandedHost = []
129
+ for host in hosts:
130
+ host = host.replace('[','').replace(']','')
131
+ octets = host.split('.')
132
+ expandedOctets = []
133
+ for octet in octets:
134
+ if '-' in octet:
135
+ # Handle wildcards
136
+ octetRange = octet.split('-')
137
+ for i in range(len(octetRange)):
138
+ if not octetRange[i] or octetRange[i] in wildCharacters:
139
+ if i == 0:
140
+ octetRange[i] = '0'
141
+ elif i == 1:
142
+ octetRange[i] = '255'
143
+
144
+ expandedOctets.append([str(i) for i in range(int(octetRange[0]),int(octetRange[1])+1)])
145
+ elif octet in wildCharacters:
146
+ expandedOctets.append([str(i) for i in range(0,256)])
147
+ else:
148
+ expandedOctets.append([octet])
149
+ # handle the first and last subnet addresses
150
+ if '0' in expandedOctets[-1]:
151
+ expandedOctets[-1].remove('0')
152
+ if '255' in expandedOctets[-1]:
153
+ expandedOctets[-1].remove('255')
154
+ #print(expandedOctets)
155
+ # Generate the expanded hosts
156
+ for ip in list(product(expandedOctets[0],expandedOctets[1],expandedOctets[2],expandedOctets[3])):
157
+ expandedHost.append('.'.join(ip))
158
+ expandedHosts.extend(expandedHost)
159
+ return expandedHosts
160
+
161
+ @cache_decorator
162
+ def readEnvFromFile(environemnt_file = ''):
163
+ '''
164
+ Read the environment variables from env_file
165
+ Returns:
166
+ dict: A dictionary of environment variables
167
+ '''
168
+ global env
169
+ try:
170
+ if env:
171
+ return env
172
+ except:
173
+ env = {}
174
+ global env_file
175
+ if environemnt_file:
176
+ envf = environemnt_file
177
+ else:
178
+ envf = env_file if env_file else DEFAULT_ENV_FILE
179
+ if os.path.exists(envf):
180
+ with open(envf,'r') as f:
181
+ for line in f:
182
+ if line.startswith('#') or not line.strip():
183
+ continue
184
+ key, value = line.replace('export ', '', 1).strip().split('=', 1)
185
+ key = key.strip().strip('"').strip("'")
186
+ value = value.strip().strip('"').strip("'")
187
+ # avoid infinite recursion
188
+ if key != value:
189
+ env[key] = value.strip('"').strip("'")
190
+ return env
191
+
192
+ @cache_decorator
193
+ def getIP(hostname,local=False):
194
+ '''
195
+ Get the IP address of the hostname
196
+
197
+ Args:
198
+ hostname (str): The hostname
199
+
200
+ Returns:
201
+ str: The IP address of the hostname
202
+ '''
203
+ global etc_hosts
204
+ # First we check if the hostname is an IP address
205
+ try:
206
+ ipaddress.ip_address(hostname)
207
+ return hostname
208
+ except ValueError:
209
+ pass
210
+ # Then we check /etc/hosts
211
+ if not etc_hosts and os.path.exists('/etc/hosts'):
212
+ with open('/etc/hosts','r') as f:
213
+ for line in f:
214
+ if line.startswith('#') or not line.strip():
215
+ continue
216
+ #ip, host = line.split()[:2]
217
+ chunks = line.split()
218
+ if len(chunks) < 2:
219
+ continue
220
+ ip = chunks[0]
221
+ for host in chunks[1:]:
222
+ etc_hosts[host] = ip
223
+ if hostname in etc_hosts:
224
+ return etc_hosts[hostname]
225
+ if local:
226
+ return None
227
+ # Then we check the DNS
228
+ try:
229
+ return socket.gethostbyname(hostname)
230
+ except:
231
+ return None
232
+
233
+ @cache_decorator
234
+ def expand_hostname(text,validate=True,no_env=False):
235
+ '''
236
+ Expand the hostname range in the text.
237
+ Will search the string for a range ( [] encloused and non enclosed number ranges).
238
+ Will expand the range, validate them using validate_expand_hostname and return a list of expanded hostnames
239
+
240
+ Args:
241
+ text (str): The text to be expanded
242
+ validate (bool, optional): Whether to validate the hostname. Defaults to True.
243
+
244
+ Returns:
245
+ set: A set of expanded hostnames
246
+ '''
247
+ expandinghosts = [text]
248
+ expandedhosts = set()
249
+ # all valid alphanumeric characters
250
+ alphanumeric = string.digits + string.ascii_letters
251
+ while len(expandinghosts) > 0:
252
+ hostname = expandinghosts.pop()
253
+ match = re.search(r'\[(.*?-.*?)\]', hostname)
254
+ if not match:
255
+ expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
256
+ continue
257
+ try:
258
+ range_start, range_end = match.group(1).split('-')
259
+ except ValueError:
260
+ expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
261
+ continue
262
+ range_start = range_start.strip()
263
+ range_end = range_end.strip()
264
+ if not range_end:
265
+ if range_start.isdigit():
266
+ range_end = '9'
267
+ elif range_start.isalpha() and range_start.islower():
268
+ range_end = 'z'
269
+ elif range_start.isalpha() and range_start.isupper():
270
+ range_end = 'Z'
271
+ else:
272
+ expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
273
+ continue
274
+ if not range_start:
275
+ if range_end.isdigit():
276
+ range_start = '0'
277
+ elif range_end.isalpha() and range_end.islower():
278
+ range_start = 'a'
279
+ elif range_end.isalpha() and range_end.isupper():
280
+ range_start = 'A'
281
+ else:
282
+ expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
283
+ continue
284
+ if range_start.isdigit() and range_end.isdigit():
285
+ padding_length = min(len(range_start), len(range_end))
286
+ format_str = "{:0" + str(padding_length) + "d}"
287
+ for i in range(int(range_start), int(range_end) + 1):
288
+ formatted_i = format_str.format(i)
289
+ if '[' in hostname:
290
+ expandinghosts.append(hostname.replace(match.group(0), formatted_i, 1))
291
+ else:
292
+ expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), formatted_i, 1),no_env=no_env) if validate else [hostname])
293
+ else:
294
+ if all(c in string.hexdigits for c in range_start + range_end):
295
+ for i in range(int(range_start, 16), int(range_end, 16)+1):
296
+ if '[' in hostname:
297
+ expandinghosts.append(hostname.replace(match.group(0), format(i, 'x'), 1))
298
+ else:
299
+ expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), format(i, 'x'), 1),no_env=no_env) if validate else [hostname])
300
+ else:
301
+ try:
302
+ start_index = alphanumeric.index(range_start)
303
+ end_index = alphanumeric.index(range_end)
304
+ for i in range(start_index, end_index + 1):
305
+ if '[' in hostname:
306
+ expandinghosts.append(hostname.replace(match.group(0), alphanumeric[i], 1))
307
+ else:
308
+ expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), alphanumeric[i], 1),no_env=no_env) if validate else [hostname])
309
+ except ValueError:
310
+ expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
311
+ return expandedhosts
312
+
313
+ @cache_decorator
314
+ def expand_hostnames(hosts,no_env=False):
315
+ '''
316
+ Expand the hostnames in the hosts list
317
+
318
+ Args:
319
+ hosts (list): A list of hostnames
320
+
321
+ Returns:
322
+ list: A list of expanded hostnames
323
+ '''
324
+ expandedhosts = []
325
+ if isinstance(hosts, str):
326
+ hosts = [hosts]
327
+ for host in hosts:
328
+ host = host.strip()
329
+ if not host:
330
+ continue
331
+ # we seperate the username from the hostname
332
+ username = None
333
+ if '@' in host:
334
+ username, host = host.split('@',1)
335
+ # first we check if the hostname is an range of IP addresses
336
+ # This is done by checking if the hostname follows four fields of
337
+ # "(((\d{1,3}|x|\*|\?)(-(\d{1,3}|x|\*|\?))?)|(\[(\d{1,3}|x|\*|\?)(-(\d{1,3}|x|\*|\?))?\]))"
338
+ # seperated by .
339
+ # If so, we expand the IP address range
340
+ if re.match(r'^((((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?))?)|(\[((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])}|x|\*|\?))?\]))(\.((((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?)(-((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?))?)|(\[((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?)(-((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?))?\]))){2}(\.(((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?))?)|(\[((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])}|x|\*|\?))?\]))$', host):
341
+ hostSetToAdd = sorted(expandIPv4Address(frozenset([host])),key=ipaddress.IPv4Address)
342
+ else:
343
+ hostSetToAdd = sorted(expand_hostname(host,no_env=no_env))
344
+ if username:
345
+ # we expand the username
346
+ username = sorted(expand_hostname(username,validate=False,no_env=no_env))
347
+ # we combine the username and hostname
348
+ hostSetToAdd = [u+'@'+h for u,h in product(username,hostSetToAdd)]
349
+ expandedhosts.extend(hostSetToAdd)
350
+ return expandedhosts
351
+
352
+ @cache_decorator
353
+ def validate_expand_hostname(hostname,no_env=False):
354
+ '''
355
+ Validate the hostname and expand it if it is a range of IP addresses
356
+
357
+ Args:
358
+ hostname (str): The hostname to be validated and expanded
359
+
360
+ Returns:
361
+ list: A list of valid hostnames
362
+ '''
363
+ # maybe it is just defined in ./target_files/hosts.sh and exported to the environment
364
+ # we will try to get the valid host name from the environment
365
+ hostname = hostname.strip('$')
366
+ if getIP(hostname,local=True):
367
+ return [hostname]
368
+ elif not no_env and hostname in os.environ:
369
+ # we will expand these hostnames again
370
+ return expand_hostnames(frozenset(os.environ[hostname].split(',')),no_env=no_env)
371
+ elif hostname in readEnvFromFile():
372
+ # we will expand these hostnames again
373
+ return expand_hostnames(frozenset(readEnvFromFile()[hostname].split(',')),no_env=no_env)
374
+ elif getIP(hostname,local=False):
375
+ return [hostname]
376
+ else:
377
+ print(f"Error: {hostname} is not a valid hostname or IP address!")
378
+ global mainReturnCode
379
+ mainReturnCode += 1
380
+ global failedHosts
381
+ failedHosts.add(hostname)
382
+ return []
383
+
384
+ def input_with_timeout_and_countdown(timeout, prompt='Please enter your selection'):
385
+ """
386
+ Read an input from the user with a timeout and a countdown.
387
+
388
+ Parameters:
389
+ timeout (int): The timeout value in seconds.
390
+ prompt (str): The prompt message to display to the user. Default is 'Please enter your selection'.
391
+
392
+ Returns:
393
+ str or None: The user input if received within the timeout, or None if no input is received.
394
+ """
395
+ import select
396
+ # Print the initial prompt with the countdown
397
+ print(f"{prompt} [{timeout}s]: ", end='', flush=True)
398
+ # Loop until the timeout
399
+ for remaining in range(timeout, 0, -1):
400
+ # If there is an input, return it
401
+ if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
402
+ return input().strip()
403
+ # Print the remaining time
404
+ print(f"\r{prompt} [{remaining}s]: ", end='', flush=True)
405
+ # Wait a second
406
+ time.sleep(1)
407
+ # If there is no input, return None
408
+ return None
409
+
410
+ def handle_reading_stream(stream,target, host):
411
+ '''
412
+ Read the stream and append the lines to the target list
413
+
414
+ Args:
415
+ stream (io.BytesIO): The stream to be read
416
+ target (list): The list to append the lines to
417
+ host (Host): The host object
418
+
419
+ Returns:
420
+ None
421
+ '''
422
+ def add_line(current_line,target, host, keepLastLine=True):
423
+ if not keepLastLine:
424
+ target.pop()
425
+ host.output.pop()
426
+ host.printedLines -= 1
427
+ current_line_str = current_line.decode('utf-8',errors='backslashreplace')
428
+ target.append(current_line_str)
429
+ host.output.append(current_line_str)
430
+ current_line = bytearray()
431
+ lastLineCommited = True
432
+ for char in iter(lambda:stream.read(1), b''):
433
+ if char == b'\n':
434
+ if (not lastLineCommited) and current_line:
435
+ add_line(current_line,target, host, keepLastLine=False)
436
+ elif lastLineCommited:
437
+ add_line(current_line,target, host, keepLastLine=True)
438
+ current_line = bytearray()
439
+ lastLineCommited = True
440
+ elif char == b'\r':
441
+ add_line(current_line,target, host, keepLastLine=lastLineCommited)
442
+ current_line = bytearray()
443
+ lastLineCommited = False
444
+ else:
445
+ current_line.extend(char)
446
+ if current_line:
447
+ add_line(current_line,target, host, keepLastLine=lastLineCommited)
448
+
449
+ def handle_writing_stream(stream,stop_event,host):
450
+ '''
451
+ Write the key presses to the stream
452
+
453
+ Args:
454
+ stream (io.BytesIO): The stream to be written to
455
+ stop_event (threading.Event): The event to stop the thread
456
+ host (Host): The host object
457
+
458
+ Returns:
459
+ None
460
+ '''
461
+ global keyPressesIn
462
+ # keyPressesIn is a list of lists.
463
+ # Each list is a list of characters to be sent to the stdin of the process at once.
464
+ # We do not send the last line as it may be incomplete.
465
+ sentInput = 0
466
+ while not stop_event.is_set():
467
+ if sentInput < len(keyPressesIn) - 1 :
468
+ stream.write(''.join(keyPressesIn[sentInput]).encode())
469
+ stream.flush()
470
+ host.output.append(' $ ' + ''.join(keyPressesIn[sentInput]).encode().decode().replace('\n', '↵'))
471
+ host.stdout.append(' $ ' + ''.join(keyPressesIn[sentInput]).encode().decode().replace('\n', '↵'))
472
+ sentInput += 1
473
+ else:
474
+ time.sleep(0.1)
475
+ if sentInput < len(keyPressesIn) - 1 :
476
+ print(f"Warning: {len(keyPressesIn)-sentInput} key presses are not sent before the process is terminated!")
477
+ # # send the last line
478
+ # if keyPressesIn and keyPressesIn[-1]:
479
+ # stream.write(''.join(keyPressesIn[-1]).encode())
480
+ # stream.flush()
481
+ # host.output.append(' $ ' + ''.join(keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
482
+ # host.stdout.append(' $ ' + ''.join(keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
483
+ return sentInput
484
+
485
+ def ssh_command(host, sem, timeout=60,passwds=None):
486
+ '''
487
+ Run the command on the host. Will format the commands accordingly. Main execution function.
488
+
489
+ Args:
490
+ host (Host): The host object
491
+ sem (threading.Semaphore): The semaphore to limit the number of concurrent SSH sessions
492
+ timeout (int, optional): The timeout for the command. Defaults to 60.
493
+ passwds (str, optional): The password for the host. Defaults to None.
494
+
495
+ Returns:
496
+ None
497
+ '''
498
+ global emo
499
+ with sem:
500
+ try:
501
+ host.username = None
502
+ host.address = host.name
503
+ if '@' in host.name:
504
+ host.username, host.address = host.name.rsplit('@',1)
505
+ if "#HOST#" in host.command.upper() or "#HOSTNAME#" in host.command.upper():
506
+ host.command = host.command.replace("#HOST#",host.address).replace("#HOSTNAME#",host.address).replace("#host#",host.address).replace("#hostname#",host.address)
507
+ if "#USER#" in host.command.upper() or "#USERNAME#" in host.command.upper():
508
+ if host.username:
509
+ host.command = host.command.replace("#USER#",host.username).replace("#USERNAME#",host.username).replace("#user#",host.username).replace("#username#",host.username)
510
+ else:
511
+ host.command = host.command.replace("#USER#",'CURRENT_USER').replace("#USERNAME#",'CURRENT_USER').replace("#user#",'CURRENT_USER').replace("#username#",'CURRENT_USER')
512
+ formatedCMD = []
513
+ if host.extraargs:
514
+ extraargs = host.extraargs.split()
515
+ else:
516
+ extraargs = []
517
+ if ipmiiInterfaceIPPrefix:
518
+ host.interface_ip_prefix = ipmiiInterfaceIPPrefix if host.ipmi and not host.interface_ip_prefix else host.interface_ip_prefix
519
+ if host.interface_ip_prefix:
520
+ try:
521
+ hostOctets = getIP(host.address,local=False).split('.')
522
+ prefixOctets = host.interface_ip_prefix.split('.')
523
+ host.address = '.'.join(prefixOctets[:3]+hostOctets[min(3,len(prefixOctets)):])
524
+ host.resolvedName = host.username + '@' if host.username else ''
525
+ host.resolvedName += host.address
526
+ except:
527
+ host.resolvedName = host.name
528
+ else:
529
+ host.resolvedName = host.name
530
+ if host.ipmi:
531
+ if host.command.startswith('ipmitool '):
532
+ host.command = host.command.replace('ipmitool ','')
533
+ if not host.username:
534
+ host.username = 'admin'
535
+ if passwds:
536
+ formatedCMD = ['bash','-c',f'ipmitool -H {host.address} -U {host.username} -P {passwds} {" ".join(extraargs)} {host.command}']
537
+ else:
538
+ formatedCMD = ['bash','-c',f'ipmitool -H {host.address} -U {host.username} {" ".join(extraargs)} {host.command}']
539
+ else:
540
+ if host.files:
541
+ if host.scp:
542
+ formatedCMD = ['scp','-rpB'] + extraargs +['--']+host.files+[f'{host.resolvedName}:{host.command}']
543
+ else:
544
+ formatedCMD = ['rsync','-ahlX','--partial','--inplace', '--info=name'] + extraargs +['--']+host.files+[f'{host.resolvedName}:{host.command}']
545
+ else:
546
+ formatedCMD = ['ssh'] + extraargs +['--']+ [host.resolvedName, host.command]
547
+ if passwds and sshpassAvailable:
548
+ formatedCMD = ['sshpass', '-p', passwds] + formatedCMD
549
+ elif passwds:
550
+ host.output.append('Warning: sshpass is not available. Please install sshpass to use password authentication.')
551
+ #host.stderr.append('Warning: sshpass is not available. Please install sshpass to use password authentication.')
552
+ host.output.append('Please provide password via live input or use ssh key authentication.')
553
+ # # try to send the password via keyPressesIn
554
+ # keyPressesIn[-1] = list(passwds) + ['\n']
555
+ # keyPressesIn.append([])
556
+ host.output.append('Running command: '+' '.join(formatedCMD))
557
+ #host.stdout = []
558
+ proc = subprocess.Popen(formatedCMD,stdout=subprocess.PIPE,stderr=subprocess.PIPE,stdin=subprocess.PIPE)
559
+ # create a thread to handle stdout
560
+ stdout_thread = threading.Thread(target=handle_reading_stream, args=(proc.stdout,host.stdout, host), daemon=True)
561
+ stdout_thread.start()
562
+ # create a thread to handle stderr
563
+ #host.stderr = []
564
+ stderr_thread = threading.Thread(target=handle_reading_stream, args=(proc.stderr,host.stderr, host), daemon=True)
565
+ stderr_thread.start()
566
+ # create a thread to handle stdin
567
+ stdin_stop_event = threading.Event()
568
+ stdin_thread = threading.Thread(target=handle_writing_stream, args=(proc.stdin,stdin_stop_event, host), daemon=True)
569
+ stdin_thread.start()
570
+ # Monitor the subprocess and terminate it after the timeout
571
+ start_time = time.time()
572
+ outLength = len(host.output)
573
+ while proc.poll() is None: # while the process is still running
574
+ if len(host.output) > outLength:
575
+ start_time = time.time()
576
+ outLength = len(host.output)
577
+ if timeout > 0:
578
+ if time.time() - start_time > timeout:
579
+ host.stderr.append('Timeout!')
580
+ host.output.append('Timeout!')
581
+ proc.send_signal(signal.SIGINT)
582
+ time.sleep(0.1)
583
+
584
+ proc.terminate()
585
+ break
586
+ elif time.time() - start_time > min(10, timeout // 2):
587
+ timeoutLine = f'Timeout in [{timeout - int(time.time() - start_time)}] seconds!'
588
+ if host.output and not host.output[-1].strip().startswith(timeoutLine):
589
+ # remove last line if it is a countdown
590
+ if host.output and host.output[-1].strip().endswith('] seconds!') and host.output[-1].strip().startswith('Timeout in ['):
591
+ host.output.pop()
592
+ host.printedLines -= 1
593
+ host.output.append(timeoutLine)
594
+ outLength = len(host.output)
595
+ if emo:
596
+ host.stderr.append('Ctrl C detected, Emergency Stop!')
597
+ host.output.append('Ctrl C detected, Emergency Stop!')
598
+ proc.send_signal(signal.SIGINT)
599
+ time.sleep(0.1)
600
+ proc.terminate()
601
+ break
602
+ time.sleep(0.1) # avoid busy-waiting
603
+ stdin_stop_event.set()
604
+ # Wait for output processing to complete
605
+ stdout_thread.join(timeout=1)
606
+ stderr_thread.join(timeout=1)
607
+ stdin_thread.join(timeout=1)
608
+ # here we handle the rest of the stdout after the subprocess returns
609
+ host.output.append(f'Pipe Closed. Trying to read the rest of the stdout...')
610
+ if not emo:
611
+ stdout = None
612
+ stderr = None
613
+ try:
614
+ stdout, stderr = proc.communicate(timeout=1)
615
+ except subprocess.TimeoutExpired:
616
+ pass
617
+ if stdout:
618
+ handle_reading_stream(io.BytesIO(stdout),host.stdout, host)
619
+ if stderr:
620
+ handle_reading_stream(io.BytesIO(stderr),host.stderr, host)
621
+ # if the last line in host.stderr is Connection to * closed., we will remove it
622
+ host.returncode = proc.poll()
623
+ if not host.returncode:
624
+ # process been killed via timeout or sigkill
625
+ if host.stderr and host.stderr[-1].strip().startswith('Timeout!'):
626
+ host.returncode = 124
627
+ elif host.stderr and host.stderr[-1].strip().startswith('Ctrl C detected, Emergency Stop!'):
628
+ host.returncode = 137
629
+ host.output.append(f'Command finished with return code {host.returncode}')
630
+ if host.stderr and host.stderr[-1].strip().startswith('Connection to ') and host.stderr[-1].strip().endswith(' closed.'):
631
+ host.stderr.pop()
632
+ except Exception as e:
633
+ import traceback
634
+ host.stderr.extend(str(e).split('\n'))
635
+ host.output.extend(str(e).split('\n'))
636
+ host.stderr.extend(traceback.format_exc().split('\n'))
637
+ host.output.extend(traceback.format_exc().split('\n'))
638
+ host.returncode = -1
639
+ # If using ipmi, we will try again using ssh if ipmi connection is not successful
640
+ if host.ipmi and host.returncode != 0 and any(['Unable to establish IPMI' in line for line in host.stderr]):
641
+ host.stderr = []
642
+ host.output.append('IPMI connection failed! Trying SSH connection...')
643
+ host.ipmi = False
644
+ host.interface_ip_prefix = None
645
+ host.command = 'ipmitool '+host.command if not host.command.startswith('ipmitool ') else host.command
646
+ ssh_command(host,sem,timeout,passwds)
647
+ # If transfering files, we will try again using scp if rsync connection is not successful
648
+ if host.files and not host.scp and host.returncode != 0 and host.stderr:
649
+ host.stderr = []
650
+ host.stdout = []
651
+ host.output.append('Rsync connection failed! Trying SCP connection...')
652
+ host.scp = True
653
+ ssh_command(host,sem,timeout,passwds)
654
+
655
+ def start_run_on_hosts(hosts, timeout=60,password=None,max_connections=4 * os.cpu_count()):
656
+ '''
657
+ Start running the command on the hosts. Wrapper function for ssh_command
658
+
659
+ Args:
660
+ hosts (list): A list of Host objects
661
+ timeout (int, optional): The timeout for the command. Defaults to 60.
662
+ password (str, optional): The password for the hosts. Defaults to None.
663
+ max_connections (int, optional): The maximum number of concurrent SSH sessions. Defaults to 4 * os.cpu_count().
664
+
665
+ Returns:
666
+ list: A list of threads that get started
667
+ '''
668
+ if len(hosts) == 0:
669
+ return []
670
+ sem = threading.Semaphore(max_connections) # Limit concurrent SSH sessions
671
+ threads = [threading.Thread(target=ssh_command, args=(host, sem,timeout,password), daemon=True) for host in hosts]
672
+ for thread in threads:
673
+ thread.start()
674
+ return threads
675
+
676
+ def get_hosts_to_display (hosts, max_num_hosts, hosts_to_display = None):
677
+ '''
678
+ Generate a list for the hosts to be displayed on the screen. This is used to display as much relevant information as possible.
679
+
680
+ Args:
681
+ hosts (list): A list of Host objects
682
+ max_num_hosts (int): The maximum number of hosts to be displayed
683
+ hosts_to_display (list, optional): The hosts that are currently displayed. Defaults to None.
684
+
685
+ Returns:
686
+ list: A list of Host objects to be displayed
687
+ '''
688
+ # We will sort the hosts by running -> failed -> finished -> waiting
689
+ # running: returncode is None and output is not empty (output will be appened immediately after the command is run)
690
+ # failed: returncode is not None and returncode is not 0
691
+ # finished: returncode is not None and returncode is 0
692
+ # waiting: returncode is None and output is empty
693
+ running_hosts = [host for host in hosts if host.returncode is None and host.output]
694
+ failed_hosts = [host for host in hosts if host.returncode is not None and host.returncode != 0]
695
+ finished_hosts = [host for host in hosts if host.returncode is not None and host.returncode == 0]
696
+ waiting_hosts = [host for host in hosts if host.returncode is None and not host.output]
697
+ new_hosts_to_display = (running_hosts + failed_hosts + finished_hosts + waiting_hosts)[:max_num_hosts]
698
+ if not hosts_to_display:
699
+ return new_hosts_to_display , {'running':len(running_hosts), 'failed':len(failed_hosts), 'finished':len(finished_hosts), 'waiting':len(waiting_hosts)}
700
+ # we will compare the new_hosts_to_display with the old one, if some hosts are not in their original position, we will change its printedLines to 0
701
+ for i, host in enumerate(new_hosts_to_display):
702
+ if host not in hosts_to_display:
703
+ host.printedLines = 0
704
+ elif i != hosts_to_display.index(host):
705
+ host.printedLines = 0
706
+ return new_hosts_to_display , {'running':len(running_hosts), 'failed':len(failed_hosts), 'finished':len(finished_hosts), 'waiting':len(waiting_hosts)}
707
+
708
+ def generate_display(stdscr, hosts, threads,lineToDisplay = -1,curserPosition = 0, min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window=DEFAULT_SINGLE_WINDOW):
709
+ try:
710
+ org_dim = stdscr.getmaxyx()
711
+ new_configured = True
712
+ # To do this, first we need to know the size of the terminal
713
+ max_y, max_x = org_dim
714
+ # we will use one line to print the aggregated stats for the hosts.
715
+ max_y -= 1
716
+ # bound the min_char_len and min_line_len to between 1 and the max_x -1 and max_y -1
717
+ min_char_len_local = min(max(1,min_char_len),max_x-1)
718
+ min_line_len_local = min(max(1,min_line_len),max_y-1)
719
+ if single_window:
720
+ min_char_len_local = max_x-1
721
+ min_line_len_local = max_y-1
722
+ # raise zero division error if the terminal is too small
723
+ if max_x < 2 or max_y < 2:
724
+ raise ZeroDivisionError
725
+ if min_char_len_local < 1 or min_line_len_local < 1:
726
+ raise ZeroDivisionError
727
+ # We need to figure out how many hosts we can fit in the terminal
728
+ # We will need at least 2 lines per host, one for its name, one for its output
729
+ # Each line will be at least 61 characters long (60 for the output, 1 for the borders)
730
+ max_num_hosts_x = max_x // (min_char_len_local + 1)
731
+ max_num_hosts_y = max_y // (min_line_len_local + 1)
732
+ max_num_hosts = max_num_hosts_x * max_num_hosts_y
733
+ if max_num_hosts < 1:
734
+ raise ZeroDivisionError
735
+ hosts_to_display , host_stats = get_hosts_to_display(hosts, max_num_hosts)
736
+ if len(hosts_to_display) == 0:
737
+ raise ZeroDivisionError
738
+ # Now we calculate the actual number of hosts we will display for x and y
739
+ optimal_len_x = max(min_char_len_local, 80)
740
+ num_hosts_x = max(min(max_num_hosts_x, max_x // optimal_len_x),1)
741
+ num_hosts_y = len(hosts_to_display) // num_hosts_x
742
+ while num_hosts_y > max_num_hosts_y:
743
+ num_hosts_x += 1
744
+ # round up for num_hosts_y
745
+ num_hosts_y = len(hosts_to_display) // num_hosts_x + (len(hosts_to_display) % num_hosts_x > 0)
746
+ if num_hosts_x > max_num_hosts_x:
747
+ num_hosts_x = 1
748
+ num_hosts_y = len(hosts_to_display)
749
+ while num_hosts_y > max_num_hosts_y:
750
+ num_hosts_x += 1
751
+ num_hosts_y = len(hosts_to_display) // num_hosts_x + (len(hosts_to_display) % num_hosts_x > 0)
752
+ break
753
+
754
+ # We calculate the size of each window
755
+ host_window_height = max_y // num_hosts_y
756
+ host_window_width = max_x // num_hosts_x
757
+ if host_window_height < 1 or host_window_width < 1:
758
+ raise ZeroDivisionError
759
+
760
+ old_stat = ''
761
+ old_bottom_stat = ''
762
+ old_cursor_position = -1
763
+ # we refresh the screen every 0.1 seconds
764
+ last_refresh_time = time.perf_counter()
765
+ stdscr.clear()
766
+ #host_window.refresh()
767
+ global keyPressesIn
768
+ stdscr.nodelay(True)
769
+ # we generate a stats window at the top of the screen
770
+ stat_window = curses.newwin(1, max_x, 0, 0)
771
+ # We create a window for each host
772
+ host_windows = []
773
+ for i, host in enumerate(hosts_to_display):
774
+ # We calculate the coordinates of the window
775
+ # We need to add 1 to y for the stats line
776
+ y = (i // num_hosts_x) * host_window_height +1
777
+ x = (i % num_hosts_x) * host_window_width
778
+ #print(f"Creating a window at {y},{x}")
779
+ # We create the window
780
+ host_window = curses.newwin(host_window_height, host_window_width, y, x)
781
+ host_windows.append(host_window)
782
+ # If there is space left, we will draw the bottom border
783
+ bottom_border = None
784
+ if y + host_window_height < org_dim[0]:
785
+ bottom_border = curses.newwin(1, max_x, y + host_window_height, 0)
786
+ bottom_border.clear()
787
+ bottom_border.addstr(0, 0, '-' * (max_x - 1))
788
+ bottom_border.refresh()
789
+ while host_stats['running'] > 0 or host_stats['waiting'] > 0:
790
+ # Check for keypress
791
+ key = stdscr.getch()
792
+ if key != -1: # -1 means no keypress
793
+ # we store the keypresses in a list of lists.
794
+ # Each list is a list of characters to be sent to the stdin of the process at once.
795
+ # When we encounter a newline, we add a new list to the list of lists. ( a new line of input )
796
+ # with open('keylog.txt','a') as f:
797
+ # f.write(str(key)+'\n')
798
+ if key == 410: # 410 is the key code for resize
799
+ raise Exception('Terminal size changed. Please reconfigure window.')
800
+ # We handle positional keys
801
+ # uparrow: 259; downarrow: 258; leftarrow: 260; rightarrow: 261
802
+ # pageup: 339; pagedown: 338; home: 262; end: 360
803
+ elif key in [259, 258, 260, 261, 339, 338, 262, 360]:
804
+ # if the key is up arrow, we will move the line to display up
805
+ if key == 259: # 259 is the key code for up arrow
806
+ lineToDisplay = max(lineToDisplay - 1, -len(keyPressesIn))
807
+ # if the key is down arrow, we will move the line to display down
808
+ elif key == 258: # 258 is the key code for down arrow
809
+ lineToDisplay = min(lineToDisplay + 1, -1)
810
+ # if the key is left arrow, we will move the cursor left
811
+ elif key == 260: # 260 is the key code for left arrow
812
+ curserPosition = min(max(curserPosition - 1, 0), len(keyPressesIn[lineToDisplay]) -1)
813
+ # if the key is right arrow, we will move the cursor right
814
+ elif key == 261: # 261 is the key code for right arrow
815
+ curserPosition = max(min(curserPosition + 1, len(keyPressesIn[lineToDisplay])), 0)
816
+ # if the key is page up, we will move the line to display up by 5 lines
817
+ elif key == 339: # 339 is the key code for page up
818
+ lineToDisplay = max(lineToDisplay - 5, -len(keyPressesIn))
819
+ # if the key is page down, we will move the line to display down by 5 lines
820
+ elif key == 338: # 338 is the key code for page down
821
+ lineToDisplay = min(lineToDisplay + 5, -1)
822
+ # if the key is home, we will move the cursor to the beginning of the line
823
+ elif key == 262: # 262 is the key code for home
824
+ curserPosition = 0
825
+ # if the key is end, we will move the cursor to the end of the line
826
+ elif key == 360: # 360 is the key code for end
827
+ curserPosition = len(keyPressesIn[lineToDisplay])
828
+ # We are left with these are keys that mofidy the current line.
829
+ else:
830
+ # This means the user have done scrolling and is committing to modify the current line.
831
+ if lineToDisplay < -1:
832
+ # We overwrite the last line (current working line) with the line to display, removing the newline at the end
833
+ keyPressesIn[-1] = keyPressesIn[lineToDisplay][:-1]
834
+ lineToDisplay = -1
835
+ curserPosition = max(0, min(curserPosition, len(keyPressesIn[lineToDisplay])))
836
+ if key == 10: # 10 is the key code for newline
837
+ keyPressesIn[-1].append(chr(key))
838
+ keyPressesIn.append([])
839
+ lineToDisplay = -1
840
+ curserPosition = 0
841
+ # if the key is backspace, we will remove the last character from the last list
842
+ elif key in [8,263]: # 8 is the key code for backspace
843
+ if curserPosition > 0:
844
+ keyPressesIn[lineToDisplay].pop(curserPosition - 1)
845
+ curserPosition -= 1
846
+ # if the key is ESC, we will clear the last list
847
+ elif key == 27: # 27 is the key code for ESC
848
+ keyPressesIn[-1] = []
849
+ curserPosition = 0
850
+ # ignore delete key
851
+ elif key in [127, 330]: # 330 is the key code for delete key
852
+ # delete the character at the cursor position
853
+ if curserPosition < len(keyPressesIn[lineToDisplay]):
854
+ keyPressesIn[lineToDisplay].pop(curserPosition)
855
+ else:
856
+ # if the key is not a special key, we will add it
857
+ keyPressesIn[lineToDisplay].insert(curserPosition, chr(key))
858
+ curserPosition += 1
859
+ # reconfigure when the terminal size changes
860
+ # raise Exception when max_y or max_x is changed, let parent handle reconfigure
861
+ if org_dim != stdscr.getmaxyx():
862
+ raise Exception('Terminal size changed. Please reconfigure window.')
863
+ # We generate the aggregated stats if user did not input anything
864
+ if not keyPressesIn[lineToDisplay]:
865
+ stats = '┍'+ f"Total: {len(hosts)} Running: {host_stats['running']} Failed: {host_stats['failed']} Finished: {host_stats['finished']} Waiting: {host_stats['waiting']}"[:max_x - 2].center(max_x - 2, "━")
866
+ else:
867
+ # we use the stat bar to display the key presses
868
+ encodedLine = ''.join(keyPressesIn[lineToDisplay]).encode().decode().strip('\n') + ' '
869
+ # # add the flashing indicator at the curse position
870
+ # if time.perf_counter() % 1 > 0.5:
871
+ # encodedLine = encodedLine[:curserPosition] + '█' + encodedLine[curserPosition:]
872
+ # else:
873
+ # encodedLine = encodedLine[:curserPosition] + ' ' + encodedLine[curserPosition:]
874
+ stats = '┍'+ f"Send CMD: {encodedLine}"[:max_x - 2].center(max_x - 2, "━")
875
+ if bottom_border:
876
+ bottom_stats = '└'+ f"Total: {len(hosts)} Running: {host_stats['running']} Failed: {host_stats['failed']} Finished: {host_stats['finished']} Waiting: {host_stats['waiting']}"[:max_x - 2].center(max_x - 2, "─")
877
+ if bottom_stats != old_bottom_stat:
878
+ old_bottom_stat = bottom_stats
879
+ bottom_border.clear()
880
+ bottom_border.addstr(0, 0, bottom_stats)
881
+ bottom_border.refresh()
882
+ if stats != old_stat or curserPosition != old_cursor_position:
883
+ old_stat = stats
884
+ old_cursor_position = curserPosition
885
+ # calculate the real curser position in stats as we centered the stats
886
+ if 'Send CMD: ' in stats:
887
+ curserPositionStats = min(min(curserPosition,len(encodedLine) -1) + stats.find('Send CMD: ')+len('Send CMD: '), max_x -2)
888
+ else:
889
+ curserPositionStats = max_x -2
890
+ stat_window.clear()
891
+ #stat_window.addstr(0, 0, stats)
892
+ # add the line with curser that inverses the color at the curser position
893
+ stat_window.addstr(0, 0, stats[:curserPositionStats], curses.color_pair(1))
894
+ stat_window.addstr(0, curserPositionStats, stats[curserPositionStats], curses.color_pair(2))
895
+ stat_window.addstr(0, curserPositionStats + 1, stats[curserPositionStats + 1:], curses.color_pair(1))
896
+ stat_window.refresh()
897
+ # set the maximum refresh rate to 100 Hz
898
+ if time.perf_counter() - last_refresh_time < 0.01:
899
+ time.sleep(max(0,0.01 - time.perf_counter() + last_refresh_time))
900
+ #stdscr.clear()
901
+ hosts_to_display, host_stats = get_hosts_to_display(hosts, max_num_hosts,hosts_to_display)
902
+ for host_window, host in zip(host_windows, hosts_to_display):
903
+ # we will only update the window if there is new output or the window is not fully printed
904
+ if new_configured or host.printedLines < len(host.output):
905
+ try:
906
+ host_window.clear()
907
+ # we will try to center the name of the host with ┼ at the beginning and end and ─ in between
908
+ linePrintOut = f'┼{(host.name+":["+host.command+"]")[:host_window_width - 2].center(host_window_width - 1, "─")}'.replace('\n', ' ').replace('\r', ' ').strip()
909
+ host_window.addstr(0, 0, linePrintOut)
910
+ # we will display the latest outputs of the host as much as we can
911
+ for i, line in enumerate(host.output[-(host_window_height - 1):]):
912
+ # print(f"Printng a line at {i + 1} with length of {len('│'+line[:host_window_width - 1])}")
913
+ # time.sleep(10)
914
+ linePrintOut = ('│'+line[:host_window_width - 2].replace('\n', ' ').replace('\r', ' ')).strip()
915
+ host_window.addstr(i + 1, 0, linePrintOut)
916
+ # we draw the rest of the available lines
917
+ for i in range(len(host.output), host_window_height - 1):
918
+ # print(f"Printng a line at {i + 1} with length of {len('│')}")
919
+ host_window.addstr(i + 1, 0, '│')
920
+ host.printedLines = len(host.output)
921
+ host_window.refresh()
922
+ except Exception as e:
923
+ # import traceback
924
+ # print(str(e).strip())
925
+ # print(traceback.format_exc().strip())
926
+ if org_dim != stdscr.getmaxyx():
927
+ raise Exception('Terminal size changed. Please reconfigure window.')
928
+ new_configured = False
929
+ last_refresh_time = time.perf_counter()
930
+
931
+ except ZeroDivisionError:
932
+ # terminial is too small, we skip the display
933
+ pass
934
+ except Exception as e:
935
+ stdscr.clear()
936
+ stdscr.refresh()
937
+ generate_display(stdscr, hosts, threads, lineToDisplay, curserPosition, min_char_len, min_line_len, single_window)
938
+
939
+ def curses_print(stdscr, hosts, threads, min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window = DEFAULT_SINGLE_WINDOW):
940
+ '''
941
+ Print the output of the hosts on the screen
942
+
943
+ Args:
944
+ stdscr (curses.window): The curses window to print the output
945
+ hosts (list): A list of Host objects
946
+ threads (list): A list of threads that are running the commands
947
+
948
+ Returns:
949
+ None
950
+ '''
951
+ # We create all the windows we need
952
+ # We initialize the color pair
953
+ curses.start_color()
954
+ curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
955
+ curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_WHITE)
956
+ curses.init_pair(3, curses.COLOR_RED, curses.COLOR_BLACK)
957
+ curses.init_pair(4, curses.COLOR_GREEN, curses.COLOR_BLACK)
958
+ curses.init_pair(5, curses.COLOR_YELLOW, curses.COLOR_BLACK)
959
+ curses.init_pair(6, curses.COLOR_BLUE, curses.COLOR_BLACK)
960
+ curses.init_pair(7, curses.COLOR_MAGENTA, curses.COLOR_BLACK)
961
+ curses.init_pair(8, curses.COLOR_CYAN, curses.COLOR_BLACK)
962
+ curses.init_pair(9, curses.COLOR_WHITE, curses.COLOR_RED)
963
+ curses.init_pair(10, curses.COLOR_WHITE, curses.COLOR_GREEN)
964
+ curses.init_pair(11, curses.COLOR_WHITE, curses.COLOR_YELLOW)
965
+ curses.init_pair(12, curses.COLOR_WHITE, curses.COLOR_BLUE)
966
+ curses.init_pair(13, curses.COLOR_WHITE, curses.COLOR_MAGENTA)
967
+ curses.init_pair(14, curses.COLOR_WHITE, curses.COLOR_CYAN)
968
+ curses.init_pair(15, curses.COLOR_BLACK, curses.COLOR_RED)
969
+ curses.init_pair(16, curses.COLOR_BLACK, curses.COLOR_GREEN)
970
+ curses.init_pair(17, curses.COLOR_BLACK, curses.COLOR_YELLOW)
971
+ curses.init_pair(18, curses.COLOR_BLACK, curses.COLOR_BLUE)
972
+ curses.init_pair(19, curses.COLOR_BLACK, curses.COLOR_MAGENTA)
973
+ curses.init_pair(20, curses.COLOR_BLACK, curses.COLOR_CYAN)
974
+ generate_display(stdscr, hosts, threads,min_char_len = min_char_len, min_line_len = min_line_len, single_window = single_window)
975
+
976
+ def print_output(hosts,usejson = False,quiet = False,greppable = False):
977
+ '''
978
+ Print / generate the output of the hosts to the terminal
979
+
980
+ Args:
981
+ hosts (list): A list of Host objects
982
+ usejson (bool, optional): Whether to print the output in JSON format. Defaults to False.
983
+ quiet (bool, optional): Whether to print the output. Defaults to False.
984
+
985
+ Returns:
986
+ str: The pretty output generated
987
+ '''
988
+ global keyPressesIn
989
+ global global_suppress_printout
990
+ hosts = [dict(host) for host in hosts]
991
+ if usejson:
992
+ # [print(dict(host)) for host in hosts]
993
+ #print(json.dumps([dict(host) for host in hosts],indent=4))
994
+ rtnStr = json.dumps(hosts,indent=4)
995
+ elif greppable:
996
+ outputs = {}
997
+ # transform hosts to dictionaries
998
+ for host in hosts:
999
+ hostPrintOut = f" | cmd: {host['command']} | stdout: "+'↵ '.join(host['stdout'])
1000
+ if host['stderr']:
1001
+ if host['stderr'][0].strip().startswith('ssh: connect to host '):
1002
+ host['stderr'][0] = 'SSH not reachable!'
1003
+ hostPrintOut += " | stderr: "+'↵ '.join(host['stderr'])
1004
+ hostPrintOut += f" | return_code: {host['returncode']}"
1005
+ if hostPrintOut not in outputs:
1006
+ outputs[hostPrintOut] = [host['name']]
1007
+ else:
1008
+ outputs[hostPrintOut].append(host['name'])
1009
+ rtnStr = ''
1010
+ for output, hosts in outputs.items():
1011
+ rtnStr += f"{','.join(hosts)}{output}\n"
1012
+ if keyPressesIn[-1]:
1013
+ CMDsOut = [''.join(cmd).encode('unicode_escape').decode().replace('\\n', '↵') for cmd in keyPressesIn if cmd]
1014
+ rtnStr += 'User Inputs: '+ '\nUser Inputs: '.join(CMDsOut)
1015
+ #rtnStr += '\n'
1016
+ else:
1017
+ outputs = {}
1018
+ for host in hosts:
1019
+ if global_suppress_printout:
1020
+ if host['returncode'] == 0:
1021
+ continue
1022
+ hostPrintOut = f" Command:\n {host['command']}\n"
1023
+ hostPrintOut += " stdout:\n "+'\n '.join(host['stdout'])
1024
+ if host['stderr']:
1025
+ if host['stderr'][0].strip().startswith('ssh: connect to host '):
1026
+ host['stderr'][0] = 'SSH not reachable!'
1027
+ hostPrintOut += "\n stderr:\n "+'\n '.join(host['stderr'])
1028
+ hostPrintOut += f"\n return_code: {host['returncode']}"
1029
+ if hostPrintOut not in outputs:
1030
+ outputs[hostPrintOut] = [host['name']]
1031
+ else:
1032
+ outputs[hostPrintOut].append(host['name'])
1033
+ rtnStr = ''
1034
+ for output, hosts in outputs.items():
1035
+ if global_suppress_printout:
1036
+ rtnStr += f'Error returncode produced by {hosts}:\n'
1037
+ rtnStr += output+'\n'
1038
+ else:
1039
+ rtnStr += '*'*80+'\n'
1040
+ rtnStr += f"These hosts: {hosts} have a response of:\n"
1041
+ rtnStr += output+'\n'
1042
+ if not global_suppress_printout or outputs:
1043
+ rtnStr += '*'*80+'\n'
1044
+ if keyPressesIn[-1]:
1045
+ CMDsOut = [''.join(cmd).encode('unicode_escape').decode().replace('\\n', '↵') for cmd in keyPressesIn if cmd]
1046
+ #rtnStr += f"Key presses: {''.join(keyPressesIn).encode('unicode_escape').decode()}\n"
1047
+ #rtnStr += f"Key presses: {keyPressesIn}\n"
1048
+ rtnStr += "User Inputs: \n "
1049
+ rtnStr += '\n '.join(CMDsOut)
1050
+ rtnStr += '\n'
1051
+ keyPressesIn = [[]]
1052
+ if global_suppress_printout and not outputs:
1053
+ rtnStr += 'Success'
1054
+ if not quiet:
1055
+ print(rtnStr)
1056
+ return rtnStr
1057
+
1058
+ sshConfigged = False
1059
+ def verify_ssh_config():
1060
+ '''
1061
+ Verify that ~/.ssh/config exists and contains the line "StrictHostKeyChecking no"
1062
+
1063
+ Args:
1064
+ None
1065
+
1066
+ Returns:
1067
+ None
1068
+ '''
1069
+ global sshConfigged
1070
+ if not sshConfigged:
1071
+ # first we make sure ~/.ssh/config exists
1072
+ config = ''
1073
+ if not os.path.exists(os.path.expanduser('~/.ssh')):
1074
+ os.makedirs(os.path.expanduser('~/.ssh'))
1075
+ if os.path.exists(os.path.expanduser('~/.ssh/config')):
1076
+ with open(os.path.expanduser('~/.ssh/config'),'r') as f:
1077
+ config = f.read()
1078
+ if config:
1079
+ if 'StrictHostKeyChecking no' not in config:
1080
+ with open(os.path.expanduser('~/.ssh/config'),'a') as f:
1081
+ f.write('\nHost *\n\tStrictHostKeyChecking no\n')
1082
+ else:
1083
+ with open(os.path.expanduser('~/.ssh/config'),'w') as f:
1084
+ f.write('Host *\n\tStrictHostKeyChecking no\n')
1085
+ sshConfigged = True
1086
+
1087
+ def signal_handler(sig, frame):
1088
+ '''
1089
+ Handle the Ctrl C signal
1090
+
1091
+ Args:
1092
+ sig (int): The signal
1093
+ frame (frame): The frame
1094
+
1095
+ Returns:
1096
+ None
1097
+ '''
1098
+ global emo
1099
+ if not emo:
1100
+ print('Ctrl C caught, exiting...')
1101
+ emo = True
1102
+ else:
1103
+ print('Ctrl C caught again, exiting immediately!')
1104
+ # wait for 0.1 seconds to allow the threads to exit
1105
+ time.sleep(0.1)
1106
+ os.system(f'pkill -ef {os.path.basename(__file__)}')
1107
+ sys.exit(0)
1108
+
1109
+
1110
+ def processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window = DEFAULT_SINGLE_WINDOW):
1111
+ global gloablUnavailableHosts
1112
+ threads = start_run_on_hosts(hosts, timeout=timeout,password=password,max_connections=max_connections)
1113
+ if not quiet and threads and not returnUnfinished and any([thread.is_alive() for thread in threads]) and sys.stdout.isatty() and os.get_terminal_size() and os.get_terminal_size().columns > 10:
1114
+ curses.wrapper(curses_print, hosts, threads, min_char_len = curses_min_char_len, min_line_len = curses_min_line_len, single_window = single_window)
1115
+ if not returnUnfinished:
1116
+ # wait until all hosts have a return code
1117
+ while any([host.returncode is None for host in hosts]):
1118
+ time.sleep(0.1)
1119
+ for thread in threads:
1120
+ thread.join(timeout=3)
1121
+ # update the unavailable hosts and global unavailable hosts
1122
+ if willUpdateUnreachableHosts:
1123
+ unavailableHosts.update([host.name for host in hosts if host.stderr and ('No route to host' in host.stderr[0].strip() or host.stderr[0].strip().startswith('Timeout!'))])
1124
+ gloablUnavailableHosts.update(unavailableHosts)
1125
+ # print the output, if the output of multiple hosts are the same, we aggragate them
1126
+ if not called:
1127
+ print_output(hosts,json,greppable=greppable)
1128
+
1129
+ @cache_decorator
1130
+ def formHostStr(host) -> str:
1131
+ """
1132
+ Forms a comma-separated string of hosts.
1133
+
1134
+ Args:
1135
+ host: A string or a set of hosts.
1136
+
1137
+ Returns:
1138
+ A string representing the hosts, separated by commas.
1139
+ """
1140
+ if not host or len(host) == 0:
1141
+ return 'EMPTY_HOSTS'
1142
+ if type(host) is str:
1143
+ host = set(host.replace(',',' ').replace('\n',' ').replace('\r',' ').replace('\t',' ').replace(';', ' ').replace('|', ' ').replace('/', ' ').replace('&',' ').split())
1144
+ else:
1145
+ host = set(host)
1146
+ if 'local_shell' in host:
1147
+ host.remove('local_shell')
1148
+ host.add('localhost')
1149
+ host = ','.join(host)
1150
+ return host
1151
+
1152
+
1153
+ @cache_decorator
1154
+ def __formCommandArgStr(oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
1155
+ quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,max_connections=DEFAULT_MAX_CONNECTIONS,
1156
+ files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,
1157
+ scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
1158
+ no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,skip_hosts = DEFAULT_SKIP_HOSTS,
1159
+ file_sync = False, error_only = DEFAULT_ERROR_ONLY,
1160
+ shortend = False) -> str:
1161
+ argsList = []
1162
+ if oneonone: argsList.append('--oneonone' if not shortend else '-11')
1163
+ if timeout and timeout != DEFAULT_TIMEOUT: argsList.append(f'--timeout={timeout}' if not shortend else f'-t={timeout}')
1164
+ if password and password != DEFAULT_PASSWORD: argsList.append(f'--password="{password}"' if not shortend else f'-p="{password}"')
1165
+ if quiet: argsList.append('--quiet' if not shortend else '-q')
1166
+ if json: argsList.append('--json' if not shortend else '-j')
1167
+ if max_connections and max_connections != DEFAULT_MAX_CONNECTIONS: argsList.append(f'--maxconnections={max_connections}' if not shortend else f'-m={max_connections}')
1168
+ if files: argsList.extend([f'--file="{file}"' for file in files] if not shortend else [f'-f="{file}"' for file in files])
1169
+ if ipmi: argsList.append('--ipmi')
1170
+ if interface_ip_prefix and interface_ip_prefix != DEFAULT_INTERFACE_IP_PREFIX: argsList.append(f'--interface_ip_prefix="{interface_ip_prefix}"' if not shortend else f'-pre="{interface_ip_prefix}"')
1171
+ if scp: argsList.append('--scp')
1172
+ if username and username != DEFAULT_USERNAME: argsList.append(f'--username="{username}"' if not shortend else f'-u="{username}"')
1173
+ if extraargs and extraargs != DEFAULT_EXTRA_ARGS: argsList.append(f'--extraargs="{extraargs}"' if not shortend else f'-ea="{extraargs}"')
1174
+ if skipUnreachable: argsList.append('--skipUnreachable' if not shortend else '-su')
1175
+ if no_env: argsList.append('--no_env')
1176
+ if greppable: argsList.append('--greppable' if not shortend else '-g')
1177
+ if error_only: argsList.append('--error_only' if not shortend else '-eo')
1178
+ if skip_hosts and skip_hosts != DEFAULT_SKIP_HOSTS: argsList.append(f'--skip_hosts="{skip_hosts}"' if not shortend else f'-sh="{skip_hosts}"')
1179
+ if file_sync: argsList.append('--file_sync' if not shortend else '-fs')
1180
+ return ' '.join(argsList)
1181
+
1182
+ def getStrCommand(hosts,commands,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
1183
+ quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,called = DEFAULT_CALLED,max_connections=DEFAULT_MAX_CONNECTIONS,
1184
+ files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = DEFAULT_RETURN_UNFINISHED,
1185
+ scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
1186
+ no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=DEFAULT_NO_START,
1187
+ skip_hosts = DEFAULT_SKIP_HOSTS, curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,
1188
+ single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY, shortend = False):
1189
+ hosts = hosts if type(hosts) == str else frozenset(hosts)
1190
+ hostStr = formHostStr(hosts)
1191
+ files = frozenset(files) if files else None
1192
+ argsStr = __formCommandArgStr(oneonone = oneonone, timeout = timeout,password = password,
1193
+ quiet = quiet,json = json,max_connections=max_connections,
1194
+ files = files,ipmi = ipmi,interface_ip_prefix = interface_ip_prefix,scp=scp,
1195
+ username=username,extraargs=extraargs,skipUnreachable=skipUnreachable,no_env=no_env,
1196
+ greppable=greppable,skip_hosts = skip_hosts, file_sync = file_sync,error_only = error_only, shortend = shortend)
1197
+ commandStr = '"' + '" "'.join(commands) + '"' if commands else ''
1198
+ return f'multissh {argsStr} {hostStr} {commandStr}'
1199
+
1200
+ def run_command_on_hosts(hosts,commands,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
1201
+ quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,called = DEFAULT_CALLED,max_connections=DEFAULT_MAX_CONNECTIONS,
1202
+ files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = DEFAULT_RETURN_UNFINISHED,
1203
+ scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
1204
+ no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=DEFAULT_NO_START,
1205
+ skip_hosts = DEFAULT_SKIP_HOSTS, curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,
1206
+ single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY):
1207
+ f'''
1208
+ Run the command on the hosts, aka multissh. main function
1209
+
1210
+ Args:
1211
+ hosts (str/iterable): A string of hosts seperated by space or comma / iterable of hosts.
1212
+ commands (list): A list of commands to run on the hosts. When using files, defines the destination of the files.
1213
+ oneonone (bool, optional): Whether to run the commands one on one. Defaults to {DEFAULT_ONE_ON_ONE}.
1214
+ timeout (int, optional): The timeout for the command. Defaults to {DEFAULT_TIMEOUT}.
1215
+ password (str, optional): The password for the hosts. Defaults to {DEFAULT_PASSWORD}.
1216
+ quiet (bool, optional): Whether to print the output. Defaults to {DEFAULT_QUIET}.
1217
+ json (bool, optional): Whether to print the output in JSON format. Defaults to {DEFAULT_JSON_MODE}.
1218
+ called (bool, optional): Whether the function is called by another function. Defaults to {DEFAULT_CALLED}.
1219
+ max_connections (int, optional): The maximum number of concurrent SSH sessions. Defaults to 4 * os.cpu_count().
1220
+ files (list, optional): A list of files to be copied to the hosts. Defaults to None.
1221
+ ipmi (bool, optional): Whether to use IPMI to connect to the hosts. Defaults to {DEFAULT_IPMI}.
1222
+ interface_ip_prefix (str, optional): The prefix of the IPMI interface. Defaults to {DEFAULT_INTERFACE_IP_PREFIX}.
1223
+ returnUnfinished (bool, optional): Whether to return the unfinished hosts. Defaults to {DEFAULT_RETURN_UNFINISHED}.
1224
+ scp (bool, optional): Whether to use scp instead of rsync. Defaults to {DEFAULT_SCP}.
1225
+ username (str, optional): The username to use to connect to the hosts. Defaults to {DEFAULT_USERNAME}.
1226
+ extraargs (str, optional): Extra arguments to pass to the ssh / rsync / scp command. Defaults to {DEFAULT_EXTRA_ARGS}.
1227
+ skipUnreachable (bool, optional): Whether to skip unreachable hosts. Defaults to {DEFAULT_SKIP_UNREACHABLE}.
1228
+ no_env (bool, optional): Whether to not read the current sat system environment variables. (Will still read from files) Defaults to {DEFAULT_NO_ENV}.
1229
+ greppable (bool, optional): Whether to print the output in greppable format. Defaults to {DEFAULT_GREPPABLE_MODE}.
1230
+ willUpdateUnreachableHosts (bool, optional): Whether to update the global unavailable hosts. Defaults to {DEFAULT_UPDATE_UNREACHABLE_HOSTS}.
1231
+ no_start (bool, optional): Whether to return the hosts without starting the command. Defaults to {DEFAULT_NO_START}.
1232
+ skip_hosts (str, optional): The hosts to skip. Defaults to {DEFAULT_SKIP_HOSTS}.
1233
+ min_char_len (int, optional): The minimum character per line of the curses output. Defaults to {DEFAULT_CURSES_MINIMUM_CHAR_LEN}.
1234
+ min_line_len (int, optional): The minimum line number for each window of the curses output. Defaults to {DEFAULT_CURSES_MINIMUM_LINE_LEN}.
1235
+ single_window (bool, optional): Whether to use a single window for the curses output. Defaults to {DEFAULT_SINGLE_WINDOW}.
1236
+ file_sync (bool, optional): Whether to use file sync mode to sync directories. Defaults to {DEFAULT_FILE_SYNC}.
1237
+
1238
+ Returns:
1239
+ list: A list of Host objects
1240
+ '''
1241
+ global gloablUnavailableHosts
1242
+ global global_suppress_printout
1243
+ if not max_connections:
1244
+ max_connections = 4 * os.cpu_count()
1245
+ elif max_connections == 0:
1246
+ max_connections = 1048576
1247
+ elif max_connections < 0:
1248
+ max_connections = (-max_connections) * os.cpu_count()
1249
+ if not commands:
1250
+ commands = []
1251
+ verify_ssh_config()
1252
+ # load global unavailable hosts only if the function is called (so using --repeat will not load the unavailable hosts again)
1253
+ if called:
1254
+ # if called,
1255
+ # if skipUnreachable is not set, we default to skip unreachable hosts within one command call
1256
+ global_suppress_printout = True
1257
+ if skipUnreachable is None:
1258
+ skipUnreachable = True
1259
+ if skipUnreachable:
1260
+ unavailableHosts = gloablUnavailableHosts
1261
+ else:
1262
+ unavailableHosts = set()
1263
+ else:
1264
+ # if run in command line ( or emulating running in command line, we default to skip unreachable hosts within one command call )
1265
+ if skipUnreachable:
1266
+ unavailableHosts = gloablUnavailableHosts
1267
+ else:
1268
+ unavailableHosts = set()
1269
+ skipUnreachable = True
1270
+ global emo
1271
+ emo = False
1272
+ # We create the hosts
1273
+ hostStr = formHostStr(hosts)
1274
+ skipHostStr = formHostStr(skip_hosts) if skip_hosts else ''
1275
+
1276
+ if username:
1277
+ userStr = f'{username.strip()}@'
1278
+ # we also append this userStr to all hostStr which does not have username already defined
1279
+ hostStr = hostStr.split(',')
1280
+ for i, host in enumerate(hostStr):
1281
+ if '@' not in host:
1282
+ hostStr[i] = userStr + host
1283
+ hostStr = ','.join(hostStr)
1284
+ if skipHostStr:
1285
+ skipHostStr = skipHostStr.split(',')
1286
+ for i, host in enumerate(skipHostStr):
1287
+ if '@' not in host:
1288
+ skipHostStr[i] = userStr + host
1289
+ skipHostStr = ','.join(skipHostStr)
1290
+ targetHostsList = expand_hostnames(frozenset(hostStr.split(',')),no_env=no_env)
1291
+ skipHostsList = expand_hostnames(frozenset(skipHostStr.split(',')),no_env=no_env)
1292
+ if skipHostsList:
1293
+ if not global_suppress_printout: print(f"Skipping hosts: {skipHostsList}")
1294
+ if files and not commands:
1295
+ # if files are specified but not target dir, we default to file sync mode
1296
+ file_sync = True
1297
+ if file_sync:
1298
+ # set the files to the union of files and commands
1299
+ files = set(files+commands) if files else set(commands)
1300
+ if files:
1301
+ # try to resolve files first (like * etc)
1302
+ pathSet = set()
1303
+ for file in files:
1304
+ try:
1305
+ pathSet.update(glob.glob(file,include_hidden=True,recursive=True))
1306
+ except:
1307
+ pathSet.update(glob.glob(file,recursive=True))
1308
+ if not pathSet:
1309
+ print(f'Warning: No source files at {files} are found after resolving globs!')
1310
+ sys.exit(66)
1311
+ if file_sync:
1312
+ # use abosolute path for file sync
1313
+ commands = [os.path.abspath(file) for file in pathSet]
1314
+ files = []
1315
+ else:
1316
+ files = list(pathSet)
1317
+ if oneonone:
1318
+ hosts = []
1319
+ if len(commands) != len(targetHostsList) - len(skipHostsList):
1320
+ print("Error: the number of commands must be the same as the number of hosts")
1321
+ print(f"Number of commands: {len(commands)}")
1322
+ print(f"Number of hosts: {len(targetHostsList - skipHostsList)}")
1323
+ sys.exit(255)
1324
+ if not global_suppress_printout:
1325
+ print('-'*80)
1326
+ print("Running in one on one mode")
1327
+ for host, command in zip(targetHostsList, commands):
1328
+ if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
1329
+ if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
1330
+ continue
1331
+ if host.strip() in skipHostsList: continue
1332
+ if file_sync:
1333
+ hosts.append(Host(host.strip(), os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
1334
+ else:
1335
+ hosts.append(Host(host.strip(), command, files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
1336
+ if not global_suppress_printout:
1337
+ print(f"Running command: {command} on host: {host}")
1338
+ if not global_suppress_printout: print('-'*80)
1339
+ if not no_start: processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
1340
+ return hosts
1341
+ else:
1342
+ allHosts = []
1343
+ if not commands:
1344
+ # run in interactive mode ssh mode
1345
+ hosts = []
1346
+ for host in targetHostsList:
1347
+ if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
1348
+ if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
1349
+ continue
1350
+ if host.strip() in skipHostsList: continue
1351
+ if file_sync:
1352
+ print(f"Error: file sync mode need to be specified with at least one path to sync.")
1353
+ return []
1354
+ elif files:
1355
+ print(f"Error: files need to be specified with at least one path to sync")
1356
+ elif ipmi:
1357
+ print(f"Error: ipmi mode is not supported in interactive mode")
1358
+ else:
1359
+ hosts.append(Host(host.strip(), '', files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
1360
+ if not global_suppress_printout:
1361
+ print('-'*80)
1362
+ print(f"Running in interactive mode on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
1363
+ print('-'*80)
1364
+ if no_start:
1365
+ print(f"Warning: no_start is set, the command will not be started. As we are in interactive mode, no action will be done.")
1366
+ else:
1367
+ processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
1368
+ return hosts
1369
+ for command in commands:
1370
+ hosts = []
1371
+ for host in targetHostsList:
1372
+ if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
1373
+ if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
1374
+ continue
1375
+ if host.strip() in skipHostsList: continue
1376
+ if file_sync:
1377
+ hosts.append(Host(host.strip(), os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
1378
+ else:
1379
+ hosts.append(Host(host.strip(), command, files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
1380
+ if not global_suppress_printout and len(commands) > 1:
1381
+ print('-'*80)
1382
+ print(f"Running command: {command} on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
1383
+ print('-'*80)
1384
+ if not no_start: processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
1385
+ allHosts += hosts
1386
+ return allHosts
1387
+
1388
+ def main():
1389
+ # We handle the signal
1390
+ signal.signal(signal.SIGINT, signal_handler)
1391
+ # We parse the arguments
1392
+ parser = argparse.ArgumentParser(description='Run a command on multiple hosts, Use #HOST# or #HOSTNAME# to replace the host name in the command')
1393
+ parser.add_argument('hosts', metavar='hosts', type=str, help='Hosts to run the command on, use "," to seperate hosts')
1394
+ parser.add_argument('commands', metavar='commands', type=str, nargs='+',help='the command to run on the hosts / the destination of the files #HOST# or #HOSTNAME# will be replaced with the host name.')
1395
+ parser.add_argument('-u','--username', type=str,help=f'The general username to use to connect to the hosts. Will get overwrote by individual username@host if specified. (default: {DEFAULT_USERNAME})',default=DEFAULT_USERNAME)
1396
+ parser.add_argument('-ea','--extraargs',type=str,help=f'Extra arguments to pass to the ssh / rsync / scp command. Put in one string for multiple arguments.Use "=" ! Ex. -ea="--delete" (default: {DEFAULT_EXTRA_ARGS})',default=DEFAULT_EXTRA_ARGS)
1397
+ parser.add_argument('-p', '--password', type=str,help=f'The password to use to connect to the hosts, (default: {DEFAULT_PASSWORD})',default=DEFAULT_PASSWORD)
1398
+ parser.add_argument("-11",'--oneonone', action='store_true', help=f"Run one corresponding command on each host. (default: {DEFAULT_ONE_ON_ONE})", default=DEFAULT_ONE_ON_ONE)
1399
+ parser.add_argument("-f","--file", action='append', help="The file to be copied to the hosts. Use -f multiple times to copy multiple files")
1400
+ parser.add_argument('--file_sync', action='store_true', help=f'Operate in file sync mode, sync path in <COMMANDS> from this machine to <HOSTS>. Treat --file <FILE> and <COMMANDS> both as source as source and destination will be the same in this mode. (default: {DEFAULT_FILE_SYNC})', default=DEFAULT_FILE_SYNC)
1401
+ parser.add_argument('--scp', action='store_true', help=f'Use scp for copying files instead of rsync. Need to use this on windows. (default: {DEFAULT_SCP})', default=DEFAULT_SCP)
1402
+ #parser.add_argument("-d",'-c',"--destination", type=str, help="The destination of the files. Same as specify with commands. Added for compatibility. Use #HOST# or #HOSTNAME# to replace the host name in the destination")
1403
+ parser.add_argument("-t","--timeout", type=int, help=f"Timeout for each command in seconds (default: 0 (disabled))", default=0)
1404
+ parser.add_argument("-r","--repeat", type=int, help=f"Repeat the command for a number of times (default: {DEFAULT_REPEAT})", default=DEFAULT_REPEAT)
1405
+ parser.add_argument("-i","--interval", type=int, help=f"Interval between repeats in seconds (default: {DEFAULT_INTERVAL})", default=DEFAULT_INTERVAL)
1406
+ parser.add_argument("--ipmi", action='store_true', help=f"Use ipmitool to run the command. (default: {DEFAULT_IPMI})", default=DEFAULT_IPMI)
1407
+ parser.add_argument("-mpre","--ipmi_interface_ip_prefix", type=str, help=f"The prefix of the IPMI interfaces (default: {DEFAULT_IPMI_INTERFACE_IP_PREFIX})", default=DEFAULT_IPMI_INTERFACE_IP_PREFIX)
1408
+ parser.add_argument("-pre","--interface_ip_prefix", type=str, help=f"The prefix of the for the interfaces (default: {DEFAULT_INTERFACE_IP_PREFIX})", default=DEFAULT_INTERFACE_IP_PREFIX)
1409
+ parser.add_argument("-q","--quiet", action='store_true', help=f"Quiet mode, no curses, only print the output. (default: {DEFAULT_QUIET})", default=DEFAULT_QUIET)
1410
+ parser.add_argument("-ww",'--window_width', type=int, help=f"The minimum character length of the curses window. (default: {DEFAULT_CURSES_MINIMUM_CHAR_LEN})", default=DEFAULT_CURSES_MINIMUM_CHAR_LEN)
1411
+ parser.add_argument("-wh",'--window_height', type=int, help=f"The minimum line height of the curses window. (default: {DEFAULT_CURSES_MINIMUM_LINE_LEN})", default=DEFAULT_CURSES_MINIMUM_LINE_LEN)
1412
+ parser.add_argument('-sw','--single_window', action='store_true', help=f'Use a single window for all hosts. (default: {DEFAULT_SINGLE_WINDOW})', default=DEFAULT_SINGLE_WINDOW)
1413
+ parser.add_argument('-eo','--error_only', action='store_true', help=f'Only print the error output. (default: {DEFAULT_ERROR_ONLY})', default=DEFAULT_ERROR_ONLY)
1414
+ parser.add_argument("-no","--nooutput", action='store_true', help=f"Do not print the output. (default: {DEFAULT_NO_OUTPUT})", default=DEFAULT_NO_OUTPUT)
1415
+ parser.add_argument('--no_env', action='store_true', help=f'Do not load the environment variables. (default: {DEFAULT_NO_ENV})', default=DEFAULT_NO_ENV)
1416
+ parser.add_argument("--env_file", type=str, help=f"The file to load the environment variables from. (default: {DEFAULT_ENV_FILE})", default=DEFAULT_ENV_FILE)
1417
+ parser.add_argument("-m","--maxconnections", type=int, help=f"Max number of connections to use (default: 4 * cpu_count)", default=DEFAULT_MAX_CONNECTIONS)
1418
+ parser.add_argument("-j","--json", action='store_true', help=F"Output in json format. (default: {DEFAULT_JSON_MODE})", default=DEFAULT_JSON_MODE)
1419
+ parser.add_argument("--success_hosts", action='store_true', help=f"Output the hosts that succeeded in summary as wells. (default: {DEFAULT_PRINT_SUCCESS_HOSTS})", default=DEFAULT_PRINT_SUCCESS_HOSTS)
1420
+ parser.add_argument("-g","--greppable", action='store_true', help=f"Output in greppable format. (default: {DEFAULT_GREPPABLE_MODE})", default=DEFAULT_GREPPABLE_MODE)
1421
+ parser.add_argument("-nw","--nowatch", action='store_true', help=f"Do not watch the output in curses modem, Use \\r. Not implemented yet. (default: {DEFAULT_NO_WATCH})", default=DEFAULT_NO_WATCH)
1422
+ parser.add_argument("-su","--skipunreachable", action='store_true', help=f"Skip unreachable hosts while using --repeat. Note: Timedout Hosts are considered unreachable. Note: multiple command sequence will still auto skip unreachable hosts. (default: {DEFAULT_SKIP_UNREACHABLE})", default=DEFAULT_SKIP_UNREACHABLE)
1423
+ parser.add_argument("-sh","--skiphosts", type=str, help=f"Skip the hosts in the list. (default: {DEFAULT_SKIP_HOSTS})", default=DEFAULT_SKIP_HOSTS)
1424
+ parser.add_argument("-V","--version", action='version', version=f'%(prog)s {version} {("with sshpass " if sshpassAvailable else "")}by pan@zopyr.us')
1425
+
1426
+ # parser.add_argument('-u', '--user', metavar='user', type=str, nargs=1,
1427
+ # help='the user to use to connect to the hosts')
1428
+ args = parser.parse_args()
1429
+
1430
+ env_file = args.env_file
1431
+ # if there are more than 1 commands, and every command only consists of one word,
1432
+ # we will ask the user to confirm if they want to run multiple commands or just one command.
1433
+ if not args.file and len(args.commands) > 1 and all([len(command.split()) == 1 for command in args.commands]):
1434
+ print(f"Multiple one word command detected, what to do? (s/f/n)")
1435
+ print(f"1: Run 1 command [{' '.join(args.commands)}] on all hosts ( default )")
1436
+ print(f"m: Run multiple commands [{', '.join(args.commands)}] on all hosts")
1437
+ print(f"n: Exit")
1438
+ inStr = input_with_timeout_and_countdown(3)
1439
+ if (not inStr) or inStr.lower().strip().startswith('1'):
1440
+ args.commands = [" ".join(args.commands)]
1441
+ print(f"\nRunning 1 command: {args.commands[0]} on all hosts")
1442
+ elif inStr.lower().strip().startswith('m'):
1443
+ print(f"\nRunning multiple commands: {', '.join(args.commands)} on all hosts")
1444
+ else:
1445
+ sys.exit(0)
1446
+
1447
+ ipmiiInterfaceIPPrefix = args.ipmi_interface_ip_prefix
1448
+
1449
+ if not args.greppable and not args.json and not args.nooutput:
1450
+ global_suppress_printout = False
1451
+
1452
+ if not global_suppress_printout:
1453
+ print('> ' + getStrCommand(args.hosts,args.commands,oneonone=args.oneonone,timeout=args.timeout,password=args.password,
1454
+ quiet=args.quiet,json=args.json,called=args.nooutput,max_connections=args.maxconnections,
1455
+ files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,username=args.username,
1456
+ extraargs=args.extraargs,skipUnreachable=args.skipunreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skiphosts,
1457
+ curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only))
1458
+ if args.error_only:
1459
+ global_suppress_printout = True
1460
+
1461
+ for i in range(args.repeat):
1462
+ if args.interval > 0 and i < args.repeat - 1:
1463
+ print(f"Sleeping for {args.interval} seconds")
1464
+ time.sleep(args.interval)
1465
+
1466
+ if not global_suppress_printout: print(f"Running the {i+1}/{args.repeat} time") if args.repeat > 1 else None
1467
+ hosts = run_command_on_hosts(args.hosts,args.commands,
1468
+ oneonone=args.oneonone,timeout=args.timeout,password=args.password,
1469
+ quiet=args.quiet,json=args.json,called=args.nooutput,max_connections=args.maxconnections,
1470
+ files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,username=args.username,
1471
+ extraargs=args.extraargs,skipUnreachable=args.skipunreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skiphosts,
1472
+ curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only)
1473
+ #print('*'*80)
1474
+
1475
+ if not global_suppress_printout: print('-'*80)
1476
+
1477
+ succeededHosts = set()
1478
+ for host in hosts:
1479
+ if host.returncode and host.returncode != 0:
1480
+ mainReturnCode += 1
1481
+ failedHosts.add(host.name)
1482
+ else:
1483
+ succeededHosts.add(host.name)
1484
+ succeededHosts -= failedHosts
1485
+ # sort the failed hosts and succeeded hosts
1486
+ failedHosts = sorted(failedHosts)
1487
+ succeededHosts = sorted(succeededHosts)
1488
+ if mainReturnCode > 0:
1489
+ if not global_suppress_printout: print(f'Complete. Failed hosts (Return Code not 0) count: {mainReturnCode}')
1490
+ # with open('/tmp/bashcmd.stdin','w') as f:
1491
+ # f.write(f"export failed_hosts={failedHosts}\n")
1492
+ if not global_suppress_printout: print(f'failed_hosts: {",".join(failedHosts)}')
1493
+ else:
1494
+ if not global_suppress_printout: print('Complete. All hosts returned 0.')
1495
+
1496
+ if args.success_hosts and not global_suppress_printout:
1497
+ print(f'succeeded_hosts: {",".join(succeededHosts)}')
1498
+
1499
+ if threading.active_count() > 1:
1500
+ if not global_suppress_printout: print(f'Remaining active thread: {threading.active_count()}')
1501
+ # os.system(f'pkill -ef {os.path.basename(__file__)}')
1502
+ # os._exit(mainReturnCode)
1503
+
1504
+ sys.exit(mainReturnCode)
1505
+
1506
+
1507
+
1508
+ if __name__ == "__main__":
1509
+ main()