multiSSH3 4.75__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of multiSSH3 might be problematic. Click here for more details.
- multiSSH3-4.75.dist-info/LICENSE +674 -0
- multiSSH3-4.75.dist-info/METADATA +305 -0
- multiSSH3-4.75.dist-info/RECORD +7 -0
- multiSSH3-4.75.dist-info/WHEEL +5 -0
- multiSSH3-4.75.dist-info/entry_points.txt +6 -0
- multiSSH3-4.75.dist-info/top_level.txt +1 -0
- multiSSH3.py +1509 -0
multiSSH3.py
ADDED
|
@@ -0,0 +1,1509 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import curses
|
|
3
|
+
import subprocess
|
|
4
|
+
import threading
|
|
5
|
+
import time,os
|
|
6
|
+
import argparse
|
|
7
|
+
from itertools import product
|
|
8
|
+
import re
|
|
9
|
+
import string
|
|
10
|
+
import ipaddress
|
|
11
|
+
import sys
|
|
12
|
+
import json
|
|
13
|
+
import socket
|
|
14
|
+
import io
|
|
15
|
+
import signal
|
|
16
|
+
import functools
|
|
17
|
+
import glob
|
|
18
|
+
try:
|
|
19
|
+
# Check if functiools.cache is available
|
|
20
|
+
cache_decorator = functools.cache
|
|
21
|
+
except AttributeError:
|
|
22
|
+
try:
|
|
23
|
+
# Check if functools.lru_cache is available
|
|
24
|
+
cache_decorator = functools.lru_cache(maxsize=None)
|
|
25
|
+
except AttributeError:
|
|
26
|
+
# If neither is available, use a dummy decorator
|
|
27
|
+
def cache_decorator(func):
|
|
28
|
+
return func
|
|
29
|
+
|
|
30
|
+
version = '4.75'
|
|
31
|
+
VERSION = version
|
|
32
|
+
|
|
33
|
+
DEFAULT_ENV_FILE = '/etc/profile.d/hosts.sh'
|
|
34
|
+
DEFAULT_USERNAME = None
|
|
35
|
+
DEFAULT_EXTRA_ARGS = None
|
|
36
|
+
DEFAULT_PASSWORD = ''
|
|
37
|
+
DEFAULT_ONE_ON_ONE = False
|
|
38
|
+
DEFAULT_FILE_SYNC = False
|
|
39
|
+
DEFAULT_SCP = False
|
|
40
|
+
DEFAULT_TIMEOUT = 50
|
|
41
|
+
DEFAULT_REPEAT = 1
|
|
42
|
+
DEFAULT_INTERVAL = 0
|
|
43
|
+
DEFAULT_IPMI = False
|
|
44
|
+
DEFAULT_INTERFACE_IP_PREFIX = None
|
|
45
|
+
DEFAULT_IPMI_INTERFACE_IP_PREFIX = None
|
|
46
|
+
DEFAULT_QUIET = False
|
|
47
|
+
DEFAULT_ERROR_ONLY = False
|
|
48
|
+
DEFAULT_NO_OUTPUT = False
|
|
49
|
+
DEFAULT_NO_ENV = False
|
|
50
|
+
DEFAULT_MAX_CONNECTIONS = 4 * os.cpu_count()
|
|
51
|
+
DEFAULT_JSON_MODE = False
|
|
52
|
+
DEFAULT_PRINT_SUCCESS_HOSTS = False
|
|
53
|
+
DEFAULT_GREPPABLE_MODE = False
|
|
54
|
+
DEFAULT_NO_WATCH = False
|
|
55
|
+
DEFAULT_SKIP_UNREACHABLE = False
|
|
56
|
+
DEFAULT_SKIP_HOSTS = ''
|
|
57
|
+
DEFAULT_CURSES_MINIMUM_CHAR_LEN = 40
|
|
58
|
+
DEFAULT_CURSES_MINIMUM_LINE_LEN = 1
|
|
59
|
+
DEFAULT_SINGLE_WINDOW = False
|
|
60
|
+
|
|
61
|
+
DEFAULT_CALLED = True
|
|
62
|
+
DEFAULT_RETURN_UNFINISHED = False
|
|
63
|
+
DEFAULT_UPDATE_UNREACHABLE_HOSTS = True
|
|
64
|
+
DEFAULT_NO_START = False
|
|
65
|
+
|
|
66
|
+
global_suppress_printout = True
|
|
67
|
+
|
|
68
|
+
mainReturnCode = 0
|
|
69
|
+
failedHosts = set()
|
|
70
|
+
class Host:
|
|
71
|
+
def __init__(self, name, command, files = None,ipmi = False,interface_ip_prefix = None,scp=False,extraargs=None):
|
|
72
|
+
self.name = name # the name of the host (hostname or IP address)
|
|
73
|
+
self.command = command # the command to run on the host
|
|
74
|
+
self.returncode = None # the return code of the command
|
|
75
|
+
self.output = [] # the output of the command for curses
|
|
76
|
+
self.stdout = [] # the stdout of the command
|
|
77
|
+
self.stderr = [] # the stderr of the command
|
|
78
|
+
self.printedLines = -1 # the number of lines printed on the screen
|
|
79
|
+
self.files = files # the files to be copied to the host
|
|
80
|
+
self.ipmi = ipmi # whether to use ipmi to connect to the host
|
|
81
|
+
self.interface_ip_prefix = interface_ip_prefix # the prefix of the ip address of the interface to be used to connect to the host
|
|
82
|
+
self.scp = scp # whether to use scp to copy files to the host
|
|
83
|
+
self.extraargs = extraargs # extra arguments to be passed to ssh
|
|
84
|
+
self.resolvedName = None # the resolved IP address of the host
|
|
85
|
+
def __iter__(self):
|
|
86
|
+
return zip(['name', 'command', 'returncode', 'stdout', 'stderr'], [self.name, self.command, self.returncode, self.stdout, self.stderr])
|
|
87
|
+
def __repr__(self):
|
|
88
|
+
# return the complete data structure
|
|
89
|
+
return f"Host(name={self.name}, command={self.command}, returncode={self.returncode}, stdout={self.stdout}, stderr={self.stderr}, output={self.output}, printedLines={self.printedLines}, files={self.files}, ipmi={self.ipmi}, interface_ip_prefix={self.interface_ip_prefix}, scp={self.scp}, extraargs={self.extraargs}, resolvedName={self.resolvedName})"
|
|
90
|
+
def __str__(self):
|
|
91
|
+
return f"Host(name={self.name}, command={self.command}, returncode={self.returncode}, stdout={self.stdout}, stderr={self.stderr})"
|
|
92
|
+
|
|
93
|
+
wildCharacters = ['*','?','x']
|
|
94
|
+
|
|
95
|
+
gloablUnavailableHosts = set()
|
|
96
|
+
|
|
97
|
+
ipmiiInterfaceIPPrefix = DEFAULT_IPMI_INTERFACE_IP_PREFIX
|
|
98
|
+
|
|
99
|
+
keyPressesIn = [[]]
|
|
100
|
+
|
|
101
|
+
emo = False
|
|
102
|
+
|
|
103
|
+
etc_hosts = {}
|
|
104
|
+
|
|
105
|
+
env_file = DEFAULT_ENV_FILE
|
|
106
|
+
|
|
107
|
+
# check if command sshpass is available
|
|
108
|
+
sshpassAvailable = False
|
|
109
|
+
try:
|
|
110
|
+
subprocess.run(['which', 'sshpass'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
|
|
111
|
+
sshpassAvailable = True
|
|
112
|
+
except:
|
|
113
|
+
pass
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@cache_decorator
|
|
117
|
+
def expandIPv4Address(hosts):
|
|
118
|
+
'''
|
|
119
|
+
Expand the IP address range in the hosts list
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
hosts (list): A list of IP addresses or IP address ranges
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
list: A list of expanded IP addresses
|
|
126
|
+
'''
|
|
127
|
+
expandedHosts = []
|
|
128
|
+
expandedHost = []
|
|
129
|
+
for host in hosts:
|
|
130
|
+
host = host.replace('[','').replace(']','')
|
|
131
|
+
octets = host.split('.')
|
|
132
|
+
expandedOctets = []
|
|
133
|
+
for octet in octets:
|
|
134
|
+
if '-' in octet:
|
|
135
|
+
# Handle wildcards
|
|
136
|
+
octetRange = octet.split('-')
|
|
137
|
+
for i in range(len(octetRange)):
|
|
138
|
+
if not octetRange[i] or octetRange[i] in wildCharacters:
|
|
139
|
+
if i == 0:
|
|
140
|
+
octetRange[i] = '0'
|
|
141
|
+
elif i == 1:
|
|
142
|
+
octetRange[i] = '255'
|
|
143
|
+
|
|
144
|
+
expandedOctets.append([str(i) for i in range(int(octetRange[0]),int(octetRange[1])+1)])
|
|
145
|
+
elif octet in wildCharacters:
|
|
146
|
+
expandedOctets.append([str(i) for i in range(0,256)])
|
|
147
|
+
else:
|
|
148
|
+
expandedOctets.append([octet])
|
|
149
|
+
# handle the first and last subnet addresses
|
|
150
|
+
if '0' in expandedOctets[-1]:
|
|
151
|
+
expandedOctets[-1].remove('0')
|
|
152
|
+
if '255' in expandedOctets[-1]:
|
|
153
|
+
expandedOctets[-1].remove('255')
|
|
154
|
+
#print(expandedOctets)
|
|
155
|
+
# Generate the expanded hosts
|
|
156
|
+
for ip in list(product(expandedOctets[0],expandedOctets[1],expandedOctets[2],expandedOctets[3])):
|
|
157
|
+
expandedHost.append('.'.join(ip))
|
|
158
|
+
expandedHosts.extend(expandedHost)
|
|
159
|
+
return expandedHosts
|
|
160
|
+
|
|
161
|
+
@cache_decorator
|
|
162
|
+
def readEnvFromFile(environemnt_file = ''):
|
|
163
|
+
'''
|
|
164
|
+
Read the environment variables from env_file
|
|
165
|
+
Returns:
|
|
166
|
+
dict: A dictionary of environment variables
|
|
167
|
+
'''
|
|
168
|
+
global env
|
|
169
|
+
try:
|
|
170
|
+
if env:
|
|
171
|
+
return env
|
|
172
|
+
except:
|
|
173
|
+
env = {}
|
|
174
|
+
global env_file
|
|
175
|
+
if environemnt_file:
|
|
176
|
+
envf = environemnt_file
|
|
177
|
+
else:
|
|
178
|
+
envf = env_file if env_file else DEFAULT_ENV_FILE
|
|
179
|
+
if os.path.exists(envf):
|
|
180
|
+
with open(envf,'r') as f:
|
|
181
|
+
for line in f:
|
|
182
|
+
if line.startswith('#') or not line.strip():
|
|
183
|
+
continue
|
|
184
|
+
key, value = line.replace('export ', '', 1).strip().split('=', 1)
|
|
185
|
+
key = key.strip().strip('"').strip("'")
|
|
186
|
+
value = value.strip().strip('"').strip("'")
|
|
187
|
+
# avoid infinite recursion
|
|
188
|
+
if key != value:
|
|
189
|
+
env[key] = value.strip('"').strip("'")
|
|
190
|
+
return env
|
|
191
|
+
|
|
192
|
+
@cache_decorator
|
|
193
|
+
def getIP(hostname,local=False):
|
|
194
|
+
'''
|
|
195
|
+
Get the IP address of the hostname
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
hostname (str): The hostname
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
str: The IP address of the hostname
|
|
202
|
+
'''
|
|
203
|
+
global etc_hosts
|
|
204
|
+
# First we check if the hostname is an IP address
|
|
205
|
+
try:
|
|
206
|
+
ipaddress.ip_address(hostname)
|
|
207
|
+
return hostname
|
|
208
|
+
except ValueError:
|
|
209
|
+
pass
|
|
210
|
+
# Then we check /etc/hosts
|
|
211
|
+
if not etc_hosts and os.path.exists('/etc/hosts'):
|
|
212
|
+
with open('/etc/hosts','r') as f:
|
|
213
|
+
for line in f:
|
|
214
|
+
if line.startswith('#') or not line.strip():
|
|
215
|
+
continue
|
|
216
|
+
#ip, host = line.split()[:2]
|
|
217
|
+
chunks = line.split()
|
|
218
|
+
if len(chunks) < 2:
|
|
219
|
+
continue
|
|
220
|
+
ip = chunks[0]
|
|
221
|
+
for host in chunks[1:]:
|
|
222
|
+
etc_hosts[host] = ip
|
|
223
|
+
if hostname in etc_hosts:
|
|
224
|
+
return etc_hosts[hostname]
|
|
225
|
+
if local:
|
|
226
|
+
return None
|
|
227
|
+
# Then we check the DNS
|
|
228
|
+
try:
|
|
229
|
+
return socket.gethostbyname(hostname)
|
|
230
|
+
except:
|
|
231
|
+
return None
|
|
232
|
+
|
|
233
|
+
@cache_decorator
|
|
234
|
+
def expand_hostname(text,validate=True,no_env=False):
|
|
235
|
+
'''
|
|
236
|
+
Expand the hostname range in the text.
|
|
237
|
+
Will search the string for a range ( [] encloused and non enclosed number ranges).
|
|
238
|
+
Will expand the range, validate them using validate_expand_hostname and return a list of expanded hostnames
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
text (str): The text to be expanded
|
|
242
|
+
validate (bool, optional): Whether to validate the hostname. Defaults to True.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
set: A set of expanded hostnames
|
|
246
|
+
'''
|
|
247
|
+
expandinghosts = [text]
|
|
248
|
+
expandedhosts = set()
|
|
249
|
+
# all valid alphanumeric characters
|
|
250
|
+
alphanumeric = string.digits + string.ascii_letters
|
|
251
|
+
while len(expandinghosts) > 0:
|
|
252
|
+
hostname = expandinghosts.pop()
|
|
253
|
+
match = re.search(r'\[(.*?-.*?)\]', hostname)
|
|
254
|
+
if not match:
|
|
255
|
+
expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
|
|
256
|
+
continue
|
|
257
|
+
try:
|
|
258
|
+
range_start, range_end = match.group(1).split('-')
|
|
259
|
+
except ValueError:
|
|
260
|
+
expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
|
|
261
|
+
continue
|
|
262
|
+
range_start = range_start.strip()
|
|
263
|
+
range_end = range_end.strip()
|
|
264
|
+
if not range_end:
|
|
265
|
+
if range_start.isdigit():
|
|
266
|
+
range_end = '9'
|
|
267
|
+
elif range_start.isalpha() and range_start.islower():
|
|
268
|
+
range_end = 'z'
|
|
269
|
+
elif range_start.isalpha() and range_start.isupper():
|
|
270
|
+
range_end = 'Z'
|
|
271
|
+
else:
|
|
272
|
+
expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
|
|
273
|
+
continue
|
|
274
|
+
if not range_start:
|
|
275
|
+
if range_end.isdigit():
|
|
276
|
+
range_start = '0'
|
|
277
|
+
elif range_end.isalpha() and range_end.islower():
|
|
278
|
+
range_start = 'a'
|
|
279
|
+
elif range_end.isalpha() and range_end.isupper():
|
|
280
|
+
range_start = 'A'
|
|
281
|
+
else:
|
|
282
|
+
expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
|
|
283
|
+
continue
|
|
284
|
+
if range_start.isdigit() and range_end.isdigit():
|
|
285
|
+
padding_length = min(len(range_start), len(range_end))
|
|
286
|
+
format_str = "{:0" + str(padding_length) + "d}"
|
|
287
|
+
for i in range(int(range_start), int(range_end) + 1):
|
|
288
|
+
formatted_i = format_str.format(i)
|
|
289
|
+
if '[' in hostname:
|
|
290
|
+
expandinghosts.append(hostname.replace(match.group(0), formatted_i, 1))
|
|
291
|
+
else:
|
|
292
|
+
expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), formatted_i, 1),no_env=no_env) if validate else [hostname])
|
|
293
|
+
else:
|
|
294
|
+
if all(c in string.hexdigits for c in range_start + range_end):
|
|
295
|
+
for i in range(int(range_start, 16), int(range_end, 16)+1):
|
|
296
|
+
if '[' in hostname:
|
|
297
|
+
expandinghosts.append(hostname.replace(match.group(0), format(i, 'x'), 1))
|
|
298
|
+
else:
|
|
299
|
+
expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), format(i, 'x'), 1),no_env=no_env) if validate else [hostname])
|
|
300
|
+
else:
|
|
301
|
+
try:
|
|
302
|
+
start_index = alphanumeric.index(range_start)
|
|
303
|
+
end_index = alphanumeric.index(range_end)
|
|
304
|
+
for i in range(start_index, end_index + 1):
|
|
305
|
+
if '[' in hostname:
|
|
306
|
+
expandinghosts.append(hostname.replace(match.group(0), alphanumeric[i], 1))
|
|
307
|
+
else:
|
|
308
|
+
expandedhosts.update(validate_expand_hostname(hostname.replace(match.group(0), alphanumeric[i], 1),no_env=no_env) if validate else [hostname])
|
|
309
|
+
except ValueError:
|
|
310
|
+
expandedhosts.update(validate_expand_hostname(hostname,no_env=no_env) if validate else [hostname])
|
|
311
|
+
return expandedhosts
|
|
312
|
+
|
|
313
|
+
@cache_decorator
|
|
314
|
+
def expand_hostnames(hosts,no_env=False):
|
|
315
|
+
'''
|
|
316
|
+
Expand the hostnames in the hosts list
|
|
317
|
+
|
|
318
|
+
Args:
|
|
319
|
+
hosts (list): A list of hostnames
|
|
320
|
+
|
|
321
|
+
Returns:
|
|
322
|
+
list: A list of expanded hostnames
|
|
323
|
+
'''
|
|
324
|
+
expandedhosts = []
|
|
325
|
+
if isinstance(hosts, str):
|
|
326
|
+
hosts = [hosts]
|
|
327
|
+
for host in hosts:
|
|
328
|
+
host = host.strip()
|
|
329
|
+
if not host:
|
|
330
|
+
continue
|
|
331
|
+
# we seperate the username from the hostname
|
|
332
|
+
username = None
|
|
333
|
+
if '@' in host:
|
|
334
|
+
username, host = host.split('@',1)
|
|
335
|
+
# first we check if the hostname is an range of IP addresses
|
|
336
|
+
# This is done by checking if the hostname follows four fields of
|
|
337
|
+
# "(((\d{1,3}|x|\*|\?)(-(\d{1,3}|x|\*|\?))?)|(\[(\d{1,3}|x|\*|\?)(-(\d{1,3}|x|\*|\?))?\]))"
|
|
338
|
+
# seperated by .
|
|
339
|
+
# If so, we expand the IP address range
|
|
340
|
+
if re.match(r'^((((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?))?)|(\[((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])}|x|\*|\?))?\]))(\.((((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?)(-((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?))?)|(\[((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?)(-((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|x|\*|\?))?\]))){2}(\.(((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?))?)|(\[((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])|x|\*|\?)(-((25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])}|x|\*|\?))?\]))$', host):
|
|
341
|
+
hostSetToAdd = sorted(expandIPv4Address(frozenset([host])),key=ipaddress.IPv4Address)
|
|
342
|
+
else:
|
|
343
|
+
hostSetToAdd = sorted(expand_hostname(host,no_env=no_env))
|
|
344
|
+
if username:
|
|
345
|
+
# we expand the username
|
|
346
|
+
username = sorted(expand_hostname(username,validate=False,no_env=no_env))
|
|
347
|
+
# we combine the username and hostname
|
|
348
|
+
hostSetToAdd = [u+'@'+h for u,h in product(username,hostSetToAdd)]
|
|
349
|
+
expandedhosts.extend(hostSetToAdd)
|
|
350
|
+
return expandedhosts
|
|
351
|
+
|
|
352
|
+
@cache_decorator
|
|
353
|
+
def validate_expand_hostname(hostname,no_env=False):
|
|
354
|
+
'''
|
|
355
|
+
Validate the hostname and expand it if it is a range of IP addresses
|
|
356
|
+
|
|
357
|
+
Args:
|
|
358
|
+
hostname (str): The hostname to be validated and expanded
|
|
359
|
+
|
|
360
|
+
Returns:
|
|
361
|
+
list: A list of valid hostnames
|
|
362
|
+
'''
|
|
363
|
+
# maybe it is just defined in ./target_files/hosts.sh and exported to the environment
|
|
364
|
+
# we will try to get the valid host name from the environment
|
|
365
|
+
hostname = hostname.strip('$')
|
|
366
|
+
if getIP(hostname,local=True):
|
|
367
|
+
return [hostname]
|
|
368
|
+
elif not no_env and hostname in os.environ:
|
|
369
|
+
# we will expand these hostnames again
|
|
370
|
+
return expand_hostnames(frozenset(os.environ[hostname].split(',')),no_env=no_env)
|
|
371
|
+
elif hostname in readEnvFromFile():
|
|
372
|
+
# we will expand these hostnames again
|
|
373
|
+
return expand_hostnames(frozenset(readEnvFromFile()[hostname].split(',')),no_env=no_env)
|
|
374
|
+
elif getIP(hostname,local=False):
|
|
375
|
+
return [hostname]
|
|
376
|
+
else:
|
|
377
|
+
print(f"Error: {hostname} is not a valid hostname or IP address!")
|
|
378
|
+
global mainReturnCode
|
|
379
|
+
mainReturnCode += 1
|
|
380
|
+
global failedHosts
|
|
381
|
+
failedHosts.add(hostname)
|
|
382
|
+
return []
|
|
383
|
+
|
|
384
|
+
def input_with_timeout_and_countdown(timeout, prompt='Please enter your selection'):
|
|
385
|
+
"""
|
|
386
|
+
Read an input from the user with a timeout and a countdown.
|
|
387
|
+
|
|
388
|
+
Parameters:
|
|
389
|
+
timeout (int): The timeout value in seconds.
|
|
390
|
+
prompt (str): The prompt message to display to the user. Default is 'Please enter your selection'.
|
|
391
|
+
|
|
392
|
+
Returns:
|
|
393
|
+
str or None: The user input if received within the timeout, or None if no input is received.
|
|
394
|
+
"""
|
|
395
|
+
import select
|
|
396
|
+
# Print the initial prompt with the countdown
|
|
397
|
+
print(f"{prompt} [{timeout}s]: ", end='', flush=True)
|
|
398
|
+
# Loop until the timeout
|
|
399
|
+
for remaining in range(timeout, 0, -1):
|
|
400
|
+
# If there is an input, return it
|
|
401
|
+
if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
|
|
402
|
+
return input().strip()
|
|
403
|
+
# Print the remaining time
|
|
404
|
+
print(f"\r{prompt} [{remaining}s]: ", end='', flush=True)
|
|
405
|
+
# Wait a second
|
|
406
|
+
time.sleep(1)
|
|
407
|
+
# If there is no input, return None
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
def handle_reading_stream(stream,target, host):
|
|
411
|
+
'''
|
|
412
|
+
Read the stream and append the lines to the target list
|
|
413
|
+
|
|
414
|
+
Args:
|
|
415
|
+
stream (io.BytesIO): The stream to be read
|
|
416
|
+
target (list): The list to append the lines to
|
|
417
|
+
host (Host): The host object
|
|
418
|
+
|
|
419
|
+
Returns:
|
|
420
|
+
None
|
|
421
|
+
'''
|
|
422
|
+
def add_line(current_line,target, host, keepLastLine=True):
|
|
423
|
+
if not keepLastLine:
|
|
424
|
+
target.pop()
|
|
425
|
+
host.output.pop()
|
|
426
|
+
host.printedLines -= 1
|
|
427
|
+
current_line_str = current_line.decode('utf-8',errors='backslashreplace')
|
|
428
|
+
target.append(current_line_str)
|
|
429
|
+
host.output.append(current_line_str)
|
|
430
|
+
current_line = bytearray()
|
|
431
|
+
lastLineCommited = True
|
|
432
|
+
for char in iter(lambda:stream.read(1), b''):
|
|
433
|
+
if char == b'\n':
|
|
434
|
+
if (not lastLineCommited) and current_line:
|
|
435
|
+
add_line(current_line,target, host, keepLastLine=False)
|
|
436
|
+
elif lastLineCommited:
|
|
437
|
+
add_line(current_line,target, host, keepLastLine=True)
|
|
438
|
+
current_line = bytearray()
|
|
439
|
+
lastLineCommited = True
|
|
440
|
+
elif char == b'\r':
|
|
441
|
+
add_line(current_line,target, host, keepLastLine=lastLineCommited)
|
|
442
|
+
current_line = bytearray()
|
|
443
|
+
lastLineCommited = False
|
|
444
|
+
else:
|
|
445
|
+
current_line.extend(char)
|
|
446
|
+
if current_line:
|
|
447
|
+
add_line(current_line,target, host, keepLastLine=lastLineCommited)
|
|
448
|
+
|
|
449
|
+
def handle_writing_stream(stream,stop_event,host):
|
|
450
|
+
'''
|
|
451
|
+
Write the key presses to the stream
|
|
452
|
+
|
|
453
|
+
Args:
|
|
454
|
+
stream (io.BytesIO): The stream to be written to
|
|
455
|
+
stop_event (threading.Event): The event to stop the thread
|
|
456
|
+
host (Host): The host object
|
|
457
|
+
|
|
458
|
+
Returns:
|
|
459
|
+
None
|
|
460
|
+
'''
|
|
461
|
+
global keyPressesIn
|
|
462
|
+
# keyPressesIn is a list of lists.
|
|
463
|
+
# Each list is a list of characters to be sent to the stdin of the process at once.
|
|
464
|
+
# We do not send the last line as it may be incomplete.
|
|
465
|
+
sentInput = 0
|
|
466
|
+
while not stop_event.is_set():
|
|
467
|
+
if sentInput < len(keyPressesIn) - 1 :
|
|
468
|
+
stream.write(''.join(keyPressesIn[sentInput]).encode())
|
|
469
|
+
stream.flush()
|
|
470
|
+
host.output.append(' $ ' + ''.join(keyPressesIn[sentInput]).encode().decode().replace('\n', '↵'))
|
|
471
|
+
host.stdout.append(' $ ' + ''.join(keyPressesIn[sentInput]).encode().decode().replace('\n', '↵'))
|
|
472
|
+
sentInput += 1
|
|
473
|
+
else:
|
|
474
|
+
time.sleep(0.1)
|
|
475
|
+
if sentInput < len(keyPressesIn) - 1 :
|
|
476
|
+
print(f"Warning: {len(keyPressesIn)-sentInput} key presses are not sent before the process is terminated!")
|
|
477
|
+
# # send the last line
|
|
478
|
+
# if keyPressesIn and keyPressesIn[-1]:
|
|
479
|
+
# stream.write(''.join(keyPressesIn[-1]).encode())
|
|
480
|
+
# stream.flush()
|
|
481
|
+
# host.output.append(' $ ' + ''.join(keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
|
|
482
|
+
# host.stdout.append(' $ ' + ''.join(keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
|
|
483
|
+
return sentInput
|
|
484
|
+
|
|
485
|
+
def ssh_command(host, sem, timeout=60,passwds=None):
|
|
486
|
+
'''
|
|
487
|
+
Run the command on the host. Will format the commands accordingly. Main execution function.
|
|
488
|
+
|
|
489
|
+
Args:
|
|
490
|
+
host (Host): The host object
|
|
491
|
+
sem (threading.Semaphore): The semaphore to limit the number of concurrent SSH sessions
|
|
492
|
+
timeout (int, optional): The timeout for the command. Defaults to 60.
|
|
493
|
+
passwds (str, optional): The password for the host. Defaults to None.
|
|
494
|
+
|
|
495
|
+
Returns:
|
|
496
|
+
None
|
|
497
|
+
'''
|
|
498
|
+
global emo
|
|
499
|
+
with sem:
|
|
500
|
+
try:
|
|
501
|
+
host.username = None
|
|
502
|
+
host.address = host.name
|
|
503
|
+
if '@' in host.name:
|
|
504
|
+
host.username, host.address = host.name.rsplit('@',1)
|
|
505
|
+
if "#HOST#" in host.command.upper() or "#HOSTNAME#" in host.command.upper():
|
|
506
|
+
host.command = host.command.replace("#HOST#",host.address).replace("#HOSTNAME#",host.address).replace("#host#",host.address).replace("#hostname#",host.address)
|
|
507
|
+
if "#USER#" in host.command.upper() or "#USERNAME#" in host.command.upper():
|
|
508
|
+
if host.username:
|
|
509
|
+
host.command = host.command.replace("#USER#",host.username).replace("#USERNAME#",host.username).replace("#user#",host.username).replace("#username#",host.username)
|
|
510
|
+
else:
|
|
511
|
+
host.command = host.command.replace("#USER#",'CURRENT_USER').replace("#USERNAME#",'CURRENT_USER').replace("#user#",'CURRENT_USER').replace("#username#",'CURRENT_USER')
|
|
512
|
+
formatedCMD = []
|
|
513
|
+
if host.extraargs:
|
|
514
|
+
extraargs = host.extraargs.split()
|
|
515
|
+
else:
|
|
516
|
+
extraargs = []
|
|
517
|
+
if ipmiiInterfaceIPPrefix:
|
|
518
|
+
host.interface_ip_prefix = ipmiiInterfaceIPPrefix if host.ipmi and not host.interface_ip_prefix else host.interface_ip_prefix
|
|
519
|
+
if host.interface_ip_prefix:
|
|
520
|
+
try:
|
|
521
|
+
hostOctets = getIP(host.address,local=False).split('.')
|
|
522
|
+
prefixOctets = host.interface_ip_prefix.split('.')
|
|
523
|
+
host.address = '.'.join(prefixOctets[:3]+hostOctets[min(3,len(prefixOctets)):])
|
|
524
|
+
host.resolvedName = host.username + '@' if host.username else ''
|
|
525
|
+
host.resolvedName += host.address
|
|
526
|
+
except:
|
|
527
|
+
host.resolvedName = host.name
|
|
528
|
+
else:
|
|
529
|
+
host.resolvedName = host.name
|
|
530
|
+
if host.ipmi:
|
|
531
|
+
if host.command.startswith('ipmitool '):
|
|
532
|
+
host.command = host.command.replace('ipmitool ','')
|
|
533
|
+
if not host.username:
|
|
534
|
+
host.username = 'admin'
|
|
535
|
+
if passwds:
|
|
536
|
+
formatedCMD = ['bash','-c',f'ipmitool -H {host.address} -U {host.username} -P {passwds} {" ".join(extraargs)} {host.command}']
|
|
537
|
+
else:
|
|
538
|
+
formatedCMD = ['bash','-c',f'ipmitool -H {host.address} -U {host.username} {" ".join(extraargs)} {host.command}']
|
|
539
|
+
else:
|
|
540
|
+
if host.files:
|
|
541
|
+
if host.scp:
|
|
542
|
+
formatedCMD = ['scp','-rpB'] + extraargs +['--']+host.files+[f'{host.resolvedName}:{host.command}']
|
|
543
|
+
else:
|
|
544
|
+
formatedCMD = ['rsync','-ahlX','--partial','--inplace', '--info=name'] + extraargs +['--']+host.files+[f'{host.resolvedName}:{host.command}']
|
|
545
|
+
else:
|
|
546
|
+
formatedCMD = ['ssh'] + extraargs +['--']+ [host.resolvedName, host.command]
|
|
547
|
+
if passwds and sshpassAvailable:
|
|
548
|
+
formatedCMD = ['sshpass', '-p', passwds] + formatedCMD
|
|
549
|
+
elif passwds:
|
|
550
|
+
host.output.append('Warning: sshpass is not available. Please install sshpass to use password authentication.')
|
|
551
|
+
#host.stderr.append('Warning: sshpass is not available. Please install sshpass to use password authentication.')
|
|
552
|
+
host.output.append('Please provide password via live input or use ssh key authentication.')
|
|
553
|
+
# # try to send the password via keyPressesIn
|
|
554
|
+
# keyPressesIn[-1] = list(passwds) + ['\n']
|
|
555
|
+
# keyPressesIn.append([])
|
|
556
|
+
host.output.append('Running command: '+' '.join(formatedCMD))
|
|
557
|
+
#host.stdout = []
|
|
558
|
+
proc = subprocess.Popen(formatedCMD,stdout=subprocess.PIPE,stderr=subprocess.PIPE,stdin=subprocess.PIPE)
|
|
559
|
+
# create a thread to handle stdout
|
|
560
|
+
stdout_thread = threading.Thread(target=handle_reading_stream, args=(proc.stdout,host.stdout, host), daemon=True)
|
|
561
|
+
stdout_thread.start()
|
|
562
|
+
# create a thread to handle stderr
|
|
563
|
+
#host.stderr = []
|
|
564
|
+
stderr_thread = threading.Thread(target=handle_reading_stream, args=(proc.stderr,host.stderr, host), daemon=True)
|
|
565
|
+
stderr_thread.start()
|
|
566
|
+
# create a thread to handle stdin
|
|
567
|
+
stdin_stop_event = threading.Event()
|
|
568
|
+
stdin_thread = threading.Thread(target=handle_writing_stream, args=(proc.stdin,stdin_stop_event, host), daemon=True)
|
|
569
|
+
stdin_thread.start()
|
|
570
|
+
# Monitor the subprocess and terminate it after the timeout
|
|
571
|
+
start_time = time.time()
|
|
572
|
+
outLength = len(host.output)
|
|
573
|
+
while proc.poll() is None: # while the process is still running
|
|
574
|
+
if len(host.output) > outLength:
|
|
575
|
+
start_time = time.time()
|
|
576
|
+
outLength = len(host.output)
|
|
577
|
+
if timeout > 0:
|
|
578
|
+
if time.time() - start_time > timeout:
|
|
579
|
+
host.stderr.append('Timeout!')
|
|
580
|
+
host.output.append('Timeout!')
|
|
581
|
+
proc.send_signal(signal.SIGINT)
|
|
582
|
+
time.sleep(0.1)
|
|
583
|
+
|
|
584
|
+
proc.terminate()
|
|
585
|
+
break
|
|
586
|
+
elif time.time() - start_time > min(10, timeout // 2):
|
|
587
|
+
timeoutLine = f'Timeout in [{timeout - int(time.time() - start_time)}] seconds!'
|
|
588
|
+
if host.output and not host.output[-1].strip().startswith(timeoutLine):
|
|
589
|
+
# remove last line if it is a countdown
|
|
590
|
+
if host.output and host.output[-1].strip().endswith('] seconds!') and host.output[-1].strip().startswith('Timeout in ['):
|
|
591
|
+
host.output.pop()
|
|
592
|
+
host.printedLines -= 1
|
|
593
|
+
host.output.append(timeoutLine)
|
|
594
|
+
outLength = len(host.output)
|
|
595
|
+
if emo:
|
|
596
|
+
host.stderr.append('Ctrl C detected, Emergency Stop!')
|
|
597
|
+
host.output.append('Ctrl C detected, Emergency Stop!')
|
|
598
|
+
proc.send_signal(signal.SIGINT)
|
|
599
|
+
time.sleep(0.1)
|
|
600
|
+
proc.terminate()
|
|
601
|
+
break
|
|
602
|
+
time.sleep(0.1) # avoid busy-waiting
|
|
603
|
+
stdin_stop_event.set()
|
|
604
|
+
# Wait for output processing to complete
|
|
605
|
+
stdout_thread.join(timeout=1)
|
|
606
|
+
stderr_thread.join(timeout=1)
|
|
607
|
+
stdin_thread.join(timeout=1)
|
|
608
|
+
# here we handle the rest of the stdout after the subprocess returns
|
|
609
|
+
host.output.append(f'Pipe Closed. Trying to read the rest of the stdout...')
|
|
610
|
+
if not emo:
|
|
611
|
+
stdout = None
|
|
612
|
+
stderr = None
|
|
613
|
+
try:
|
|
614
|
+
stdout, stderr = proc.communicate(timeout=1)
|
|
615
|
+
except subprocess.TimeoutExpired:
|
|
616
|
+
pass
|
|
617
|
+
if stdout:
|
|
618
|
+
handle_reading_stream(io.BytesIO(stdout),host.stdout, host)
|
|
619
|
+
if stderr:
|
|
620
|
+
handle_reading_stream(io.BytesIO(stderr),host.stderr, host)
|
|
621
|
+
# if the last line in host.stderr is Connection to * closed., we will remove it
|
|
622
|
+
host.returncode = proc.poll()
|
|
623
|
+
if not host.returncode:
|
|
624
|
+
# process been killed via timeout or sigkill
|
|
625
|
+
if host.stderr and host.stderr[-1].strip().startswith('Timeout!'):
|
|
626
|
+
host.returncode = 124
|
|
627
|
+
elif host.stderr and host.stderr[-1].strip().startswith('Ctrl C detected, Emergency Stop!'):
|
|
628
|
+
host.returncode = 137
|
|
629
|
+
host.output.append(f'Command finished with return code {host.returncode}')
|
|
630
|
+
if host.stderr and host.stderr[-1].strip().startswith('Connection to ') and host.stderr[-1].strip().endswith(' closed.'):
|
|
631
|
+
host.stderr.pop()
|
|
632
|
+
except Exception as e:
|
|
633
|
+
import traceback
|
|
634
|
+
host.stderr.extend(str(e).split('\n'))
|
|
635
|
+
host.output.extend(str(e).split('\n'))
|
|
636
|
+
host.stderr.extend(traceback.format_exc().split('\n'))
|
|
637
|
+
host.output.extend(traceback.format_exc().split('\n'))
|
|
638
|
+
host.returncode = -1
|
|
639
|
+
# If using ipmi, we will try again using ssh if ipmi connection is not successful
|
|
640
|
+
if host.ipmi and host.returncode != 0 and any(['Unable to establish IPMI' in line for line in host.stderr]):
|
|
641
|
+
host.stderr = []
|
|
642
|
+
host.output.append('IPMI connection failed! Trying SSH connection...')
|
|
643
|
+
host.ipmi = False
|
|
644
|
+
host.interface_ip_prefix = None
|
|
645
|
+
host.command = 'ipmitool '+host.command if not host.command.startswith('ipmitool ') else host.command
|
|
646
|
+
ssh_command(host,sem,timeout,passwds)
|
|
647
|
+
# If transfering files, we will try again using scp if rsync connection is not successful
|
|
648
|
+
if host.files and not host.scp and host.returncode != 0 and host.stderr:
|
|
649
|
+
host.stderr = []
|
|
650
|
+
host.stdout = []
|
|
651
|
+
host.output.append('Rsync connection failed! Trying SCP connection...')
|
|
652
|
+
host.scp = True
|
|
653
|
+
ssh_command(host,sem,timeout,passwds)
|
|
654
|
+
|
|
655
|
+
def start_run_on_hosts(hosts, timeout=60,password=None,max_connections=4 * os.cpu_count()):
|
|
656
|
+
'''
|
|
657
|
+
Start running the command on the hosts. Wrapper function for ssh_command
|
|
658
|
+
|
|
659
|
+
Args:
|
|
660
|
+
hosts (list): A list of Host objects
|
|
661
|
+
timeout (int, optional): The timeout for the command. Defaults to 60.
|
|
662
|
+
password (str, optional): The password for the hosts. Defaults to None.
|
|
663
|
+
max_connections (int, optional): The maximum number of concurrent SSH sessions. Defaults to 4 * os.cpu_count().
|
|
664
|
+
|
|
665
|
+
Returns:
|
|
666
|
+
list: A list of threads that get started
|
|
667
|
+
'''
|
|
668
|
+
if len(hosts) == 0:
|
|
669
|
+
return []
|
|
670
|
+
sem = threading.Semaphore(max_connections) # Limit concurrent SSH sessions
|
|
671
|
+
threads = [threading.Thread(target=ssh_command, args=(host, sem,timeout,password), daemon=True) for host in hosts]
|
|
672
|
+
for thread in threads:
|
|
673
|
+
thread.start()
|
|
674
|
+
return threads
|
|
675
|
+
|
|
676
|
+
def get_hosts_to_display (hosts, max_num_hosts, hosts_to_display = None):
|
|
677
|
+
'''
|
|
678
|
+
Generate a list for the hosts to be displayed on the screen. This is used to display as much relevant information as possible.
|
|
679
|
+
|
|
680
|
+
Args:
|
|
681
|
+
hosts (list): A list of Host objects
|
|
682
|
+
max_num_hosts (int): The maximum number of hosts to be displayed
|
|
683
|
+
hosts_to_display (list, optional): The hosts that are currently displayed. Defaults to None.
|
|
684
|
+
|
|
685
|
+
Returns:
|
|
686
|
+
list: A list of Host objects to be displayed
|
|
687
|
+
'''
|
|
688
|
+
# We will sort the hosts by running -> failed -> finished -> waiting
|
|
689
|
+
# running: returncode is None and output is not empty (output will be appened immediately after the command is run)
|
|
690
|
+
# failed: returncode is not None and returncode is not 0
|
|
691
|
+
# finished: returncode is not None and returncode is 0
|
|
692
|
+
# waiting: returncode is None and output is empty
|
|
693
|
+
running_hosts = [host for host in hosts if host.returncode is None and host.output]
|
|
694
|
+
failed_hosts = [host for host in hosts if host.returncode is not None and host.returncode != 0]
|
|
695
|
+
finished_hosts = [host for host in hosts if host.returncode is not None and host.returncode == 0]
|
|
696
|
+
waiting_hosts = [host for host in hosts if host.returncode is None and not host.output]
|
|
697
|
+
new_hosts_to_display = (running_hosts + failed_hosts + finished_hosts + waiting_hosts)[:max_num_hosts]
|
|
698
|
+
if not hosts_to_display:
|
|
699
|
+
return new_hosts_to_display , {'running':len(running_hosts), 'failed':len(failed_hosts), 'finished':len(finished_hosts), 'waiting':len(waiting_hosts)}
|
|
700
|
+
# we will compare the new_hosts_to_display with the old one, if some hosts are not in their original position, we will change its printedLines to 0
|
|
701
|
+
for i, host in enumerate(new_hosts_to_display):
|
|
702
|
+
if host not in hosts_to_display:
|
|
703
|
+
host.printedLines = 0
|
|
704
|
+
elif i != hosts_to_display.index(host):
|
|
705
|
+
host.printedLines = 0
|
|
706
|
+
return new_hosts_to_display , {'running':len(running_hosts), 'failed':len(failed_hosts), 'finished':len(finished_hosts), 'waiting':len(waiting_hosts)}
|
|
707
|
+
|
|
708
|
+
def generate_display(stdscr, hosts, threads,lineToDisplay = -1,curserPosition = 0, min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window=DEFAULT_SINGLE_WINDOW):
|
|
709
|
+
try:
|
|
710
|
+
org_dim = stdscr.getmaxyx()
|
|
711
|
+
new_configured = True
|
|
712
|
+
# To do this, first we need to know the size of the terminal
|
|
713
|
+
max_y, max_x = org_dim
|
|
714
|
+
# we will use one line to print the aggregated stats for the hosts.
|
|
715
|
+
max_y -= 1
|
|
716
|
+
# bound the min_char_len and min_line_len to between 1 and the max_x -1 and max_y -1
|
|
717
|
+
min_char_len_local = min(max(1,min_char_len),max_x-1)
|
|
718
|
+
min_line_len_local = min(max(1,min_line_len),max_y-1)
|
|
719
|
+
if single_window:
|
|
720
|
+
min_char_len_local = max_x-1
|
|
721
|
+
min_line_len_local = max_y-1
|
|
722
|
+
# raise zero division error if the terminal is too small
|
|
723
|
+
if max_x < 2 or max_y < 2:
|
|
724
|
+
raise ZeroDivisionError
|
|
725
|
+
if min_char_len_local < 1 or min_line_len_local < 1:
|
|
726
|
+
raise ZeroDivisionError
|
|
727
|
+
# We need to figure out how many hosts we can fit in the terminal
|
|
728
|
+
# We will need at least 2 lines per host, one for its name, one for its output
|
|
729
|
+
# Each line will be at least 61 characters long (60 for the output, 1 for the borders)
|
|
730
|
+
max_num_hosts_x = max_x // (min_char_len_local + 1)
|
|
731
|
+
max_num_hosts_y = max_y // (min_line_len_local + 1)
|
|
732
|
+
max_num_hosts = max_num_hosts_x * max_num_hosts_y
|
|
733
|
+
if max_num_hosts < 1:
|
|
734
|
+
raise ZeroDivisionError
|
|
735
|
+
hosts_to_display , host_stats = get_hosts_to_display(hosts, max_num_hosts)
|
|
736
|
+
if len(hosts_to_display) == 0:
|
|
737
|
+
raise ZeroDivisionError
|
|
738
|
+
# Now we calculate the actual number of hosts we will display for x and y
|
|
739
|
+
optimal_len_x = max(min_char_len_local, 80)
|
|
740
|
+
num_hosts_x = max(min(max_num_hosts_x, max_x // optimal_len_x),1)
|
|
741
|
+
num_hosts_y = len(hosts_to_display) // num_hosts_x
|
|
742
|
+
while num_hosts_y > max_num_hosts_y:
|
|
743
|
+
num_hosts_x += 1
|
|
744
|
+
# round up for num_hosts_y
|
|
745
|
+
num_hosts_y = len(hosts_to_display) // num_hosts_x + (len(hosts_to_display) % num_hosts_x > 0)
|
|
746
|
+
if num_hosts_x > max_num_hosts_x:
|
|
747
|
+
num_hosts_x = 1
|
|
748
|
+
num_hosts_y = len(hosts_to_display)
|
|
749
|
+
while num_hosts_y > max_num_hosts_y:
|
|
750
|
+
num_hosts_x += 1
|
|
751
|
+
num_hosts_y = len(hosts_to_display) // num_hosts_x + (len(hosts_to_display) % num_hosts_x > 0)
|
|
752
|
+
break
|
|
753
|
+
|
|
754
|
+
# We calculate the size of each window
|
|
755
|
+
host_window_height = max_y // num_hosts_y
|
|
756
|
+
host_window_width = max_x // num_hosts_x
|
|
757
|
+
if host_window_height < 1 or host_window_width < 1:
|
|
758
|
+
raise ZeroDivisionError
|
|
759
|
+
|
|
760
|
+
old_stat = ''
|
|
761
|
+
old_bottom_stat = ''
|
|
762
|
+
old_cursor_position = -1
|
|
763
|
+
# we refresh the screen every 0.1 seconds
|
|
764
|
+
last_refresh_time = time.perf_counter()
|
|
765
|
+
stdscr.clear()
|
|
766
|
+
#host_window.refresh()
|
|
767
|
+
global keyPressesIn
|
|
768
|
+
stdscr.nodelay(True)
|
|
769
|
+
# we generate a stats window at the top of the screen
|
|
770
|
+
stat_window = curses.newwin(1, max_x, 0, 0)
|
|
771
|
+
# We create a window for each host
|
|
772
|
+
host_windows = []
|
|
773
|
+
for i, host in enumerate(hosts_to_display):
|
|
774
|
+
# We calculate the coordinates of the window
|
|
775
|
+
# We need to add 1 to y for the stats line
|
|
776
|
+
y = (i // num_hosts_x) * host_window_height +1
|
|
777
|
+
x = (i % num_hosts_x) * host_window_width
|
|
778
|
+
#print(f"Creating a window at {y},{x}")
|
|
779
|
+
# We create the window
|
|
780
|
+
host_window = curses.newwin(host_window_height, host_window_width, y, x)
|
|
781
|
+
host_windows.append(host_window)
|
|
782
|
+
# If there is space left, we will draw the bottom border
|
|
783
|
+
bottom_border = None
|
|
784
|
+
if y + host_window_height < org_dim[0]:
|
|
785
|
+
bottom_border = curses.newwin(1, max_x, y + host_window_height, 0)
|
|
786
|
+
bottom_border.clear()
|
|
787
|
+
bottom_border.addstr(0, 0, '-' * (max_x - 1))
|
|
788
|
+
bottom_border.refresh()
|
|
789
|
+
while host_stats['running'] > 0 or host_stats['waiting'] > 0:
|
|
790
|
+
# Check for keypress
|
|
791
|
+
key = stdscr.getch()
|
|
792
|
+
if key != -1: # -1 means no keypress
|
|
793
|
+
# we store the keypresses in a list of lists.
|
|
794
|
+
# Each list is a list of characters to be sent to the stdin of the process at once.
|
|
795
|
+
# When we encounter a newline, we add a new list to the list of lists. ( a new line of input )
|
|
796
|
+
# with open('keylog.txt','a') as f:
|
|
797
|
+
# f.write(str(key)+'\n')
|
|
798
|
+
if key == 410: # 410 is the key code for resize
|
|
799
|
+
raise Exception('Terminal size changed. Please reconfigure window.')
|
|
800
|
+
# We handle positional keys
|
|
801
|
+
# uparrow: 259; downarrow: 258; leftarrow: 260; rightarrow: 261
|
|
802
|
+
# pageup: 339; pagedown: 338; home: 262; end: 360
|
|
803
|
+
elif key in [259, 258, 260, 261, 339, 338, 262, 360]:
|
|
804
|
+
# if the key is up arrow, we will move the line to display up
|
|
805
|
+
if key == 259: # 259 is the key code for up arrow
|
|
806
|
+
lineToDisplay = max(lineToDisplay - 1, -len(keyPressesIn))
|
|
807
|
+
# if the key is down arrow, we will move the line to display down
|
|
808
|
+
elif key == 258: # 258 is the key code for down arrow
|
|
809
|
+
lineToDisplay = min(lineToDisplay + 1, -1)
|
|
810
|
+
# if the key is left arrow, we will move the cursor left
|
|
811
|
+
elif key == 260: # 260 is the key code for left arrow
|
|
812
|
+
curserPosition = min(max(curserPosition - 1, 0), len(keyPressesIn[lineToDisplay]) -1)
|
|
813
|
+
# if the key is right arrow, we will move the cursor right
|
|
814
|
+
elif key == 261: # 261 is the key code for right arrow
|
|
815
|
+
curserPosition = max(min(curserPosition + 1, len(keyPressesIn[lineToDisplay])), 0)
|
|
816
|
+
# if the key is page up, we will move the line to display up by 5 lines
|
|
817
|
+
elif key == 339: # 339 is the key code for page up
|
|
818
|
+
lineToDisplay = max(lineToDisplay - 5, -len(keyPressesIn))
|
|
819
|
+
# if the key is page down, we will move the line to display down by 5 lines
|
|
820
|
+
elif key == 338: # 338 is the key code for page down
|
|
821
|
+
lineToDisplay = min(lineToDisplay + 5, -1)
|
|
822
|
+
# if the key is home, we will move the cursor to the beginning of the line
|
|
823
|
+
elif key == 262: # 262 is the key code for home
|
|
824
|
+
curserPosition = 0
|
|
825
|
+
# if the key is end, we will move the cursor to the end of the line
|
|
826
|
+
elif key == 360: # 360 is the key code for end
|
|
827
|
+
curserPosition = len(keyPressesIn[lineToDisplay])
|
|
828
|
+
# We are left with these are keys that mofidy the current line.
|
|
829
|
+
else:
|
|
830
|
+
# This means the user have done scrolling and is committing to modify the current line.
|
|
831
|
+
if lineToDisplay < -1:
|
|
832
|
+
# We overwrite the last line (current working line) with the line to display, removing the newline at the end
|
|
833
|
+
keyPressesIn[-1] = keyPressesIn[lineToDisplay][:-1]
|
|
834
|
+
lineToDisplay = -1
|
|
835
|
+
curserPosition = max(0, min(curserPosition, len(keyPressesIn[lineToDisplay])))
|
|
836
|
+
if key == 10: # 10 is the key code for newline
|
|
837
|
+
keyPressesIn[-1].append(chr(key))
|
|
838
|
+
keyPressesIn.append([])
|
|
839
|
+
lineToDisplay = -1
|
|
840
|
+
curserPosition = 0
|
|
841
|
+
# if the key is backspace, we will remove the last character from the last list
|
|
842
|
+
elif key in [8,263]: # 8 is the key code for backspace
|
|
843
|
+
if curserPosition > 0:
|
|
844
|
+
keyPressesIn[lineToDisplay].pop(curserPosition - 1)
|
|
845
|
+
curserPosition -= 1
|
|
846
|
+
# if the key is ESC, we will clear the last list
|
|
847
|
+
elif key == 27: # 27 is the key code for ESC
|
|
848
|
+
keyPressesIn[-1] = []
|
|
849
|
+
curserPosition = 0
|
|
850
|
+
# ignore delete key
|
|
851
|
+
elif key in [127, 330]: # 330 is the key code for delete key
|
|
852
|
+
# delete the character at the cursor position
|
|
853
|
+
if curserPosition < len(keyPressesIn[lineToDisplay]):
|
|
854
|
+
keyPressesIn[lineToDisplay].pop(curserPosition)
|
|
855
|
+
else:
|
|
856
|
+
# if the key is not a special key, we will add it
|
|
857
|
+
keyPressesIn[lineToDisplay].insert(curserPosition, chr(key))
|
|
858
|
+
curserPosition += 1
|
|
859
|
+
# reconfigure when the terminal size changes
|
|
860
|
+
# raise Exception when max_y or max_x is changed, let parent handle reconfigure
|
|
861
|
+
if org_dim != stdscr.getmaxyx():
|
|
862
|
+
raise Exception('Terminal size changed. Please reconfigure window.')
|
|
863
|
+
# We generate the aggregated stats if user did not input anything
|
|
864
|
+
if not keyPressesIn[lineToDisplay]:
|
|
865
|
+
stats = '┍'+ f"Total: {len(hosts)} Running: {host_stats['running']} Failed: {host_stats['failed']} Finished: {host_stats['finished']} Waiting: {host_stats['waiting']}"[:max_x - 2].center(max_x - 2, "━")
|
|
866
|
+
else:
|
|
867
|
+
# we use the stat bar to display the key presses
|
|
868
|
+
encodedLine = ''.join(keyPressesIn[lineToDisplay]).encode().decode().strip('\n') + ' '
|
|
869
|
+
# # add the flashing indicator at the curse position
|
|
870
|
+
# if time.perf_counter() % 1 > 0.5:
|
|
871
|
+
# encodedLine = encodedLine[:curserPosition] + '█' + encodedLine[curserPosition:]
|
|
872
|
+
# else:
|
|
873
|
+
# encodedLine = encodedLine[:curserPosition] + ' ' + encodedLine[curserPosition:]
|
|
874
|
+
stats = '┍'+ f"Send CMD: {encodedLine}"[:max_x - 2].center(max_x - 2, "━")
|
|
875
|
+
if bottom_border:
|
|
876
|
+
bottom_stats = '└'+ f"Total: {len(hosts)} Running: {host_stats['running']} Failed: {host_stats['failed']} Finished: {host_stats['finished']} Waiting: {host_stats['waiting']}"[:max_x - 2].center(max_x - 2, "─")
|
|
877
|
+
if bottom_stats != old_bottom_stat:
|
|
878
|
+
old_bottom_stat = bottom_stats
|
|
879
|
+
bottom_border.clear()
|
|
880
|
+
bottom_border.addstr(0, 0, bottom_stats)
|
|
881
|
+
bottom_border.refresh()
|
|
882
|
+
if stats != old_stat or curserPosition != old_cursor_position:
|
|
883
|
+
old_stat = stats
|
|
884
|
+
old_cursor_position = curserPosition
|
|
885
|
+
# calculate the real curser position in stats as we centered the stats
|
|
886
|
+
if 'Send CMD: ' in stats:
|
|
887
|
+
curserPositionStats = min(min(curserPosition,len(encodedLine) -1) + stats.find('Send CMD: ')+len('Send CMD: '), max_x -2)
|
|
888
|
+
else:
|
|
889
|
+
curserPositionStats = max_x -2
|
|
890
|
+
stat_window.clear()
|
|
891
|
+
#stat_window.addstr(0, 0, stats)
|
|
892
|
+
# add the line with curser that inverses the color at the curser position
|
|
893
|
+
stat_window.addstr(0, 0, stats[:curserPositionStats], curses.color_pair(1))
|
|
894
|
+
stat_window.addstr(0, curserPositionStats, stats[curserPositionStats], curses.color_pair(2))
|
|
895
|
+
stat_window.addstr(0, curserPositionStats + 1, stats[curserPositionStats + 1:], curses.color_pair(1))
|
|
896
|
+
stat_window.refresh()
|
|
897
|
+
# set the maximum refresh rate to 100 Hz
|
|
898
|
+
if time.perf_counter() - last_refresh_time < 0.01:
|
|
899
|
+
time.sleep(max(0,0.01 - time.perf_counter() + last_refresh_time))
|
|
900
|
+
#stdscr.clear()
|
|
901
|
+
hosts_to_display, host_stats = get_hosts_to_display(hosts, max_num_hosts,hosts_to_display)
|
|
902
|
+
for host_window, host in zip(host_windows, hosts_to_display):
|
|
903
|
+
# we will only update the window if there is new output or the window is not fully printed
|
|
904
|
+
if new_configured or host.printedLines < len(host.output):
|
|
905
|
+
try:
|
|
906
|
+
host_window.clear()
|
|
907
|
+
# we will try to center the name of the host with ┼ at the beginning and end and ─ in between
|
|
908
|
+
linePrintOut = f'┼{(host.name+":["+host.command+"]")[:host_window_width - 2].center(host_window_width - 1, "─")}'.replace('\n', ' ').replace('\r', ' ').strip()
|
|
909
|
+
host_window.addstr(0, 0, linePrintOut)
|
|
910
|
+
# we will display the latest outputs of the host as much as we can
|
|
911
|
+
for i, line in enumerate(host.output[-(host_window_height - 1):]):
|
|
912
|
+
# print(f"Printng a line at {i + 1} with length of {len('│'+line[:host_window_width - 1])}")
|
|
913
|
+
# time.sleep(10)
|
|
914
|
+
linePrintOut = ('│'+line[:host_window_width - 2].replace('\n', ' ').replace('\r', ' ')).strip()
|
|
915
|
+
host_window.addstr(i + 1, 0, linePrintOut)
|
|
916
|
+
# we draw the rest of the available lines
|
|
917
|
+
for i in range(len(host.output), host_window_height - 1):
|
|
918
|
+
# print(f"Printng a line at {i + 1} with length of {len('│')}")
|
|
919
|
+
host_window.addstr(i + 1, 0, '│')
|
|
920
|
+
host.printedLines = len(host.output)
|
|
921
|
+
host_window.refresh()
|
|
922
|
+
except Exception as e:
|
|
923
|
+
# import traceback
|
|
924
|
+
# print(str(e).strip())
|
|
925
|
+
# print(traceback.format_exc().strip())
|
|
926
|
+
if org_dim != stdscr.getmaxyx():
|
|
927
|
+
raise Exception('Terminal size changed. Please reconfigure window.')
|
|
928
|
+
new_configured = False
|
|
929
|
+
last_refresh_time = time.perf_counter()
|
|
930
|
+
|
|
931
|
+
except ZeroDivisionError:
|
|
932
|
+
# terminial is too small, we skip the display
|
|
933
|
+
pass
|
|
934
|
+
except Exception as e:
|
|
935
|
+
stdscr.clear()
|
|
936
|
+
stdscr.refresh()
|
|
937
|
+
generate_display(stdscr, hosts, threads, lineToDisplay, curserPosition, min_char_len, min_line_len, single_window)
|
|
938
|
+
|
|
939
|
+
def curses_print(stdscr, hosts, threads, min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window = DEFAULT_SINGLE_WINDOW):
|
|
940
|
+
'''
|
|
941
|
+
Print the output of the hosts on the screen
|
|
942
|
+
|
|
943
|
+
Args:
|
|
944
|
+
stdscr (curses.window): The curses window to print the output
|
|
945
|
+
hosts (list): A list of Host objects
|
|
946
|
+
threads (list): A list of threads that are running the commands
|
|
947
|
+
|
|
948
|
+
Returns:
|
|
949
|
+
None
|
|
950
|
+
'''
|
|
951
|
+
# We create all the windows we need
|
|
952
|
+
# We initialize the color pair
|
|
953
|
+
curses.start_color()
|
|
954
|
+
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
|
|
955
|
+
curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_WHITE)
|
|
956
|
+
curses.init_pair(3, curses.COLOR_RED, curses.COLOR_BLACK)
|
|
957
|
+
curses.init_pair(4, curses.COLOR_GREEN, curses.COLOR_BLACK)
|
|
958
|
+
curses.init_pair(5, curses.COLOR_YELLOW, curses.COLOR_BLACK)
|
|
959
|
+
curses.init_pair(6, curses.COLOR_BLUE, curses.COLOR_BLACK)
|
|
960
|
+
curses.init_pair(7, curses.COLOR_MAGENTA, curses.COLOR_BLACK)
|
|
961
|
+
curses.init_pair(8, curses.COLOR_CYAN, curses.COLOR_BLACK)
|
|
962
|
+
curses.init_pair(9, curses.COLOR_WHITE, curses.COLOR_RED)
|
|
963
|
+
curses.init_pair(10, curses.COLOR_WHITE, curses.COLOR_GREEN)
|
|
964
|
+
curses.init_pair(11, curses.COLOR_WHITE, curses.COLOR_YELLOW)
|
|
965
|
+
curses.init_pair(12, curses.COLOR_WHITE, curses.COLOR_BLUE)
|
|
966
|
+
curses.init_pair(13, curses.COLOR_WHITE, curses.COLOR_MAGENTA)
|
|
967
|
+
curses.init_pair(14, curses.COLOR_WHITE, curses.COLOR_CYAN)
|
|
968
|
+
curses.init_pair(15, curses.COLOR_BLACK, curses.COLOR_RED)
|
|
969
|
+
curses.init_pair(16, curses.COLOR_BLACK, curses.COLOR_GREEN)
|
|
970
|
+
curses.init_pair(17, curses.COLOR_BLACK, curses.COLOR_YELLOW)
|
|
971
|
+
curses.init_pair(18, curses.COLOR_BLACK, curses.COLOR_BLUE)
|
|
972
|
+
curses.init_pair(19, curses.COLOR_BLACK, curses.COLOR_MAGENTA)
|
|
973
|
+
curses.init_pair(20, curses.COLOR_BLACK, curses.COLOR_CYAN)
|
|
974
|
+
generate_display(stdscr, hosts, threads,min_char_len = min_char_len, min_line_len = min_line_len, single_window = single_window)
|
|
975
|
+
|
|
976
|
+
def print_output(hosts,usejson = False,quiet = False,greppable = False):
|
|
977
|
+
'''
|
|
978
|
+
Print / generate the output of the hosts to the terminal
|
|
979
|
+
|
|
980
|
+
Args:
|
|
981
|
+
hosts (list): A list of Host objects
|
|
982
|
+
usejson (bool, optional): Whether to print the output in JSON format. Defaults to False.
|
|
983
|
+
quiet (bool, optional): Whether to print the output. Defaults to False.
|
|
984
|
+
|
|
985
|
+
Returns:
|
|
986
|
+
str: The pretty output generated
|
|
987
|
+
'''
|
|
988
|
+
global keyPressesIn
|
|
989
|
+
global global_suppress_printout
|
|
990
|
+
hosts = [dict(host) for host in hosts]
|
|
991
|
+
if usejson:
|
|
992
|
+
# [print(dict(host)) for host in hosts]
|
|
993
|
+
#print(json.dumps([dict(host) for host in hosts],indent=4))
|
|
994
|
+
rtnStr = json.dumps(hosts,indent=4)
|
|
995
|
+
elif greppable:
|
|
996
|
+
outputs = {}
|
|
997
|
+
# transform hosts to dictionaries
|
|
998
|
+
for host in hosts:
|
|
999
|
+
hostPrintOut = f" | cmd: {host['command']} | stdout: "+'↵ '.join(host['stdout'])
|
|
1000
|
+
if host['stderr']:
|
|
1001
|
+
if host['stderr'][0].strip().startswith('ssh: connect to host '):
|
|
1002
|
+
host['stderr'][0] = 'SSH not reachable!'
|
|
1003
|
+
hostPrintOut += " | stderr: "+'↵ '.join(host['stderr'])
|
|
1004
|
+
hostPrintOut += f" | return_code: {host['returncode']}"
|
|
1005
|
+
if hostPrintOut not in outputs:
|
|
1006
|
+
outputs[hostPrintOut] = [host['name']]
|
|
1007
|
+
else:
|
|
1008
|
+
outputs[hostPrintOut].append(host['name'])
|
|
1009
|
+
rtnStr = ''
|
|
1010
|
+
for output, hosts in outputs.items():
|
|
1011
|
+
rtnStr += f"{','.join(hosts)}{output}\n"
|
|
1012
|
+
if keyPressesIn[-1]:
|
|
1013
|
+
CMDsOut = [''.join(cmd).encode('unicode_escape').decode().replace('\\n', '↵') for cmd in keyPressesIn if cmd]
|
|
1014
|
+
rtnStr += 'User Inputs: '+ '\nUser Inputs: '.join(CMDsOut)
|
|
1015
|
+
#rtnStr += '\n'
|
|
1016
|
+
else:
|
|
1017
|
+
outputs = {}
|
|
1018
|
+
for host in hosts:
|
|
1019
|
+
if global_suppress_printout:
|
|
1020
|
+
if host['returncode'] == 0:
|
|
1021
|
+
continue
|
|
1022
|
+
hostPrintOut = f" Command:\n {host['command']}\n"
|
|
1023
|
+
hostPrintOut += " stdout:\n "+'\n '.join(host['stdout'])
|
|
1024
|
+
if host['stderr']:
|
|
1025
|
+
if host['stderr'][0].strip().startswith('ssh: connect to host '):
|
|
1026
|
+
host['stderr'][0] = 'SSH not reachable!'
|
|
1027
|
+
hostPrintOut += "\n stderr:\n "+'\n '.join(host['stderr'])
|
|
1028
|
+
hostPrintOut += f"\n return_code: {host['returncode']}"
|
|
1029
|
+
if hostPrintOut not in outputs:
|
|
1030
|
+
outputs[hostPrintOut] = [host['name']]
|
|
1031
|
+
else:
|
|
1032
|
+
outputs[hostPrintOut].append(host['name'])
|
|
1033
|
+
rtnStr = ''
|
|
1034
|
+
for output, hosts in outputs.items():
|
|
1035
|
+
if global_suppress_printout:
|
|
1036
|
+
rtnStr += f'Error returncode produced by {hosts}:\n'
|
|
1037
|
+
rtnStr += output+'\n'
|
|
1038
|
+
else:
|
|
1039
|
+
rtnStr += '*'*80+'\n'
|
|
1040
|
+
rtnStr += f"These hosts: {hosts} have a response of:\n"
|
|
1041
|
+
rtnStr += output+'\n'
|
|
1042
|
+
if not global_suppress_printout or outputs:
|
|
1043
|
+
rtnStr += '*'*80+'\n'
|
|
1044
|
+
if keyPressesIn[-1]:
|
|
1045
|
+
CMDsOut = [''.join(cmd).encode('unicode_escape').decode().replace('\\n', '↵') for cmd in keyPressesIn if cmd]
|
|
1046
|
+
#rtnStr += f"Key presses: {''.join(keyPressesIn).encode('unicode_escape').decode()}\n"
|
|
1047
|
+
#rtnStr += f"Key presses: {keyPressesIn}\n"
|
|
1048
|
+
rtnStr += "User Inputs: \n "
|
|
1049
|
+
rtnStr += '\n '.join(CMDsOut)
|
|
1050
|
+
rtnStr += '\n'
|
|
1051
|
+
keyPressesIn = [[]]
|
|
1052
|
+
if global_suppress_printout and not outputs:
|
|
1053
|
+
rtnStr += 'Success'
|
|
1054
|
+
if not quiet:
|
|
1055
|
+
print(rtnStr)
|
|
1056
|
+
return rtnStr
|
|
1057
|
+
|
|
1058
|
+
sshConfigged = False
|
|
1059
|
+
def verify_ssh_config():
|
|
1060
|
+
'''
|
|
1061
|
+
Verify that ~/.ssh/config exists and contains the line "StrictHostKeyChecking no"
|
|
1062
|
+
|
|
1063
|
+
Args:
|
|
1064
|
+
None
|
|
1065
|
+
|
|
1066
|
+
Returns:
|
|
1067
|
+
None
|
|
1068
|
+
'''
|
|
1069
|
+
global sshConfigged
|
|
1070
|
+
if not sshConfigged:
|
|
1071
|
+
# first we make sure ~/.ssh/config exists
|
|
1072
|
+
config = ''
|
|
1073
|
+
if not os.path.exists(os.path.expanduser('~/.ssh')):
|
|
1074
|
+
os.makedirs(os.path.expanduser('~/.ssh'))
|
|
1075
|
+
if os.path.exists(os.path.expanduser('~/.ssh/config')):
|
|
1076
|
+
with open(os.path.expanduser('~/.ssh/config'),'r') as f:
|
|
1077
|
+
config = f.read()
|
|
1078
|
+
if config:
|
|
1079
|
+
if 'StrictHostKeyChecking no' not in config:
|
|
1080
|
+
with open(os.path.expanduser('~/.ssh/config'),'a') as f:
|
|
1081
|
+
f.write('\nHost *\n\tStrictHostKeyChecking no\n')
|
|
1082
|
+
else:
|
|
1083
|
+
with open(os.path.expanduser('~/.ssh/config'),'w') as f:
|
|
1084
|
+
f.write('Host *\n\tStrictHostKeyChecking no\n')
|
|
1085
|
+
sshConfigged = True
|
|
1086
|
+
|
|
1087
|
+
def signal_handler(sig, frame):
|
|
1088
|
+
'''
|
|
1089
|
+
Handle the Ctrl C signal
|
|
1090
|
+
|
|
1091
|
+
Args:
|
|
1092
|
+
sig (int): The signal
|
|
1093
|
+
frame (frame): The frame
|
|
1094
|
+
|
|
1095
|
+
Returns:
|
|
1096
|
+
None
|
|
1097
|
+
'''
|
|
1098
|
+
global emo
|
|
1099
|
+
if not emo:
|
|
1100
|
+
print('Ctrl C caught, exiting...')
|
|
1101
|
+
emo = True
|
|
1102
|
+
else:
|
|
1103
|
+
print('Ctrl C caught again, exiting immediately!')
|
|
1104
|
+
# wait for 0.1 seconds to allow the threads to exit
|
|
1105
|
+
time.sleep(0.1)
|
|
1106
|
+
os.system(f'pkill -ef {os.path.basename(__file__)}')
|
|
1107
|
+
sys.exit(0)
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,single_window = DEFAULT_SINGLE_WINDOW):
|
|
1111
|
+
global gloablUnavailableHosts
|
|
1112
|
+
threads = start_run_on_hosts(hosts, timeout=timeout,password=password,max_connections=max_connections)
|
|
1113
|
+
if not quiet and threads and not returnUnfinished and any([thread.is_alive() for thread in threads]) and sys.stdout.isatty() and os.get_terminal_size() and os.get_terminal_size().columns > 10:
|
|
1114
|
+
curses.wrapper(curses_print, hosts, threads, min_char_len = curses_min_char_len, min_line_len = curses_min_line_len, single_window = single_window)
|
|
1115
|
+
if not returnUnfinished:
|
|
1116
|
+
# wait until all hosts have a return code
|
|
1117
|
+
while any([host.returncode is None for host in hosts]):
|
|
1118
|
+
time.sleep(0.1)
|
|
1119
|
+
for thread in threads:
|
|
1120
|
+
thread.join(timeout=3)
|
|
1121
|
+
# update the unavailable hosts and global unavailable hosts
|
|
1122
|
+
if willUpdateUnreachableHosts:
|
|
1123
|
+
unavailableHosts.update([host.name for host in hosts if host.stderr and ('No route to host' in host.stderr[0].strip() or host.stderr[0].strip().startswith('Timeout!'))])
|
|
1124
|
+
gloablUnavailableHosts.update(unavailableHosts)
|
|
1125
|
+
# print the output, if the output of multiple hosts are the same, we aggragate them
|
|
1126
|
+
if not called:
|
|
1127
|
+
print_output(hosts,json,greppable=greppable)
|
|
1128
|
+
|
|
1129
|
+
@cache_decorator
|
|
1130
|
+
def formHostStr(host) -> str:
|
|
1131
|
+
"""
|
|
1132
|
+
Forms a comma-separated string of hosts.
|
|
1133
|
+
|
|
1134
|
+
Args:
|
|
1135
|
+
host: A string or a set of hosts.
|
|
1136
|
+
|
|
1137
|
+
Returns:
|
|
1138
|
+
A string representing the hosts, separated by commas.
|
|
1139
|
+
"""
|
|
1140
|
+
if not host or len(host) == 0:
|
|
1141
|
+
return 'EMPTY_HOSTS'
|
|
1142
|
+
if type(host) is str:
|
|
1143
|
+
host = set(host.replace(',',' ').replace('\n',' ').replace('\r',' ').replace('\t',' ').replace(';', ' ').replace('|', ' ').replace('/', ' ').replace('&',' ').split())
|
|
1144
|
+
else:
|
|
1145
|
+
host = set(host)
|
|
1146
|
+
if 'local_shell' in host:
|
|
1147
|
+
host.remove('local_shell')
|
|
1148
|
+
host.add('localhost')
|
|
1149
|
+
host = ','.join(host)
|
|
1150
|
+
return host
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
@cache_decorator
|
|
1154
|
+
def __formCommandArgStr(oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
1155
|
+
quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,max_connections=DEFAULT_MAX_CONNECTIONS,
|
|
1156
|
+
files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,
|
|
1157
|
+
scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
|
|
1158
|
+
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,skip_hosts = DEFAULT_SKIP_HOSTS,
|
|
1159
|
+
file_sync = False, error_only = DEFAULT_ERROR_ONLY,
|
|
1160
|
+
shortend = False) -> str:
|
|
1161
|
+
argsList = []
|
|
1162
|
+
if oneonone: argsList.append('--oneonone' if not shortend else '-11')
|
|
1163
|
+
if timeout and timeout != DEFAULT_TIMEOUT: argsList.append(f'--timeout={timeout}' if not shortend else f'-t={timeout}')
|
|
1164
|
+
if password and password != DEFAULT_PASSWORD: argsList.append(f'--password="{password}"' if not shortend else f'-p="{password}"')
|
|
1165
|
+
if quiet: argsList.append('--quiet' if not shortend else '-q')
|
|
1166
|
+
if json: argsList.append('--json' if not shortend else '-j')
|
|
1167
|
+
if max_connections and max_connections != DEFAULT_MAX_CONNECTIONS: argsList.append(f'--maxconnections={max_connections}' if not shortend else f'-m={max_connections}')
|
|
1168
|
+
if files: argsList.extend([f'--file="{file}"' for file in files] if not shortend else [f'-f="{file}"' for file in files])
|
|
1169
|
+
if ipmi: argsList.append('--ipmi')
|
|
1170
|
+
if interface_ip_prefix and interface_ip_prefix != DEFAULT_INTERFACE_IP_PREFIX: argsList.append(f'--interface_ip_prefix="{interface_ip_prefix}"' if not shortend else f'-pre="{interface_ip_prefix}"')
|
|
1171
|
+
if scp: argsList.append('--scp')
|
|
1172
|
+
if username and username != DEFAULT_USERNAME: argsList.append(f'--username="{username}"' if not shortend else f'-u="{username}"')
|
|
1173
|
+
if extraargs and extraargs != DEFAULT_EXTRA_ARGS: argsList.append(f'--extraargs="{extraargs}"' if not shortend else f'-ea="{extraargs}"')
|
|
1174
|
+
if skipUnreachable: argsList.append('--skipUnreachable' if not shortend else '-su')
|
|
1175
|
+
if no_env: argsList.append('--no_env')
|
|
1176
|
+
if greppable: argsList.append('--greppable' if not shortend else '-g')
|
|
1177
|
+
if error_only: argsList.append('--error_only' if not shortend else '-eo')
|
|
1178
|
+
if skip_hosts and skip_hosts != DEFAULT_SKIP_HOSTS: argsList.append(f'--skip_hosts="{skip_hosts}"' if not shortend else f'-sh="{skip_hosts}"')
|
|
1179
|
+
if file_sync: argsList.append('--file_sync' if not shortend else '-fs')
|
|
1180
|
+
return ' '.join(argsList)
|
|
1181
|
+
|
|
1182
|
+
def getStrCommand(hosts,commands,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
1183
|
+
quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,called = DEFAULT_CALLED,max_connections=DEFAULT_MAX_CONNECTIONS,
|
|
1184
|
+
files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = DEFAULT_RETURN_UNFINISHED,
|
|
1185
|
+
scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
|
|
1186
|
+
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=DEFAULT_NO_START,
|
|
1187
|
+
skip_hosts = DEFAULT_SKIP_HOSTS, curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,
|
|
1188
|
+
single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY, shortend = False):
|
|
1189
|
+
hosts = hosts if type(hosts) == str else frozenset(hosts)
|
|
1190
|
+
hostStr = formHostStr(hosts)
|
|
1191
|
+
files = frozenset(files) if files else None
|
|
1192
|
+
argsStr = __formCommandArgStr(oneonone = oneonone, timeout = timeout,password = password,
|
|
1193
|
+
quiet = quiet,json = json,max_connections=max_connections,
|
|
1194
|
+
files = files,ipmi = ipmi,interface_ip_prefix = interface_ip_prefix,scp=scp,
|
|
1195
|
+
username=username,extraargs=extraargs,skipUnreachable=skipUnreachable,no_env=no_env,
|
|
1196
|
+
greppable=greppable,skip_hosts = skip_hosts, file_sync = file_sync,error_only = error_only, shortend = shortend)
|
|
1197
|
+
commandStr = '"' + '" "'.join(commands) + '"' if commands else ''
|
|
1198
|
+
return f'multissh {argsStr} {hostStr} {commandStr}'
|
|
1199
|
+
|
|
1200
|
+
def run_command_on_hosts(hosts,commands,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
1201
|
+
quiet = DEFAULT_QUIET,json = DEFAULT_JSON_MODE,called = DEFAULT_CALLED,max_connections=DEFAULT_MAX_CONNECTIONS,
|
|
1202
|
+
files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = DEFAULT_RETURN_UNFINISHED,
|
|
1203
|
+
scp=DEFAULT_SCP,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
|
|
1204
|
+
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=DEFAULT_NO_START,
|
|
1205
|
+
skip_hosts = DEFAULT_SKIP_HOSTS, curses_min_char_len = DEFAULT_CURSES_MINIMUM_CHAR_LEN, curses_min_line_len = DEFAULT_CURSES_MINIMUM_LINE_LEN,
|
|
1206
|
+
single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY):
|
|
1207
|
+
f'''
|
|
1208
|
+
Run the command on the hosts, aka multissh. main function
|
|
1209
|
+
|
|
1210
|
+
Args:
|
|
1211
|
+
hosts (str/iterable): A string of hosts seperated by space or comma / iterable of hosts.
|
|
1212
|
+
commands (list): A list of commands to run on the hosts. When using files, defines the destination of the files.
|
|
1213
|
+
oneonone (bool, optional): Whether to run the commands one on one. Defaults to {DEFAULT_ONE_ON_ONE}.
|
|
1214
|
+
timeout (int, optional): The timeout for the command. Defaults to {DEFAULT_TIMEOUT}.
|
|
1215
|
+
password (str, optional): The password for the hosts. Defaults to {DEFAULT_PASSWORD}.
|
|
1216
|
+
quiet (bool, optional): Whether to print the output. Defaults to {DEFAULT_QUIET}.
|
|
1217
|
+
json (bool, optional): Whether to print the output in JSON format. Defaults to {DEFAULT_JSON_MODE}.
|
|
1218
|
+
called (bool, optional): Whether the function is called by another function. Defaults to {DEFAULT_CALLED}.
|
|
1219
|
+
max_connections (int, optional): The maximum number of concurrent SSH sessions. Defaults to 4 * os.cpu_count().
|
|
1220
|
+
files (list, optional): A list of files to be copied to the hosts. Defaults to None.
|
|
1221
|
+
ipmi (bool, optional): Whether to use IPMI to connect to the hosts. Defaults to {DEFAULT_IPMI}.
|
|
1222
|
+
interface_ip_prefix (str, optional): The prefix of the IPMI interface. Defaults to {DEFAULT_INTERFACE_IP_PREFIX}.
|
|
1223
|
+
returnUnfinished (bool, optional): Whether to return the unfinished hosts. Defaults to {DEFAULT_RETURN_UNFINISHED}.
|
|
1224
|
+
scp (bool, optional): Whether to use scp instead of rsync. Defaults to {DEFAULT_SCP}.
|
|
1225
|
+
username (str, optional): The username to use to connect to the hosts. Defaults to {DEFAULT_USERNAME}.
|
|
1226
|
+
extraargs (str, optional): Extra arguments to pass to the ssh / rsync / scp command. Defaults to {DEFAULT_EXTRA_ARGS}.
|
|
1227
|
+
skipUnreachable (bool, optional): Whether to skip unreachable hosts. Defaults to {DEFAULT_SKIP_UNREACHABLE}.
|
|
1228
|
+
no_env (bool, optional): Whether to not read the current sat system environment variables. (Will still read from files) Defaults to {DEFAULT_NO_ENV}.
|
|
1229
|
+
greppable (bool, optional): Whether to print the output in greppable format. Defaults to {DEFAULT_GREPPABLE_MODE}.
|
|
1230
|
+
willUpdateUnreachableHosts (bool, optional): Whether to update the global unavailable hosts. Defaults to {DEFAULT_UPDATE_UNREACHABLE_HOSTS}.
|
|
1231
|
+
no_start (bool, optional): Whether to return the hosts without starting the command. Defaults to {DEFAULT_NO_START}.
|
|
1232
|
+
skip_hosts (str, optional): The hosts to skip. Defaults to {DEFAULT_SKIP_HOSTS}.
|
|
1233
|
+
min_char_len (int, optional): The minimum character per line of the curses output. Defaults to {DEFAULT_CURSES_MINIMUM_CHAR_LEN}.
|
|
1234
|
+
min_line_len (int, optional): The minimum line number for each window of the curses output. Defaults to {DEFAULT_CURSES_MINIMUM_LINE_LEN}.
|
|
1235
|
+
single_window (bool, optional): Whether to use a single window for the curses output. Defaults to {DEFAULT_SINGLE_WINDOW}.
|
|
1236
|
+
file_sync (bool, optional): Whether to use file sync mode to sync directories. Defaults to {DEFAULT_FILE_SYNC}.
|
|
1237
|
+
|
|
1238
|
+
Returns:
|
|
1239
|
+
list: A list of Host objects
|
|
1240
|
+
'''
|
|
1241
|
+
global gloablUnavailableHosts
|
|
1242
|
+
global global_suppress_printout
|
|
1243
|
+
if not max_connections:
|
|
1244
|
+
max_connections = 4 * os.cpu_count()
|
|
1245
|
+
elif max_connections == 0:
|
|
1246
|
+
max_connections = 1048576
|
|
1247
|
+
elif max_connections < 0:
|
|
1248
|
+
max_connections = (-max_connections) * os.cpu_count()
|
|
1249
|
+
if not commands:
|
|
1250
|
+
commands = []
|
|
1251
|
+
verify_ssh_config()
|
|
1252
|
+
# load global unavailable hosts only if the function is called (so using --repeat will not load the unavailable hosts again)
|
|
1253
|
+
if called:
|
|
1254
|
+
# if called,
|
|
1255
|
+
# if skipUnreachable is not set, we default to skip unreachable hosts within one command call
|
|
1256
|
+
global_suppress_printout = True
|
|
1257
|
+
if skipUnreachable is None:
|
|
1258
|
+
skipUnreachable = True
|
|
1259
|
+
if skipUnreachable:
|
|
1260
|
+
unavailableHosts = gloablUnavailableHosts
|
|
1261
|
+
else:
|
|
1262
|
+
unavailableHosts = set()
|
|
1263
|
+
else:
|
|
1264
|
+
# if run in command line ( or emulating running in command line, we default to skip unreachable hosts within one command call )
|
|
1265
|
+
if skipUnreachable:
|
|
1266
|
+
unavailableHosts = gloablUnavailableHosts
|
|
1267
|
+
else:
|
|
1268
|
+
unavailableHosts = set()
|
|
1269
|
+
skipUnreachable = True
|
|
1270
|
+
global emo
|
|
1271
|
+
emo = False
|
|
1272
|
+
# We create the hosts
|
|
1273
|
+
hostStr = formHostStr(hosts)
|
|
1274
|
+
skipHostStr = formHostStr(skip_hosts) if skip_hosts else ''
|
|
1275
|
+
|
|
1276
|
+
if username:
|
|
1277
|
+
userStr = f'{username.strip()}@'
|
|
1278
|
+
# we also append this userStr to all hostStr which does not have username already defined
|
|
1279
|
+
hostStr = hostStr.split(',')
|
|
1280
|
+
for i, host in enumerate(hostStr):
|
|
1281
|
+
if '@' not in host:
|
|
1282
|
+
hostStr[i] = userStr + host
|
|
1283
|
+
hostStr = ','.join(hostStr)
|
|
1284
|
+
if skipHostStr:
|
|
1285
|
+
skipHostStr = skipHostStr.split(',')
|
|
1286
|
+
for i, host in enumerate(skipHostStr):
|
|
1287
|
+
if '@' not in host:
|
|
1288
|
+
skipHostStr[i] = userStr + host
|
|
1289
|
+
skipHostStr = ','.join(skipHostStr)
|
|
1290
|
+
targetHostsList = expand_hostnames(frozenset(hostStr.split(',')),no_env=no_env)
|
|
1291
|
+
skipHostsList = expand_hostnames(frozenset(skipHostStr.split(',')),no_env=no_env)
|
|
1292
|
+
if skipHostsList:
|
|
1293
|
+
if not global_suppress_printout: print(f"Skipping hosts: {skipHostsList}")
|
|
1294
|
+
if files and not commands:
|
|
1295
|
+
# if files are specified but not target dir, we default to file sync mode
|
|
1296
|
+
file_sync = True
|
|
1297
|
+
if file_sync:
|
|
1298
|
+
# set the files to the union of files and commands
|
|
1299
|
+
files = set(files+commands) if files else set(commands)
|
|
1300
|
+
if files:
|
|
1301
|
+
# try to resolve files first (like * etc)
|
|
1302
|
+
pathSet = set()
|
|
1303
|
+
for file in files:
|
|
1304
|
+
try:
|
|
1305
|
+
pathSet.update(glob.glob(file,include_hidden=True,recursive=True))
|
|
1306
|
+
except:
|
|
1307
|
+
pathSet.update(glob.glob(file,recursive=True))
|
|
1308
|
+
if not pathSet:
|
|
1309
|
+
print(f'Warning: No source files at {files} are found after resolving globs!')
|
|
1310
|
+
sys.exit(66)
|
|
1311
|
+
if file_sync:
|
|
1312
|
+
# use abosolute path for file sync
|
|
1313
|
+
commands = [os.path.abspath(file) for file in pathSet]
|
|
1314
|
+
files = []
|
|
1315
|
+
else:
|
|
1316
|
+
files = list(pathSet)
|
|
1317
|
+
if oneonone:
|
|
1318
|
+
hosts = []
|
|
1319
|
+
if len(commands) != len(targetHostsList) - len(skipHostsList):
|
|
1320
|
+
print("Error: the number of commands must be the same as the number of hosts")
|
|
1321
|
+
print(f"Number of commands: {len(commands)}")
|
|
1322
|
+
print(f"Number of hosts: {len(targetHostsList - skipHostsList)}")
|
|
1323
|
+
sys.exit(255)
|
|
1324
|
+
if not global_suppress_printout:
|
|
1325
|
+
print('-'*80)
|
|
1326
|
+
print("Running in one on one mode")
|
|
1327
|
+
for host, command in zip(targetHostsList, commands):
|
|
1328
|
+
if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
|
|
1329
|
+
if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
|
|
1330
|
+
continue
|
|
1331
|
+
if host.strip() in skipHostsList: continue
|
|
1332
|
+
if file_sync:
|
|
1333
|
+
hosts.append(Host(host.strip(), os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
|
|
1334
|
+
else:
|
|
1335
|
+
hosts.append(Host(host.strip(), command, files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
|
|
1336
|
+
if not global_suppress_printout:
|
|
1337
|
+
print(f"Running command: {command} on host: {host}")
|
|
1338
|
+
if not global_suppress_printout: print('-'*80)
|
|
1339
|
+
if not no_start: processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
|
|
1340
|
+
return hosts
|
|
1341
|
+
else:
|
|
1342
|
+
allHosts = []
|
|
1343
|
+
if not commands:
|
|
1344
|
+
# run in interactive mode ssh mode
|
|
1345
|
+
hosts = []
|
|
1346
|
+
for host in targetHostsList:
|
|
1347
|
+
if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
|
|
1348
|
+
if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
|
|
1349
|
+
continue
|
|
1350
|
+
if host.strip() in skipHostsList: continue
|
|
1351
|
+
if file_sync:
|
|
1352
|
+
print(f"Error: file sync mode need to be specified with at least one path to sync.")
|
|
1353
|
+
return []
|
|
1354
|
+
elif files:
|
|
1355
|
+
print(f"Error: files need to be specified with at least one path to sync")
|
|
1356
|
+
elif ipmi:
|
|
1357
|
+
print(f"Error: ipmi mode is not supported in interactive mode")
|
|
1358
|
+
else:
|
|
1359
|
+
hosts.append(Host(host.strip(), '', files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
|
|
1360
|
+
if not global_suppress_printout:
|
|
1361
|
+
print('-'*80)
|
|
1362
|
+
print(f"Running in interactive mode on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
|
|
1363
|
+
print('-'*80)
|
|
1364
|
+
if no_start:
|
|
1365
|
+
print(f"Warning: no_start is set, the command will not be started. As we are in interactive mode, no action will be done.")
|
|
1366
|
+
else:
|
|
1367
|
+
processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
|
|
1368
|
+
return hosts
|
|
1369
|
+
for command in commands:
|
|
1370
|
+
hosts = []
|
|
1371
|
+
for host in targetHostsList:
|
|
1372
|
+
if not ipmi and skipUnreachable and host.strip() in unavailableHosts:
|
|
1373
|
+
if not global_suppress_printout: print(f"Skipping unavailable host: {host}")
|
|
1374
|
+
continue
|
|
1375
|
+
if host.strip() in skipHostsList: continue
|
|
1376
|
+
if file_sync:
|
|
1377
|
+
hosts.append(Host(host.strip(), os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
|
|
1378
|
+
else:
|
|
1379
|
+
hosts.append(Host(host.strip(), command, files = files,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs))
|
|
1380
|
+
if not global_suppress_printout and len(commands) > 1:
|
|
1381
|
+
print('-'*80)
|
|
1382
|
+
print(f"Running command: {command} on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
|
|
1383
|
+
print('-'*80)
|
|
1384
|
+
if not no_start: processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, quiet, json, called, greppable,unavailableHosts,willUpdateUnreachableHosts,curses_min_char_len = curses_min_char_len, curses_min_line_len = curses_min_line_len,single_window=single_window)
|
|
1385
|
+
allHosts += hosts
|
|
1386
|
+
return allHosts
|
|
1387
|
+
|
|
1388
|
+
def main():
|
|
1389
|
+
# We handle the signal
|
|
1390
|
+
signal.signal(signal.SIGINT, signal_handler)
|
|
1391
|
+
# We parse the arguments
|
|
1392
|
+
parser = argparse.ArgumentParser(description='Run a command on multiple hosts, Use #HOST# or #HOSTNAME# to replace the host name in the command')
|
|
1393
|
+
parser.add_argument('hosts', metavar='hosts', type=str, help='Hosts to run the command on, use "," to seperate hosts')
|
|
1394
|
+
parser.add_argument('commands', metavar='commands', type=str, nargs='+',help='the command to run on the hosts / the destination of the files #HOST# or #HOSTNAME# will be replaced with the host name.')
|
|
1395
|
+
parser.add_argument('-u','--username', type=str,help=f'The general username to use to connect to the hosts. Will get overwrote by individual username@host if specified. (default: {DEFAULT_USERNAME})',default=DEFAULT_USERNAME)
|
|
1396
|
+
parser.add_argument('-ea','--extraargs',type=str,help=f'Extra arguments to pass to the ssh / rsync / scp command. Put in one string for multiple arguments.Use "=" ! Ex. -ea="--delete" (default: {DEFAULT_EXTRA_ARGS})',default=DEFAULT_EXTRA_ARGS)
|
|
1397
|
+
parser.add_argument('-p', '--password', type=str,help=f'The password to use to connect to the hosts, (default: {DEFAULT_PASSWORD})',default=DEFAULT_PASSWORD)
|
|
1398
|
+
parser.add_argument("-11",'--oneonone', action='store_true', help=f"Run one corresponding command on each host. (default: {DEFAULT_ONE_ON_ONE})", default=DEFAULT_ONE_ON_ONE)
|
|
1399
|
+
parser.add_argument("-f","--file", action='append', help="The file to be copied to the hosts. Use -f multiple times to copy multiple files")
|
|
1400
|
+
parser.add_argument('--file_sync', action='store_true', help=f'Operate in file sync mode, sync path in <COMMANDS> from this machine to <HOSTS>. Treat --file <FILE> and <COMMANDS> both as source as source and destination will be the same in this mode. (default: {DEFAULT_FILE_SYNC})', default=DEFAULT_FILE_SYNC)
|
|
1401
|
+
parser.add_argument('--scp', action='store_true', help=f'Use scp for copying files instead of rsync. Need to use this on windows. (default: {DEFAULT_SCP})', default=DEFAULT_SCP)
|
|
1402
|
+
#parser.add_argument("-d",'-c',"--destination", type=str, help="The destination of the files. Same as specify with commands. Added for compatibility. Use #HOST# or #HOSTNAME# to replace the host name in the destination")
|
|
1403
|
+
parser.add_argument("-t","--timeout", type=int, help=f"Timeout for each command in seconds (default: 0 (disabled))", default=0)
|
|
1404
|
+
parser.add_argument("-r","--repeat", type=int, help=f"Repeat the command for a number of times (default: {DEFAULT_REPEAT})", default=DEFAULT_REPEAT)
|
|
1405
|
+
parser.add_argument("-i","--interval", type=int, help=f"Interval between repeats in seconds (default: {DEFAULT_INTERVAL})", default=DEFAULT_INTERVAL)
|
|
1406
|
+
parser.add_argument("--ipmi", action='store_true', help=f"Use ipmitool to run the command. (default: {DEFAULT_IPMI})", default=DEFAULT_IPMI)
|
|
1407
|
+
parser.add_argument("-mpre","--ipmi_interface_ip_prefix", type=str, help=f"The prefix of the IPMI interfaces (default: {DEFAULT_IPMI_INTERFACE_IP_PREFIX})", default=DEFAULT_IPMI_INTERFACE_IP_PREFIX)
|
|
1408
|
+
parser.add_argument("-pre","--interface_ip_prefix", type=str, help=f"The prefix of the for the interfaces (default: {DEFAULT_INTERFACE_IP_PREFIX})", default=DEFAULT_INTERFACE_IP_PREFIX)
|
|
1409
|
+
parser.add_argument("-q","--quiet", action='store_true', help=f"Quiet mode, no curses, only print the output. (default: {DEFAULT_QUIET})", default=DEFAULT_QUIET)
|
|
1410
|
+
parser.add_argument("-ww",'--window_width', type=int, help=f"The minimum character length of the curses window. (default: {DEFAULT_CURSES_MINIMUM_CHAR_LEN})", default=DEFAULT_CURSES_MINIMUM_CHAR_LEN)
|
|
1411
|
+
parser.add_argument("-wh",'--window_height', type=int, help=f"The minimum line height of the curses window. (default: {DEFAULT_CURSES_MINIMUM_LINE_LEN})", default=DEFAULT_CURSES_MINIMUM_LINE_LEN)
|
|
1412
|
+
parser.add_argument('-sw','--single_window', action='store_true', help=f'Use a single window for all hosts. (default: {DEFAULT_SINGLE_WINDOW})', default=DEFAULT_SINGLE_WINDOW)
|
|
1413
|
+
parser.add_argument('-eo','--error_only', action='store_true', help=f'Only print the error output. (default: {DEFAULT_ERROR_ONLY})', default=DEFAULT_ERROR_ONLY)
|
|
1414
|
+
parser.add_argument("-no","--nooutput", action='store_true', help=f"Do not print the output. (default: {DEFAULT_NO_OUTPUT})", default=DEFAULT_NO_OUTPUT)
|
|
1415
|
+
parser.add_argument('--no_env', action='store_true', help=f'Do not load the environment variables. (default: {DEFAULT_NO_ENV})', default=DEFAULT_NO_ENV)
|
|
1416
|
+
parser.add_argument("--env_file", type=str, help=f"The file to load the environment variables from. (default: {DEFAULT_ENV_FILE})", default=DEFAULT_ENV_FILE)
|
|
1417
|
+
parser.add_argument("-m","--maxconnections", type=int, help=f"Max number of connections to use (default: 4 * cpu_count)", default=DEFAULT_MAX_CONNECTIONS)
|
|
1418
|
+
parser.add_argument("-j","--json", action='store_true', help=F"Output in json format. (default: {DEFAULT_JSON_MODE})", default=DEFAULT_JSON_MODE)
|
|
1419
|
+
parser.add_argument("--success_hosts", action='store_true', help=f"Output the hosts that succeeded in summary as wells. (default: {DEFAULT_PRINT_SUCCESS_HOSTS})", default=DEFAULT_PRINT_SUCCESS_HOSTS)
|
|
1420
|
+
parser.add_argument("-g","--greppable", action='store_true', help=f"Output in greppable format. (default: {DEFAULT_GREPPABLE_MODE})", default=DEFAULT_GREPPABLE_MODE)
|
|
1421
|
+
parser.add_argument("-nw","--nowatch", action='store_true', help=f"Do not watch the output in curses modem, Use \\r. Not implemented yet. (default: {DEFAULT_NO_WATCH})", default=DEFAULT_NO_WATCH)
|
|
1422
|
+
parser.add_argument("-su","--skipunreachable", action='store_true', help=f"Skip unreachable hosts while using --repeat. Note: Timedout Hosts are considered unreachable. Note: multiple command sequence will still auto skip unreachable hosts. (default: {DEFAULT_SKIP_UNREACHABLE})", default=DEFAULT_SKIP_UNREACHABLE)
|
|
1423
|
+
parser.add_argument("-sh","--skiphosts", type=str, help=f"Skip the hosts in the list. (default: {DEFAULT_SKIP_HOSTS})", default=DEFAULT_SKIP_HOSTS)
|
|
1424
|
+
parser.add_argument("-V","--version", action='version', version=f'%(prog)s {version} {("with sshpass " if sshpassAvailable else "")}by pan@zopyr.us')
|
|
1425
|
+
|
|
1426
|
+
# parser.add_argument('-u', '--user', metavar='user', type=str, nargs=1,
|
|
1427
|
+
# help='the user to use to connect to the hosts')
|
|
1428
|
+
args = parser.parse_args()
|
|
1429
|
+
|
|
1430
|
+
env_file = args.env_file
|
|
1431
|
+
# if there are more than 1 commands, and every command only consists of one word,
|
|
1432
|
+
# we will ask the user to confirm if they want to run multiple commands or just one command.
|
|
1433
|
+
if not args.file and len(args.commands) > 1 and all([len(command.split()) == 1 for command in args.commands]):
|
|
1434
|
+
print(f"Multiple one word command detected, what to do? (s/f/n)")
|
|
1435
|
+
print(f"1: Run 1 command [{' '.join(args.commands)}] on all hosts ( default )")
|
|
1436
|
+
print(f"m: Run multiple commands [{', '.join(args.commands)}] on all hosts")
|
|
1437
|
+
print(f"n: Exit")
|
|
1438
|
+
inStr = input_with_timeout_and_countdown(3)
|
|
1439
|
+
if (not inStr) or inStr.lower().strip().startswith('1'):
|
|
1440
|
+
args.commands = [" ".join(args.commands)]
|
|
1441
|
+
print(f"\nRunning 1 command: {args.commands[0]} on all hosts")
|
|
1442
|
+
elif inStr.lower().strip().startswith('m'):
|
|
1443
|
+
print(f"\nRunning multiple commands: {', '.join(args.commands)} on all hosts")
|
|
1444
|
+
else:
|
|
1445
|
+
sys.exit(0)
|
|
1446
|
+
|
|
1447
|
+
ipmiiInterfaceIPPrefix = args.ipmi_interface_ip_prefix
|
|
1448
|
+
|
|
1449
|
+
if not args.greppable and not args.json and not args.nooutput:
|
|
1450
|
+
global_suppress_printout = False
|
|
1451
|
+
|
|
1452
|
+
if not global_suppress_printout:
|
|
1453
|
+
print('> ' + getStrCommand(args.hosts,args.commands,oneonone=args.oneonone,timeout=args.timeout,password=args.password,
|
|
1454
|
+
quiet=args.quiet,json=args.json,called=args.nooutput,max_connections=args.maxconnections,
|
|
1455
|
+
files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,username=args.username,
|
|
1456
|
+
extraargs=args.extraargs,skipUnreachable=args.skipunreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skiphosts,
|
|
1457
|
+
curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only))
|
|
1458
|
+
if args.error_only:
|
|
1459
|
+
global_suppress_printout = True
|
|
1460
|
+
|
|
1461
|
+
for i in range(args.repeat):
|
|
1462
|
+
if args.interval > 0 and i < args.repeat - 1:
|
|
1463
|
+
print(f"Sleeping for {args.interval} seconds")
|
|
1464
|
+
time.sleep(args.interval)
|
|
1465
|
+
|
|
1466
|
+
if not global_suppress_printout: print(f"Running the {i+1}/{args.repeat} time") if args.repeat > 1 else None
|
|
1467
|
+
hosts = run_command_on_hosts(args.hosts,args.commands,
|
|
1468
|
+
oneonone=args.oneonone,timeout=args.timeout,password=args.password,
|
|
1469
|
+
quiet=args.quiet,json=args.json,called=args.nooutput,max_connections=args.maxconnections,
|
|
1470
|
+
files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,username=args.username,
|
|
1471
|
+
extraargs=args.extraargs,skipUnreachable=args.skipunreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skiphosts,
|
|
1472
|
+
curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only)
|
|
1473
|
+
#print('*'*80)
|
|
1474
|
+
|
|
1475
|
+
if not global_suppress_printout: print('-'*80)
|
|
1476
|
+
|
|
1477
|
+
succeededHosts = set()
|
|
1478
|
+
for host in hosts:
|
|
1479
|
+
if host.returncode and host.returncode != 0:
|
|
1480
|
+
mainReturnCode += 1
|
|
1481
|
+
failedHosts.add(host.name)
|
|
1482
|
+
else:
|
|
1483
|
+
succeededHosts.add(host.name)
|
|
1484
|
+
succeededHosts -= failedHosts
|
|
1485
|
+
# sort the failed hosts and succeeded hosts
|
|
1486
|
+
failedHosts = sorted(failedHosts)
|
|
1487
|
+
succeededHosts = sorted(succeededHosts)
|
|
1488
|
+
if mainReturnCode > 0:
|
|
1489
|
+
if not global_suppress_printout: print(f'Complete. Failed hosts (Return Code not 0) count: {mainReturnCode}')
|
|
1490
|
+
# with open('/tmp/bashcmd.stdin','w') as f:
|
|
1491
|
+
# f.write(f"export failed_hosts={failedHosts}\n")
|
|
1492
|
+
if not global_suppress_printout: print(f'failed_hosts: {",".join(failedHosts)}')
|
|
1493
|
+
else:
|
|
1494
|
+
if not global_suppress_printout: print('Complete. All hosts returned 0.')
|
|
1495
|
+
|
|
1496
|
+
if args.success_hosts and not global_suppress_printout:
|
|
1497
|
+
print(f'succeeded_hosts: {",".join(succeededHosts)}')
|
|
1498
|
+
|
|
1499
|
+
if threading.active_count() > 1:
|
|
1500
|
+
if not global_suppress_printout: print(f'Remaining active thread: {threading.active_count()}')
|
|
1501
|
+
# os.system(f'pkill -ef {os.path.basename(__file__)}')
|
|
1502
|
+
# os._exit(mainReturnCode)
|
|
1503
|
+
|
|
1504
|
+
sys.exit(mainReturnCode)
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
if __name__ == "__main__":
|
|
1509
|
+
main()
|