multiSSH3 5.80__py3-none-any.whl → 6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of multiSSH3 might be problematic. Click here for more details.
- multiSSH3.py +1375 -477
- multissh3-6.2.dist-info/METADATA +873 -0
- multissh3-6.2.dist-info/RECORD +6 -0
- multissh3-5.80.dist-info/METADATA +0 -379
- multissh3-5.80.dist-info/RECORD +0 -6
- {multissh3-5.80.dist-info → multissh3-6.2.dist-info}/WHEEL +0 -0
- {multissh3-5.80.dist-info → multissh3-6.2.dist-info}/entry_points.txt +0 -0
- {multissh3-5.80.dist-info → multissh3-6.2.dist-info}/top_level.txt +0 -0
multiSSH3.py
CHANGED
|
@@ -6,6 +6,33 @@
|
|
|
6
6
|
# "ipaddress",
|
|
7
7
|
# ]
|
|
8
8
|
# ///
|
|
9
|
+
import argparse
|
|
10
|
+
import functools
|
|
11
|
+
import getpass
|
|
12
|
+
import glob
|
|
13
|
+
import io
|
|
14
|
+
import ipaddress
|
|
15
|
+
import itertools
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
import os
|
|
19
|
+
import queue
|
|
20
|
+
import re
|
|
21
|
+
import shutil
|
|
22
|
+
import signal
|
|
23
|
+
import socket
|
|
24
|
+
import string
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import tempfile
|
|
28
|
+
import textwrap
|
|
29
|
+
import threading
|
|
30
|
+
import time
|
|
31
|
+
import typing
|
|
32
|
+
import uuid
|
|
33
|
+
from collections import Counter, deque, defaultdict
|
|
34
|
+
from itertools import count, product
|
|
35
|
+
|
|
9
36
|
__curses_available = False
|
|
10
37
|
__resource_lib_available = False
|
|
11
38
|
try:
|
|
@@ -20,45 +47,47 @@ try:
|
|
|
20
47
|
except ImportError:
|
|
21
48
|
pass
|
|
22
49
|
|
|
23
|
-
import subprocess
|
|
24
|
-
import threading
|
|
25
|
-
import time
|
|
26
|
-
import os
|
|
27
|
-
import argparse
|
|
28
|
-
from itertools import product
|
|
29
|
-
import re
|
|
30
|
-
import string
|
|
31
|
-
import ipaddress
|
|
32
|
-
import sys
|
|
33
|
-
import json
|
|
34
|
-
import socket
|
|
35
|
-
import io
|
|
36
|
-
import signal
|
|
37
|
-
import functools
|
|
38
|
-
import glob
|
|
39
|
-
import shutil
|
|
40
|
-
import getpass
|
|
41
|
-
import uuid
|
|
42
|
-
import tempfile
|
|
43
|
-
import math
|
|
44
|
-
from itertools import count
|
|
45
|
-
import queue
|
|
46
|
-
|
|
47
50
|
try:
|
|
48
51
|
# Check if functiools.cache is available
|
|
49
|
-
cache_decorator = functools.cache
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
52
|
+
# cache_decorator = functools.cache
|
|
53
|
+
def cache_decorator(user_function):
|
|
54
|
+
def _make_hashable(item):
|
|
55
|
+
if isinstance(item, typing.Mapping):
|
|
56
|
+
# Sort items so that {'a':1, 'b':2} and {'b':2, 'a':1} hash the same
|
|
57
|
+
return tuple(
|
|
58
|
+
( _make_hashable(k), _make_hashable(v) )
|
|
59
|
+
for k, v in sorted(item.items(), key=lambda item: item[0])
|
|
60
|
+
)
|
|
61
|
+
if isinstance(item, (list, set, tuple)):
|
|
62
|
+
return tuple(_make_hashable(e) for e in item)
|
|
63
|
+
# Fallback: assume item is already hashable
|
|
64
|
+
return item
|
|
65
|
+
def decorating_function(user_function):
|
|
66
|
+
# Create the real cached function
|
|
67
|
+
cached_func = functools.lru_cache(maxsize=None)(user_function)
|
|
68
|
+
@functools.wraps(user_function)
|
|
69
|
+
def wrapper(*args, **kwargs):
|
|
70
|
+
# Convert all args/kwargs to hashable equivalents
|
|
71
|
+
hashable_args = tuple(_make_hashable(a) for a in args)
|
|
72
|
+
hashable_kwargs = {
|
|
73
|
+
k: _make_hashable(v) for k, v in kwargs.items()
|
|
74
|
+
}
|
|
75
|
+
# Call the lru-cached version
|
|
76
|
+
return cached_func(*hashable_args, **hashable_kwargs)
|
|
77
|
+
# Expose cache statistics and clear method
|
|
78
|
+
wrapper.cache_info = cached_func.cache_info
|
|
79
|
+
wrapper.cache_clear = cached_func.cache_clear
|
|
80
|
+
return wrapper
|
|
81
|
+
return decorating_function(user_function)
|
|
82
|
+
except Exception:
|
|
83
|
+
# If lrucache is not available, use a dummy decorator
|
|
84
|
+
print('Warning: functools.lru_cache is not available, multiSSH3 will run slower without cache.',file=sys.stderr)
|
|
85
|
+
def cache_decorator(func):
|
|
86
|
+
return func
|
|
87
|
+
version = '6.02'
|
|
59
88
|
VERSION = version
|
|
60
89
|
__version__ = version
|
|
61
|
-
COMMIT_DATE = '2025-
|
|
90
|
+
COMMIT_DATE = '2025-11-10'
|
|
62
91
|
|
|
63
92
|
CONFIG_FILE_CHAIN = ['./multiSSH3.config.json',
|
|
64
93
|
'~/multiSSH3.config.json',
|
|
@@ -67,16 +96,22 @@ CONFIG_FILE_CHAIN = ['./multiSSH3.config.json',
|
|
|
67
96
|
'/etc/multiSSH3.d/multiSSH3.config.json',
|
|
68
97
|
'/etc/multiSSH3.config.json'] # The first one has the highest priority
|
|
69
98
|
|
|
99
|
+
ERRORS = []
|
|
70
100
|
|
|
71
101
|
# TODO: Add terminal TUI
|
|
72
102
|
|
|
73
103
|
#%% ------------ Pre Helper Functions ----------------
|
|
74
104
|
def eprint(*args, **kwargs):
|
|
105
|
+
global ERRORS
|
|
75
106
|
try:
|
|
76
|
-
|
|
107
|
+
if 'file' in kwargs:
|
|
108
|
+
print(*args, **kwargs)
|
|
109
|
+
else:
|
|
110
|
+
print(*args, file=sys.stderr, **kwargs)
|
|
77
111
|
except Exception as e:
|
|
78
112
|
print(f"Error: Cannot print to stderr: {e}")
|
|
79
113
|
print(*args, **kwargs)
|
|
114
|
+
ERRORS.append(' '.join(map(str,args)))
|
|
80
115
|
|
|
81
116
|
def _exit_with_code(code, message=None):
|
|
82
117
|
'''
|
|
@@ -118,33 +153,6 @@ def signal_handler(sig, frame):
|
|
|
118
153
|
os.system(f'pkill -ef {os.path.basename(__file__)}')
|
|
119
154
|
_exit_with_code(1, 'Exiting immediately due to Ctrl C')
|
|
120
155
|
|
|
121
|
-
# def input_with_timeout_and_countdown(timeout, prompt='Please enter your selection'):
|
|
122
|
-
# """
|
|
123
|
-
# Read an input from the user with a timeout and a countdown.
|
|
124
|
-
|
|
125
|
-
# Parameters:
|
|
126
|
-
# timeout (int): The timeout value in seconds.
|
|
127
|
-
# prompt (str): The prompt message to display to the user. Default is 'Please enter your selection'.
|
|
128
|
-
|
|
129
|
-
# Returns:
|
|
130
|
-
# str or None: The user input if received within the timeout, or None if no input is received.
|
|
131
|
-
# """
|
|
132
|
-
# import select
|
|
133
|
-
# # Print the initial prompt with the countdown
|
|
134
|
-
# eprint(f"{prompt} [{timeout}s]: ", end='', flush=True)
|
|
135
|
-
# # Loop until the timeout
|
|
136
|
-
# for remaining in range(timeout, 0, -1):
|
|
137
|
-
# # If there is an input, return it
|
|
138
|
-
# # this only works on linux
|
|
139
|
-
# if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
|
|
140
|
-
# return input().strip()
|
|
141
|
-
# # Print the remaining time
|
|
142
|
-
# eprint(f"\r{prompt} [{remaining}s]: ", end='', flush=True)
|
|
143
|
-
# # Wait a second
|
|
144
|
-
# time.sleep(1)
|
|
145
|
-
# # If there is no input, return None
|
|
146
|
-
# return None
|
|
147
|
-
|
|
148
156
|
def input_with_timeout_and_countdown(timeout, prompt='Please enter your selection'):
|
|
149
157
|
"""
|
|
150
158
|
Read input from the user with a timeout (cross-platform).
|
|
@@ -221,7 +229,7 @@ def getIP(hostname: str,local=False):
|
|
|
221
229
|
# Then we check the DNS
|
|
222
230
|
try:
|
|
223
231
|
return socket.gethostbyname(hostname)
|
|
224
|
-
except:
|
|
232
|
+
except Exception:
|
|
225
233
|
return None
|
|
226
234
|
|
|
227
235
|
|
|
@@ -264,6 +272,7 @@ class Host:
|
|
|
264
272
|
self.output_buffer = io.BytesIO()
|
|
265
273
|
self.stdout_buffer = io.BytesIO()
|
|
266
274
|
self.stderr_buffer = io.BytesIO()
|
|
275
|
+
self.thread = None
|
|
267
276
|
|
|
268
277
|
def __iter__(self):
|
|
269
278
|
return zip(['name', 'command', 'returncode', 'stdout', 'stderr'], [self.name, self.command, self.returncode, self.stdout, self.stderr])
|
|
@@ -276,6 +285,13 @@ extraargs={self.extraargs}, resolvedName={self.resolvedName}, i={self.i}, uuid={
|
|
|
276
285
|
identity_file={self.identity_file}, ip={self.ip}, current_color_pair={self.current_color_pair}"
|
|
277
286
|
def __str__(self):
|
|
278
287
|
return f"Host(name={self.name}, command={self.command}, returncode={self.returncode}, stdout={self.stdout}, stderr={self.stderr})"
|
|
288
|
+
def get_output_hash(self):
|
|
289
|
+
return hash((
|
|
290
|
+
self.command,
|
|
291
|
+
tuple(self.stdout),
|
|
292
|
+
tuple(self.stderr),
|
|
293
|
+
self.returncode
|
|
294
|
+
))
|
|
279
295
|
|
|
280
296
|
#%% ------------ Load Defaults ( Config ) File ----------------
|
|
281
297
|
def load_config_file(config_file):
|
|
@@ -293,8 +309,8 @@ def load_config_file(config_file):
|
|
|
293
309
|
try:
|
|
294
310
|
with open(config_file,'r') as f:
|
|
295
311
|
config = json.load(f)
|
|
296
|
-
except:
|
|
297
|
-
eprint(f"Error: Cannot load config file {config_file!r}")
|
|
312
|
+
except Exception as e:
|
|
313
|
+
eprint(f"Error: Cannot load config file {config_file!r}: {e}")
|
|
298
314
|
return {}
|
|
299
315
|
return config
|
|
300
316
|
|
|
@@ -305,7 +321,7 @@ DEFAULT_HOSTS = 'all'
|
|
|
305
321
|
DEFAULT_USERNAME = None
|
|
306
322
|
DEFAULT_PASSWORD = ''
|
|
307
323
|
DEFAULT_IDENTITY_FILE = None
|
|
308
|
-
|
|
324
|
+
DEFAULT_SSH_KEY_SEARCH_PATH = '~/.ssh/'
|
|
309
325
|
DEFAULT_USE_KEY = False
|
|
310
326
|
DEFAULT_EXTRA_ARGS = None
|
|
311
327
|
DEFAULT_ONE_ON_ONE = False
|
|
@@ -319,15 +335,26 @@ DEFAULT_INTERVAL = 0
|
|
|
319
335
|
DEFAULT_IPMI = False
|
|
320
336
|
DEFAULT_IPMI_INTERFACE_IP_PREFIX = ''
|
|
321
337
|
DEFAULT_INTERFACE_IP_PREFIX = None
|
|
338
|
+
DEFAULT_IPMI_USERNAME = 'ADMIN'
|
|
339
|
+
DEFAULT_IPMI_PASSWORD = ''
|
|
322
340
|
DEFAULT_NO_WATCH = False
|
|
323
|
-
|
|
324
|
-
|
|
341
|
+
DEFAULT_WINDOW_WIDTH = 40
|
|
342
|
+
DEFAULT_WINDOW_HEIGHT = 1
|
|
325
343
|
DEFAULT_SINGLE_WINDOW = False
|
|
326
344
|
DEFAULT_ERROR_ONLY = False
|
|
327
345
|
DEFAULT_NO_OUTPUT = False
|
|
328
346
|
DEFAULT_RETURN_ZERO = False
|
|
329
347
|
DEFAULT_NO_ENV = False
|
|
330
|
-
DEFAULT_ENV_FILE = '
|
|
348
|
+
DEFAULT_ENV_FILE = ''
|
|
349
|
+
DEFAULT_ENV_FILES = ['/etc/profile.d/hosts.sh',
|
|
350
|
+
'~/.bashrc',
|
|
351
|
+
'~/.zshrc',
|
|
352
|
+
'~/host.env',
|
|
353
|
+
'~/hosts.env',
|
|
354
|
+
'.env',
|
|
355
|
+
'host.env',
|
|
356
|
+
'hosts.env',
|
|
357
|
+
]
|
|
331
358
|
DEFAULT_NO_HISTORY = False
|
|
332
359
|
DEFAULT_HISTORY_FILE = '~/.mssh_history'
|
|
333
360
|
DEFAULT_MAX_CONNECTIONS = 4 * os.cpu_count()
|
|
@@ -337,7 +364,9 @@ DEFAULT_GREPPABLE_MODE = False
|
|
|
337
364
|
DEFAULT_SKIP_UNREACHABLE = True
|
|
338
365
|
DEFAULT_SKIP_HOSTS = ''
|
|
339
366
|
DEFAULT_ENCODING = 'utf-8'
|
|
367
|
+
DEFAULT_DIFF_DISPLAY_THRESHOLD = 0.6
|
|
340
368
|
SSH_STRICT_HOST_KEY_CHECKING = False
|
|
369
|
+
FORCE_TRUECOLOR = False
|
|
341
370
|
ERROR_MESSAGES_TO_IGNORE = [
|
|
342
371
|
'Pseudo-terminal will not be allocated because stdin is not a terminal',
|
|
343
372
|
'Connection to .* closed',
|
|
@@ -347,6 +376,24 @@ ERROR_MESSAGES_TO_IGNORE = [
|
|
|
347
376
|
'Killed by signal',
|
|
348
377
|
'Connection reset by peer',
|
|
349
378
|
]
|
|
379
|
+
__DEFAULT_COLOR_PALETTE = {
|
|
380
|
+
'cyan': (86, 173, 188),
|
|
381
|
+
'green': (114, 180, 43),
|
|
382
|
+
'magenta': (140, 107, 200),
|
|
383
|
+
'red': (196, 38, 94),
|
|
384
|
+
'white': (227, 227, 221),
|
|
385
|
+
'yellow': (179, 180, 43),
|
|
386
|
+
'blue': (106, 126, 200),
|
|
387
|
+
'bright_black': (102, 102, 102),
|
|
388
|
+
'bright_blue': (129, 154, 255),
|
|
389
|
+
'bright_cyan': (102, 217, 239),
|
|
390
|
+
'bright_green': (126, 226, 46),
|
|
391
|
+
'bright_magenta': (174, 129, 255),
|
|
392
|
+
'bright_red': (249, 38, 114),
|
|
393
|
+
'bright_white': (248, 248, 242),
|
|
394
|
+
'bright_yellow': (226, 226, 46),
|
|
395
|
+
}
|
|
396
|
+
COLOR_PALETTE = __DEFAULT_COLOR_PALETTE.copy()
|
|
350
397
|
_DEFAULT_CALLED = True
|
|
351
398
|
_DEFAULT_RETURN_UNFINISHED = False
|
|
352
399
|
_DEFAULT_UPDATE_UNREACHABLE_HOSTS = True
|
|
@@ -366,6 +413,9 @@ if __ERROR_MESSAGES_TO_IGNORE_REGEX:
|
|
|
366
413
|
__ERROR_MESSAGES_TO_IGNORE_REGEX = re.compile(__ERROR_MESSAGES_TO_IGNORE_REGEX)
|
|
367
414
|
else:
|
|
368
415
|
__ERROR_MESSAGES_TO_IGNORE_REGEX = re.compile('|'.join(ERROR_MESSAGES_TO_IGNORE))
|
|
416
|
+
if DEFAULT_ENV_FILE:
|
|
417
|
+
if DEFAULT_ENV_FILE not in DEFAULT_ENV_FILES:
|
|
418
|
+
DEFAULT_ENV_FILES.append(DEFAULT_ENV_FILE)
|
|
369
419
|
|
|
370
420
|
#%% Load mssh Functional Global Variables
|
|
371
421
|
__global_suppress_printout = False
|
|
@@ -373,7 +423,7 @@ __mainReturnCode = 0
|
|
|
373
423
|
__failedHosts = set()
|
|
374
424
|
__wildCharacters = ['*','?','x']
|
|
375
425
|
_no_env = DEFAULT_NO_ENV
|
|
376
|
-
|
|
426
|
+
_env_files = DEFAULT_ENV_FILES
|
|
377
427
|
__globalUnavailableHosts = dict()
|
|
378
428
|
__ipmiiInterfaceIPPrefix = DEFAULT_IPMI_INTERFACE_IP_PREFIX
|
|
379
429
|
__keyPressesIn = [[]]
|
|
@@ -386,6 +436,12 @@ __max_connections_nofile_limit_supported = 0
|
|
|
386
436
|
__thread_start_delay = 0
|
|
387
437
|
_encoding = DEFAULT_ENCODING
|
|
388
438
|
__returnZero = DEFAULT_RETURN_ZERO
|
|
439
|
+
__running_threads = set()
|
|
440
|
+
__control_master_string = '''Host *
|
|
441
|
+
ControlMaster auto
|
|
442
|
+
ControlPath /run/user/%i/ssh_sockets_%C
|
|
443
|
+
ControlPersist 3600
|
|
444
|
+
'''
|
|
389
445
|
if __resource_lib_available:
|
|
390
446
|
# Get the current limits
|
|
391
447
|
_, __system_nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
@@ -433,12 +489,12 @@ def check_path(program_name):
|
|
|
433
489
|
|
|
434
490
|
[check_path(program) for program in _binCalled]
|
|
435
491
|
|
|
436
|
-
def find_ssh_key_file(searchPath =
|
|
492
|
+
def find_ssh_key_file(searchPath = DEFAULT_SSH_KEY_SEARCH_PATH):
|
|
437
493
|
'''
|
|
438
494
|
Find the ssh public key file
|
|
439
495
|
|
|
440
496
|
Args:
|
|
441
|
-
searchPath (str, optional): The path to search. Defaults to
|
|
497
|
+
searchPath (str, optional): The path to search. Defaults to DEFAULT_SSH_KEY_SEARCH_PATH.
|
|
442
498
|
|
|
443
499
|
Returns:
|
|
444
500
|
str: The path to the ssh key file
|
|
@@ -454,35 +510,65 @@ def find_ssh_key_file(searchPath = DEDAULT_SSH_KEY_SEARCH_PATH):
|
|
|
454
510
|
return None
|
|
455
511
|
|
|
456
512
|
@cache_decorator
|
|
457
|
-
def readEnvFromFile(
|
|
513
|
+
def readEnvFromFile():
|
|
458
514
|
'''
|
|
459
515
|
Read the environment variables from env_file
|
|
460
516
|
Returns:
|
|
461
517
|
dict: A dictionary of environment variables
|
|
462
518
|
'''
|
|
463
|
-
global
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
envf =
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
if line.startswith('#') or not line
|
|
519
|
+
global _env_files
|
|
520
|
+
global _no_env
|
|
521
|
+
envfs = _env_files if _env_files else DEFAULT_ENV_FILES
|
|
522
|
+
translator = str.maketrans('&|"', ';;\'')
|
|
523
|
+
replacement_re = re.compile(r'\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})')
|
|
524
|
+
environemnt = {}
|
|
525
|
+
scrubCounter = 0
|
|
526
|
+
for envf in envfs:
|
|
527
|
+
envf = os.path.expanduser(os.path.expandvars(envf))
|
|
528
|
+
if os.path.exists(envf):
|
|
529
|
+
with open(envf,'r') as f:
|
|
530
|
+
lines = f.readlines()
|
|
531
|
+
for line in lines:
|
|
532
|
+
line = line.strip()
|
|
533
|
+
if not line or line.startswith('#') or '=' not in line:
|
|
478
534
|
continue
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
535
|
+
line = line.translate(translator)
|
|
536
|
+
commands = re.split(r";(?=(?:[^']*'[^']*')*[^']*$)", line)
|
|
537
|
+
for command in commands:
|
|
538
|
+
if not command or command.startswith('#') or '=' not in command or command.startswith('alias '):
|
|
539
|
+
continue
|
|
540
|
+
fields = re.split(r" (?=(?:[^']*'[^']*')*[^']*$)", command)
|
|
541
|
+
for field in fields:
|
|
542
|
+
try:
|
|
543
|
+
if field.startswith('export '):
|
|
544
|
+
field = field.replace('export ', '', 1).strip()
|
|
545
|
+
if not field or field.startswith('#') or '=' not in field:
|
|
546
|
+
continue
|
|
547
|
+
key, _, values = field.partition('=')
|
|
548
|
+
key = key.strip().strip("'")
|
|
549
|
+
values = values.strip().strip("'")
|
|
550
|
+
if '$' in values:
|
|
551
|
+
scrubCounter += 16
|
|
552
|
+
if key and values and key != values:
|
|
553
|
+
environemnt[key] = values
|
|
554
|
+
except Exception:
|
|
555
|
+
continue
|
|
556
|
+
while scrubCounter:
|
|
557
|
+
scrubCounter -= 1
|
|
558
|
+
found = False
|
|
559
|
+
for key, value in environemnt.items():
|
|
560
|
+
if '$' in value:
|
|
561
|
+
for match in replacement_re.findall(value):
|
|
562
|
+
ref_key = match.strip('${}')
|
|
563
|
+
ref_value = environemnt.get(ref_key) if ref_key != key else None
|
|
564
|
+
if not ref_value and not _no_env:
|
|
565
|
+
ref_value = os.environ.get(ref_key)
|
|
566
|
+
if ref_value:
|
|
567
|
+
environemnt[key] = value.replace(match, ref_value)
|
|
568
|
+
found = True
|
|
569
|
+
if not found:
|
|
570
|
+
break
|
|
571
|
+
return environemnt
|
|
486
572
|
|
|
487
573
|
def replace_magic_strings(string,keys,value,case_sensitive=False):
|
|
488
574
|
'''
|
|
@@ -580,6 +666,428 @@ def pretty_format_table(data, delimiter = '\t',header = None):
|
|
|
580
666
|
outTable.append(row_format.format(*row))
|
|
581
667
|
return '\n'.join(outTable) + '\n'
|
|
582
668
|
|
|
669
|
+
def join_threads(threads=__running_threads,timeout=None):
|
|
670
|
+
'''
|
|
671
|
+
Join threads
|
|
672
|
+
|
|
673
|
+
@params:
|
|
674
|
+
threads: The threads to join
|
|
675
|
+
timeout: The timeout
|
|
676
|
+
|
|
677
|
+
@returns:
|
|
678
|
+
None
|
|
679
|
+
'''
|
|
680
|
+
global __running_threads
|
|
681
|
+
for thread in threads:
|
|
682
|
+
thread.join(timeout=timeout)
|
|
683
|
+
if threads is __running_threads:
|
|
684
|
+
__running_threads = {t for t in threads if t.is_alive()}
|
|
685
|
+
|
|
686
|
+
def format_commands(commands):
|
|
687
|
+
if not commands:
|
|
688
|
+
commands = []
|
|
689
|
+
else:
|
|
690
|
+
commands = [commands] if isinstance(commands,str) else commands
|
|
691
|
+
# reformat commands into a list of strings, join the iterables if they are not strings
|
|
692
|
+
try:
|
|
693
|
+
commands = [' '.join(command) if not isinstance(command,str) else command for command in commands]
|
|
694
|
+
except Exception as e:
|
|
695
|
+
eprint(f"Warning: commands should ideally be a list of strings. Now mssh had failed to convert {commands!r} to a list of strings. Continuing anyway but expect failures. Error: {e}")
|
|
696
|
+
return commands
|
|
697
|
+
|
|
698
|
+
class OrderedMultiSet(deque):
|
|
699
|
+
"""
|
|
700
|
+
A deque extension with O(1) average lookup time.
|
|
701
|
+
Maintains all deque functionality while tracking item counts.
|
|
702
|
+
"""
|
|
703
|
+
def __init__(self, iterable=None, maxlen=None):
|
|
704
|
+
"""Initialize with optional iterable and maxlen."""
|
|
705
|
+
super().__init__(maxlen=maxlen)
|
|
706
|
+
self._counter = Counter()
|
|
707
|
+
if iterable is not None:
|
|
708
|
+
self.extend(iterable)
|
|
709
|
+
def append(self, item):
|
|
710
|
+
"""Add item to the right end. O(1)."""
|
|
711
|
+
if len(self) == self.maxlen:
|
|
712
|
+
self._counter -= Counter([self[0]])
|
|
713
|
+
# self._counter[self[0]] -= 1
|
|
714
|
+
# self._counter += Counter()
|
|
715
|
+
super().append(item)
|
|
716
|
+
self._counter[item] += 1
|
|
717
|
+
def appendleft(self, item):
|
|
718
|
+
"""Add item to the left end. O(1)."""
|
|
719
|
+
if len(self) == self.maxlen:
|
|
720
|
+
self._counter -= Counter([self[-1]])
|
|
721
|
+
super().appendleft(item)
|
|
722
|
+
self._counter[item] += 1
|
|
723
|
+
def pop(self):
|
|
724
|
+
"""Remove and return item from right end. O(1)."""
|
|
725
|
+
try:
|
|
726
|
+
item = super().pop()
|
|
727
|
+
self._counter -= Counter([item])
|
|
728
|
+
return item
|
|
729
|
+
except IndexError:
|
|
730
|
+
return None
|
|
731
|
+
def popleft(self):
|
|
732
|
+
"""Remove and return item from left end. O(1)."""
|
|
733
|
+
try:
|
|
734
|
+
item = super().popleft()
|
|
735
|
+
self._counter -= Counter([item])
|
|
736
|
+
return item
|
|
737
|
+
except IndexError:
|
|
738
|
+
return None
|
|
739
|
+
def put(self, item):
|
|
740
|
+
"""Alias for append, but return removed item - add to right end (FIFO put)."""
|
|
741
|
+
removed = None
|
|
742
|
+
if len(self) == self.maxlen:
|
|
743
|
+
removed = self[0] # Item that will be removed
|
|
744
|
+
self._counter -= Counter([removed])
|
|
745
|
+
super().append(item)
|
|
746
|
+
self._counter[item] += 1
|
|
747
|
+
return removed
|
|
748
|
+
def put_left(self, item):
|
|
749
|
+
"""Alias for appendleft, but return removed item - add to left end (LIFO put)."""
|
|
750
|
+
removed = None
|
|
751
|
+
if len(self) == self.maxlen:
|
|
752
|
+
removed = self[-1] # Item that will be removed
|
|
753
|
+
self._counter -= Counter([removed])
|
|
754
|
+
super().appendleft(item)
|
|
755
|
+
self._counter[item] += 1
|
|
756
|
+
return removed
|
|
757
|
+
def get(self):
|
|
758
|
+
"""Alias for popleft - remove from left end (FIFO get)."""
|
|
759
|
+
return self.popleft()
|
|
760
|
+
def remove(self, value):
|
|
761
|
+
"""Remove first occurrence of value. O(n)."""
|
|
762
|
+
if value not in self._counter:
|
|
763
|
+
return None
|
|
764
|
+
super().remove(value)
|
|
765
|
+
self._counter -= Counter([value])
|
|
766
|
+
def clear(self):
|
|
767
|
+
"""Remove all items. O(1)."""
|
|
768
|
+
super().clear()
|
|
769
|
+
self._counter.clear()
|
|
770
|
+
def extend(self, iterable):
|
|
771
|
+
"""Extend deque by appending elements from iterable. O(k)."""
|
|
772
|
+
# if maxlen is set, and the new length exceeds maxlen, we clear then efficiently extend
|
|
773
|
+
try:
|
|
774
|
+
if not self.maxlen or len(self) + len(iterable) <= self.maxlen:
|
|
775
|
+
super().extend(iterable)
|
|
776
|
+
self._counter.update(iterable)
|
|
777
|
+
elif len(iterable) >= self.maxlen:
|
|
778
|
+
self.clear()
|
|
779
|
+
if isinstance(iterable, (list, tuple)):
|
|
780
|
+
iterable = iterable[-self.maxlen:]
|
|
781
|
+
else:
|
|
782
|
+
iterable = itertools.islice(iterable, len(iterable) - self.maxlen, None)
|
|
783
|
+
super().extend(iterable)
|
|
784
|
+
self._counter.update(iterable)
|
|
785
|
+
else:
|
|
786
|
+
num_to_keep = self.maxlen - len(iterable)
|
|
787
|
+
self.truncateright(num_to_keep)
|
|
788
|
+
super().extend(iterable)
|
|
789
|
+
self._counter.update(iterable)
|
|
790
|
+
except TypeError:
|
|
791
|
+
return self.extend(list(iterable))
|
|
792
|
+
def extendleft(self, iterable):
|
|
793
|
+
"""Extend left side by appending elements from iterable. O(k)."""
|
|
794
|
+
# if maxlen is set, and the new length exceeds maxlen, we clear then efficiently extendleft
|
|
795
|
+
try:
|
|
796
|
+
if not self.maxlen or len(self) + len(iterable) <= self.maxlen:
|
|
797
|
+
super().extendleft(iterable)
|
|
798
|
+
self._counter.update(iterable)
|
|
799
|
+
elif len(iterable) >= self.maxlen:
|
|
800
|
+
self.clear()
|
|
801
|
+
if isinstance(iterable, (list, tuple)):
|
|
802
|
+
iterable = iterable[:self.maxlen]
|
|
803
|
+
else:
|
|
804
|
+
iterable = itertools.islice(iterable, 0, self.maxlen)
|
|
805
|
+
super().extendleft(iterable)
|
|
806
|
+
self._counter.update(iterable)
|
|
807
|
+
else:
|
|
808
|
+
num_to_keep = self.maxlen - len(iterable)
|
|
809
|
+
self.truncate(num_to_keep)
|
|
810
|
+
super().extendleft(iterable)
|
|
811
|
+
self._counter.update(iterable)
|
|
812
|
+
except TypeError:
|
|
813
|
+
return self.extendleft(list(iterable))
|
|
814
|
+
def update(self, iterable):
|
|
815
|
+
"""Extend deque by appending elements from iterable. Alias for extend. O(k)."""
|
|
816
|
+
return self.extend(iterable)
|
|
817
|
+
def updateleft(self, iterable):
|
|
818
|
+
"""Extend left side by appending elements from iterable. Alias for extendleft. O(k)."""
|
|
819
|
+
return self.extendleft(iterable)
|
|
820
|
+
def truncate(self, n):
|
|
821
|
+
"""Truncate to keep left n items. O(n)."""
|
|
822
|
+
kept = list(itertools.islice(self, n))
|
|
823
|
+
dropped = Counter(itertools.islice(self, n, None))
|
|
824
|
+
super().clear()
|
|
825
|
+
super().extend(kept)
|
|
826
|
+
self._counter -= dropped
|
|
827
|
+
def truncateright(self, n):
|
|
828
|
+
"""Truncate to keep right n items. O(n)."""
|
|
829
|
+
kept = list(itertools.islice(self, len(self) - n, None))
|
|
830
|
+
dropped = Counter(itertools.islice(self, 0, len(self) - n))
|
|
831
|
+
super().clear()
|
|
832
|
+
super().extend(kept)
|
|
833
|
+
self._counter -= dropped
|
|
834
|
+
def rotate(self, n=1):
|
|
835
|
+
"""Rotate deque n steps to the right. O(k) where k = min(n, len)."""
|
|
836
|
+
super().rotate(n)
|
|
837
|
+
def __contains__(self, item):
|
|
838
|
+
"""Check if item exists in deque. O(1) average."""
|
|
839
|
+
# return item in self._counter
|
|
840
|
+
return super().__contains__(item)
|
|
841
|
+
def count(self, item):
|
|
842
|
+
"""Return number of occurrences of item. O(1)."""
|
|
843
|
+
return self._counter[item]
|
|
844
|
+
def __setitem__(self, index, value):
|
|
845
|
+
"""Set item at index. O(1) for access, O(1) for counter update."""
|
|
846
|
+
old_value = self[index]
|
|
847
|
+
super().__setitem__(index, value)
|
|
848
|
+
self._counter -= Counter([old_value])
|
|
849
|
+
self._counter[value] += 1
|
|
850
|
+
return old_value
|
|
851
|
+
def __delitem__(self, index):
|
|
852
|
+
"""Delete item at index. O(n) for deletion, O(1) for counter update."""
|
|
853
|
+
value = self[index]
|
|
854
|
+
super().__delitem__(index)
|
|
855
|
+
self._counter -= Counter([value])
|
|
856
|
+
return value
|
|
857
|
+
def insert(self, index, value):
|
|
858
|
+
"""Insert value at index. O(n) for insertion, O(1) for counter update."""
|
|
859
|
+
super().insert(index, value)
|
|
860
|
+
self._counter[value] += 1
|
|
861
|
+
def reverse(self):
|
|
862
|
+
"""Reverse deque in place. O(n)."""
|
|
863
|
+
super().reverse()
|
|
864
|
+
def copy(self):
|
|
865
|
+
"""Create a shallow copy. O(n)."""
|
|
866
|
+
new_deque = OrderedMultiSet(maxlen=self.maxlen)
|
|
867
|
+
new_deque.extend(self)
|
|
868
|
+
return new_deque
|
|
869
|
+
def __copy__(self):
|
|
870
|
+
"""Support for copy.copy()."""
|
|
871
|
+
return self.copy()
|
|
872
|
+
def __repr__(self):
|
|
873
|
+
"""String representation."""
|
|
874
|
+
if self.maxlen is not None:
|
|
875
|
+
return f"OrderedMultiSet({list(self)}, maxlen={self.maxlen})"
|
|
876
|
+
return f"OrderedMultiSet({list(self)})"
|
|
877
|
+
def peek(self):
|
|
878
|
+
"""Return leftmost item without removing it."""
|
|
879
|
+
try:
|
|
880
|
+
return self[0]
|
|
881
|
+
except IndexError:
|
|
882
|
+
return None
|
|
883
|
+
def peek_right(self):
|
|
884
|
+
"""Return rightmost item without removing it."""
|
|
885
|
+
try:
|
|
886
|
+
return self[-1]
|
|
887
|
+
except IndexError:
|
|
888
|
+
return None
|
|
889
|
+
def __iadd__(self, value):
|
|
890
|
+
return self.extend(value)
|
|
891
|
+
def __add__(self, value):
|
|
892
|
+
new_deque = self.copy()
|
|
893
|
+
new_deque.extend(value)
|
|
894
|
+
return new_deque
|
|
895
|
+
def __mul__(self, value):
|
|
896
|
+
new_deque = OrderedMultiSet(maxlen=self.maxlen)
|
|
897
|
+
for _ in range(value):
|
|
898
|
+
new_deque.extend(self)
|
|
899
|
+
return new_deque
|
|
900
|
+
def __imul__(self, value):
|
|
901
|
+
if value <= 0:
|
|
902
|
+
self.clear()
|
|
903
|
+
return self
|
|
904
|
+
for _ in range(value - 1):
|
|
905
|
+
self.extend(self)
|
|
906
|
+
return self
|
|
907
|
+
def __eq__(self, value):
|
|
908
|
+
if isinstance(value, OrderedMultiSet):
|
|
909
|
+
return self._counter == value._counter
|
|
910
|
+
return super().__eq__(value)
|
|
911
|
+
|
|
912
|
+
def get_terminal_size():
|
|
913
|
+
'''
|
|
914
|
+
Get the terminal size
|
|
915
|
+
|
|
916
|
+
@params:
|
|
917
|
+
None
|
|
918
|
+
|
|
919
|
+
@returns:
|
|
920
|
+
(int,int): the number of columns and rows of the terminal
|
|
921
|
+
'''
|
|
922
|
+
try:
|
|
923
|
+
import os
|
|
924
|
+
_tsize = os.get_terminal_size()
|
|
925
|
+
except Exception:
|
|
926
|
+
try:
|
|
927
|
+
import fcntl
|
|
928
|
+
import struct
|
|
929
|
+
import termios
|
|
930
|
+
packed = fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0))
|
|
931
|
+
_tsize = struct.unpack('HHHH', packed)[:2]
|
|
932
|
+
except Exception:
|
|
933
|
+
import shutil
|
|
934
|
+
_tsize = shutil.get_terminal_size(fallback=(120, 30))
|
|
935
|
+
return _tsize
|
|
936
|
+
|
|
937
|
+
@cache_decorator
|
|
938
|
+
def get_terminal_color_capability():
|
|
939
|
+
global FORCE_TRUECOLOR
|
|
940
|
+
if not sys.stdout.isatty():
|
|
941
|
+
return 'None'
|
|
942
|
+
term = os.environ.get("TERM", "")
|
|
943
|
+
if term == "dumb":
|
|
944
|
+
return 'None'
|
|
945
|
+
elif term == "linux":
|
|
946
|
+
return '8'
|
|
947
|
+
elif FORCE_TRUECOLOR:
|
|
948
|
+
return '24bit'
|
|
949
|
+
colorterm = os.environ.get("COLORTERM", "")
|
|
950
|
+
if colorterm in ("truecolor", "24bit", "24-bit"):
|
|
951
|
+
return '24bit'
|
|
952
|
+
if term in ("xterm-truecolor", "xterm-24bit", "xterm-kitty", "alacritty", "wezterm", "foot", "terminology"):
|
|
953
|
+
return '24bit'
|
|
954
|
+
elif "256" in term:
|
|
955
|
+
return '256'
|
|
956
|
+
elif "16" in term:
|
|
957
|
+
return '16'
|
|
958
|
+
try:
|
|
959
|
+
curses.setupterm()
|
|
960
|
+
colors = curses.tigetnum("colors")
|
|
961
|
+
# tigetnum returns -1 if the capability isn’t defined
|
|
962
|
+
if colors >= 16777216:
|
|
963
|
+
return '24bit'
|
|
964
|
+
elif colors >= 256:
|
|
965
|
+
return '256'
|
|
966
|
+
elif colors >= 16:
|
|
967
|
+
return '16'
|
|
968
|
+
elif colors > 0:
|
|
969
|
+
return '8'
|
|
970
|
+
else:
|
|
971
|
+
return 'None'
|
|
972
|
+
except Exception:
|
|
973
|
+
return 'None'
|
|
974
|
+
|
|
975
|
+
@cache_decorator
|
|
976
|
+
def rgb_to_ansi_color_string(r, g, b):
|
|
977
|
+
"""
|
|
978
|
+
Return an ANSI escape sequence setting the foreground to (r,g,b)
|
|
979
|
+
approximated to the terminal's capability, or '' if none.
|
|
980
|
+
"""
|
|
981
|
+
cap = get_terminal_color_capability()
|
|
982
|
+
if cap == 'None':
|
|
983
|
+
return ''
|
|
984
|
+
if cap == '24bit':
|
|
985
|
+
return f'\x1b[38;2;{r};{g};{b}m'
|
|
986
|
+
if cap == '256':
|
|
987
|
+
idx = _rgb_to_256_color(r, g, b)
|
|
988
|
+
return f'\x1b[38;5;{idx}m'
|
|
989
|
+
if cap == '16':
|
|
990
|
+
idx = _rgb_to_16_color(r, g, b)
|
|
991
|
+
# 0–7 = 30–37, 8–15 = 90–97
|
|
992
|
+
if idx < 8:
|
|
993
|
+
return f'\x1b[{30 + idx}m'
|
|
994
|
+
else:
|
|
995
|
+
return f'\x1b[{90 + (idx - 8)}m'
|
|
996
|
+
if cap == '8':
|
|
997
|
+
idx = _rgb_to_8_color(r, g, b)
|
|
998
|
+
return f'\x1b[{30 + idx}m'
|
|
999
|
+
return ''
|
|
1000
|
+
|
|
1001
|
+
def _rgb_to_256_color(r, g, b):
|
|
1002
|
+
"""
|
|
1003
|
+
Map (r,g,b) to the 256-color cube or grayscale ramp.
|
|
1004
|
+
"""
|
|
1005
|
+
# if it’s already gray, use the 232–255 grayscale ramp
|
|
1006
|
+
if r == g == b:
|
|
1007
|
+
# 24 shades from 232 to 255
|
|
1008
|
+
return 232 + int(round(r / 255 * 23))
|
|
1009
|
+
# else map each channel to 0–5
|
|
1010
|
+
def to6(v):
|
|
1011
|
+
return int(round(v / 255 * 5))
|
|
1012
|
+
r6, g6, b6 = to6(r), to6(g), to6(b)
|
|
1013
|
+
return 16 + 36 * r6 + 6 * g6 + b6
|
|
1014
|
+
|
|
1015
|
+
def _rgb_to_16_color(r, g, b):
|
|
1016
|
+
"""
|
|
1017
|
+
Pick the nearest of the 16 ANSI standard colors.
|
|
1018
|
+
Returns an index 0-15.
|
|
1019
|
+
"""
|
|
1020
|
+
palette = [
|
|
1021
|
+
(0, 0, 0), # 0 black
|
|
1022
|
+
(128, 0, 0), # 1 red
|
|
1023
|
+
(0, 128, 0), # 2 green
|
|
1024
|
+
(128, 128, 0), # 3 yellow
|
|
1025
|
+
(0, 0, 128), # 4 blue
|
|
1026
|
+
(128, 0, 128), # 5 magenta
|
|
1027
|
+
(0, 128, 128), # 6 cyan
|
|
1028
|
+
(192, 192, 192), # 7 white (light gray)
|
|
1029
|
+
(128, 128, 128), # 8 bright black (dark gray)
|
|
1030
|
+
(255, 0, 0), # 9 bright red
|
|
1031
|
+
(0, 255, 0), # 10 bright green
|
|
1032
|
+
(255, 255, 0), # 11 bright yellow
|
|
1033
|
+
(0, 0, 255), # 12 bright blue
|
|
1034
|
+
(255, 0, 255), # 13 bright magenta
|
|
1035
|
+
(0, 255, 255), # 14 bright cyan
|
|
1036
|
+
(255, 255, 255), # 15 bright white
|
|
1037
|
+
]
|
|
1038
|
+
best_idx = 0
|
|
1039
|
+
best_dist = float('inf')
|
|
1040
|
+
for i, (pr, pg, pb) in enumerate(palette):
|
|
1041
|
+
dist = (r - pr)**2 + (g - pg)**2 + (b - pb)**2
|
|
1042
|
+
if dist < best_dist:
|
|
1043
|
+
best_dist = dist
|
|
1044
|
+
best_idx = i
|
|
1045
|
+
return best_idx
|
|
1046
|
+
|
|
1047
|
+
def _rgb_to_8_color(r, g, b):
|
|
1048
|
+
"""
|
|
1049
|
+
Reduce to 8 colors by mapping to the 16-color index then clamping 0-7.
|
|
1050
|
+
"""
|
|
1051
|
+
return _rgb_to_16_color(r//2, g//2, b//2)
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def int_to_color(hash_value, min_brightness=100,max_brightness=220):
|
|
1055
|
+
r = (hash_value >> 16) & 0xFF
|
|
1056
|
+
g = (hash_value >> 8) & 0xFF
|
|
1057
|
+
b = hash_value & 0xFF
|
|
1058
|
+
brightness = math.sqrt(0.299 * r**2 + 0.587 * g**2 + 0.114 * b**2)
|
|
1059
|
+
if brightness < min_brightness:
|
|
1060
|
+
return int_to_color(hash(str(hash_value)), min_brightness, max_brightness)
|
|
1061
|
+
if brightness > max_brightness:
|
|
1062
|
+
return int_to_color(hash(str(hash_value)), min_brightness, max_brightness)
|
|
1063
|
+
return (r, g, b)
|
|
1064
|
+
|
|
1065
|
+
__previous_color_rgb = ()
|
|
1066
|
+
@cache_decorator
|
|
1067
|
+
def int_to_unique_ansi_color(number):
|
|
1068
|
+
'''
|
|
1069
|
+
Convert a number to a unique ANSI color code
|
|
1070
|
+
|
|
1071
|
+
Args:
|
|
1072
|
+
number (int): The number to convert
|
|
1073
|
+
Returns:
|
|
1074
|
+
int: The ANSI color code
|
|
1075
|
+
'''
|
|
1076
|
+
global __previous_color_rgb
|
|
1077
|
+
# Use a hash function to generate a consistent integer from the string
|
|
1078
|
+
color_capability = get_terminal_color_capability()
|
|
1079
|
+
if color_capability == 'None':
|
|
1080
|
+
return ''
|
|
1081
|
+
if color_capability == '24bit':
|
|
1082
|
+
r, g, b = int_to_color(number)
|
|
1083
|
+
else:
|
|
1084
|
+
# for 256 colors and below, reduce brightness threshold as we do not have many color to work with
|
|
1085
|
+
r, g, b = int_to_color(number, min_brightness=70, max_brightness=190)
|
|
1086
|
+
if sum(abs(a - b) for a, b in zip((r, g, b), __previous_color_rgb)) <= 256:
|
|
1087
|
+
r, g, b = int_to_color(hash(str(number)))
|
|
1088
|
+
__previous_color_rgb = (r, g, b)
|
|
1089
|
+
return rgb_to_ansi_color_string(r, g, b)
|
|
1090
|
+
|
|
583
1091
|
#%% ------------ Compacting Hostnames ----------------
|
|
584
1092
|
def __tokenize_hostname(hostname):
|
|
585
1093
|
"""
|
|
@@ -924,17 +1432,22 @@ def compact_hostnames(Hostnames,verify = True):
|
|
|
924
1432
|
['sub-s[1-2]']
|
|
925
1433
|
"""
|
|
926
1434
|
global __global_suppress_printout
|
|
927
|
-
if not isinstance(Hostnames, frozenset):
|
|
928
|
-
|
|
929
|
-
else:
|
|
930
|
-
|
|
1435
|
+
# if not isinstance(Hostnames, frozenset):
|
|
1436
|
+
# hostSet = frozenset(Hostnames)
|
|
1437
|
+
# else:
|
|
1438
|
+
# hostSet = Hostnames
|
|
1439
|
+
hostSet = frozenset(expand_hostnames(
|
|
1440
|
+
hostname.strip()
|
|
1441
|
+
for hostnames_str in Hostnames
|
|
1442
|
+
for hostname in hostnames_str.split(',')
|
|
1443
|
+
))
|
|
931
1444
|
compact_hosts = __compact_hostnames(hostSet)
|
|
932
1445
|
if verify:
|
|
933
|
-
if
|
|
1446
|
+
if frozenset(expand_hostnames(compact_hosts)) != hostSet:
|
|
934
1447
|
if not __global_suppress_printout:
|
|
935
1448
|
eprint(f"Error compacting hostnames: {hostSet} -> {compact_hosts}")
|
|
936
1449
|
compact_hosts = hostSet
|
|
937
|
-
return compact_hosts
|
|
1450
|
+
return sorted(compact_hosts)
|
|
938
1451
|
|
|
939
1452
|
#%% ------------ Expanding Hostnames ----------------
|
|
940
1453
|
@cache_decorator
|
|
@@ -1159,51 +1672,53 @@ def __handle_reading_stream(stream,target, host,buffer:io.BytesIO):
|
|
|
1159
1672
|
buffer.truncate(0)
|
|
1160
1673
|
host.output_buffer.seek(0)
|
|
1161
1674
|
host.output_buffer.truncate(0)
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
continue
|
|
1168
|
-
elif char == b'\r':
|
|
1169
|
-
buffer.seek(0)
|
|
1170
|
-
host.output_buffer.seek(0)
|
|
1171
|
-
elif char == b'\x08':
|
|
1172
|
-
# backspace
|
|
1173
|
-
if buffer.tell() > 0:
|
|
1174
|
-
buffer.seek(buffer.tell() - 1)
|
|
1175
|
-
buffer.truncate()
|
|
1176
|
-
if host.output_buffer.tell() > 0:
|
|
1177
|
-
host.output_buffer.seek(host.output_buffer.tell() - 1)
|
|
1178
|
-
host.output_buffer.truncate()
|
|
1179
|
-
else:
|
|
1180
|
-
# normal character
|
|
1181
|
-
buffer.write(char)
|
|
1182
|
-
host.output_buffer.write(char)
|
|
1183
|
-
# if the length of the buffer is greater than 100, we try to decode the buffer to find if there are any unicode line change chars
|
|
1184
|
-
if buffer.tell() % 100 == 0 and buffer.tell() > 0:
|
|
1185
|
-
try:
|
|
1186
|
-
# try to decode the buffer to find if there are any unicode line change chars
|
|
1187
|
-
decodedLine = buffer.getvalue().decode(_encoding,errors='backslashreplace')
|
|
1188
|
-
lines = decodedLine.splitlines()
|
|
1189
|
-
if len(lines) > 1:
|
|
1190
|
-
# if there are multiple lines, we add them to the target
|
|
1191
|
-
for line in lines[:-1]:
|
|
1192
|
-
# for all lines except the last one, we add them to the target
|
|
1193
|
-
target.append(line)
|
|
1194
|
-
host.output.append(line)
|
|
1195
|
-
host.lineNumToPrintSet.add(len(host.output)-1)
|
|
1196
|
-
# we keep the last line in the buffer
|
|
1197
|
-
buffer.seek(0)
|
|
1198
|
-
buffer.truncate(0)
|
|
1199
|
-
buffer.write(lines[-1].encode(_encoding,errors='backslashreplace'))
|
|
1200
|
-
host.output_buffer.seek(0)
|
|
1201
|
-
host.output_buffer.truncate(0)
|
|
1202
|
-
host.output_buffer.write(lines[-1].encode(_encoding,errors='backslashreplace'))
|
|
1203
|
-
|
|
1204
|
-
except UnicodeDecodeError:
|
|
1205
|
-
# if there is a unicode decode error, we just skip this character
|
|
1675
|
+
try:
|
|
1676
|
+
for char in iter(lambda:stream.read(1), b''):
|
|
1677
|
+
host.lastUpdateTime = time.monotonic()
|
|
1678
|
+
if char == b'\n':
|
|
1679
|
+
add_line(buffer,target, host)
|
|
1206
1680
|
continue
|
|
1681
|
+
elif char == b'\r':
|
|
1682
|
+
buffer.seek(0)
|
|
1683
|
+
host.output_buffer.seek(0)
|
|
1684
|
+
elif char == b'\x08':
|
|
1685
|
+
# backspace
|
|
1686
|
+
if buffer.tell() > 0:
|
|
1687
|
+
buffer.seek(buffer.tell() - 1)
|
|
1688
|
+
buffer.truncate()
|
|
1689
|
+
if host.output_buffer.tell() > 0:
|
|
1690
|
+
host.output_buffer.seek(host.output_buffer.tell() - 1)
|
|
1691
|
+
host.output_buffer.truncate()
|
|
1692
|
+
else:
|
|
1693
|
+
# normal character
|
|
1694
|
+
buffer.write(char)
|
|
1695
|
+
host.output_buffer.write(char)
|
|
1696
|
+
# if the length of the buffer is greater than 100, we try to decode the buffer to find if there are any unicode line change chars
|
|
1697
|
+
if buffer.tell() % 100 == 0 and buffer.tell() > 0:
|
|
1698
|
+
try:
|
|
1699
|
+
# try to decode the buffer to find if there are any unicode line change chars
|
|
1700
|
+
decodedLine = buffer.getvalue().decode(_encoding,errors='backslashreplace')
|
|
1701
|
+
lines = decodedLine.splitlines()
|
|
1702
|
+
if len(lines) > 1:
|
|
1703
|
+
# if there are multiple lines, we add them to the target
|
|
1704
|
+
for line in lines[:-1]:
|
|
1705
|
+
# for all lines except the last one, we add them to the target
|
|
1706
|
+
target.append(line)
|
|
1707
|
+
host.output.append(line)
|
|
1708
|
+
host.lineNumToPrintSet.add(len(host.output)-1)
|
|
1709
|
+
# we keep the last line in the buffer
|
|
1710
|
+
buffer.seek(0)
|
|
1711
|
+
buffer.truncate(0)
|
|
1712
|
+
buffer.write(lines[-1].encode(_encoding,errors='backslashreplace'))
|
|
1713
|
+
host.output_buffer.seek(0)
|
|
1714
|
+
host.output_buffer.truncate(0)
|
|
1715
|
+
host.output_buffer.write(lines[-1].encode(_encoding,errors='backslashreplace'))
|
|
1716
|
+
|
|
1717
|
+
except UnicodeDecodeError:
|
|
1718
|
+
# if there is a unicode decode error, we just skip this character
|
|
1719
|
+
continue
|
|
1720
|
+
except ValueError:
|
|
1721
|
+
pass
|
|
1207
1722
|
if buffer.tell() > 0:
|
|
1208
1723
|
# if there is still some data in the buffer, we add it to the target
|
|
1209
1724
|
add_line(buffer,target, host)
|
|
@@ -1247,7 +1762,7 @@ def __handle_writing_stream(stream,stop_event,host):
|
|
|
1247
1762
|
# host.output.append(' $ ' + ''.join(__keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
|
|
1248
1763
|
# host.stdout.append(' $ ' + ''.join(__keyPressesIn[-1]).encode().decode().replace('\n', '↵'))
|
|
1249
1764
|
return sentInputPos
|
|
1250
|
-
|
|
1765
|
+
|
|
1251
1766
|
def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
1252
1767
|
'''
|
|
1253
1768
|
Run the command on the host. Will format the commands accordingly. Main execution function.
|
|
@@ -1266,6 +1781,11 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1266
1781
|
global __ipmiiInterfaceIPPrefix
|
|
1267
1782
|
global _binPaths
|
|
1268
1783
|
global __DEBUG_MODE
|
|
1784
|
+
global DEFAULT_IPMI_USERNAME
|
|
1785
|
+
global DEFAULT_IPMI_PASSWORD
|
|
1786
|
+
global DEFAULT_USERNAME
|
|
1787
|
+
global DEFAULT_PASSWORD
|
|
1788
|
+
global SSH_STRICT_HOST_KEY_CHECKING
|
|
1269
1789
|
if retry_limit < 0:
|
|
1270
1790
|
host.output.append('Error: Retry limit reached!')
|
|
1271
1791
|
host.stderr.append('Error: Retry limit reached!')
|
|
@@ -1293,6 +1813,7 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1293
1813
|
host.command = replace_magic_strings(host.command,['#I#'],str(host.i),case_sensitive=False)
|
|
1294
1814
|
host.command = replace_magic_strings(host.command,['#PASSWD#','#PASSWORD#'],passwds,case_sensitive=False)
|
|
1295
1815
|
host.command = replace_magic_strings(host.command,['#UUID#'],str(host.uuid),case_sensitive=False)
|
|
1816
|
+
host.command = replace_magic_strings(host.command,['#IP#'],str(host.ip),case_sensitive=False)
|
|
1296
1817
|
formatedCMD = []
|
|
1297
1818
|
if host.extraargs and isinstance(host.extraargs, str):
|
|
1298
1819
|
extraargs = host.extraargs.split()
|
|
@@ -1309,7 +1830,7 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1309
1830
|
host.address = '.'.join(prefixOctets[:3]+hostOctets[min(3,len(prefixOctets)):])
|
|
1310
1831
|
host.resolvedName = host.username + '@' if host.username else ''
|
|
1311
1832
|
host.resolvedName += host.address
|
|
1312
|
-
except:
|
|
1833
|
+
except Exception:
|
|
1313
1834
|
host.resolvedName = host.name
|
|
1314
1835
|
else:
|
|
1315
1836
|
host.resolvedName = host.name
|
|
@@ -1321,22 +1842,27 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1321
1842
|
host.command = host.command.replace('ipmitool ','')
|
|
1322
1843
|
elif host.command.startswith(_binPaths['ipmitool']):
|
|
1323
1844
|
host.command = host.command.replace(_binPaths['ipmitool'],'')
|
|
1324
|
-
if not host.username:
|
|
1325
|
-
|
|
1845
|
+
if not host.username or host.username == DEFAULT_USERNAME:
|
|
1846
|
+
if DEFAULT_IPMI_USERNAME:
|
|
1847
|
+
host.username = DEFAULT_IPMI_USERNAME
|
|
1848
|
+
elif DEFAULT_USERNAME:
|
|
1849
|
+
host.username = DEFAULT_USERNAME
|
|
1850
|
+
else:
|
|
1851
|
+
host.username = 'ADMIN'
|
|
1852
|
+
if not passwds or passwds == DEFAULT_PASSWORD:
|
|
1853
|
+
if DEFAULT_IPMI_PASSWORD:
|
|
1854
|
+
passwds = DEFAULT_IPMI_PASSWORD
|
|
1855
|
+
elif DEFAULT_PASSWORD:
|
|
1856
|
+
passwds = DEFAULT_PASSWORD
|
|
1857
|
+
else:
|
|
1858
|
+
host.output.append('Warning: Password not provided for ipmi! Using a default password `admin`.')
|
|
1859
|
+
passwds = 'admin'
|
|
1326
1860
|
if not host.command:
|
|
1327
1861
|
host.command = 'power status'
|
|
1328
1862
|
if 'sh' in _binPaths:
|
|
1329
|
-
|
|
1330
|
-
formatedCMD = [_binPaths['sh'],'-c',f'ipmitool -H {host.address} -U {host.username} -P {passwds} {" ".join(extraargs)} {host.command}']
|
|
1331
|
-
else:
|
|
1332
|
-
host.output.append('Warning: Password not provided for ipmi! Using a default password `admin`.')
|
|
1333
|
-
formatedCMD = [_binPaths['sh'],'-c',f'ipmitool -H {host.address} -U {host.username} -P admin {" ".join(extraargs)} {host.command}']
|
|
1863
|
+
formatedCMD = [_binPaths['sh'],'-c',f'ipmitool -H {host.address} -U {host.username} -P {passwds} {" ".join(extraargs)} {host.command}']
|
|
1334
1864
|
else:
|
|
1335
|
-
|
|
1336
|
-
formatedCMD = [_binPaths['ipmitool'],f'-H {host.address}',f'-U {host.username}',f'-P {passwds}'] + extraargs + [host.command]
|
|
1337
|
-
else:
|
|
1338
|
-
host.output.append('Warning: Password not provided for ipmi! Using a default password `admin`.')
|
|
1339
|
-
formatedCMD = [_binPaths['ipmitool'],f'-H {host.address}',f'-U {host.username}',f'-P admin'] + extraargs + [host.command]
|
|
1865
|
+
formatedCMD = [_binPaths['ipmitool'],f'-H {host.address}',f'-U {host.username}',f'-P {passwds}'] + extraargs + [host.command]
|
|
1340
1866
|
elif 'ssh' in _binPaths:
|
|
1341
1867
|
host.output.append('Ipmitool not found on the local machine! Trying ipmitool on the remote machine...')
|
|
1342
1868
|
if __DEBUG_MODE:
|
|
@@ -1486,8 +2012,6 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1486
2012
|
stdout_thread.join(timeout=1)
|
|
1487
2013
|
stderr_thread.join(timeout=1)
|
|
1488
2014
|
stdin_thread.join(timeout=1)
|
|
1489
|
-
# here we handle the rest of the stdout after the subprocess returns
|
|
1490
|
-
host.output.append(f'Pipe Closed. Trying to read the rest of the stdout...')
|
|
1491
2015
|
if not _emo:
|
|
1492
2016
|
stdout = None
|
|
1493
2017
|
stderr = None
|
|
@@ -1496,8 +2020,10 @@ def run_command(host, sem, timeout=60,passwds=None, retry_limit = 5):
|
|
|
1496
2020
|
except subprocess.TimeoutExpired:
|
|
1497
2021
|
pass
|
|
1498
2022
|
if stdout:
|
|
2023
|
+
host.output.append('Trying to read the rest of the stdout...')
|
|
1499
2024
|
__handle_reading_stream(io.BytesIO(stdout),host.stdout, host,host.stdout_buffer)
|
|
1500
2025
|
if stderr:
|
|
2026
|
+
host.output.append('Trying to read the rest of the stderr...')
|
|
1501
2027
|
__handle_reading_stream(io.BytesIO(stderr),host.stderr, host,host.stderr_buffer)
|
|
1502
2028
|
# if the last line in host.stderr is Connection to * closed., we will remove it
|
|
1503
2029
|
host.returncode = proc.poll()
|
|
@@ -1569,8 +2095,9 @@ def start_run_on_hosts(hosts, timeout=60,password=None,max_connections=4 * os.cp
|
|
|
1569
2095
|
return []
|
|
1570
2096
|
sem = threading.Semaphore(max_connections) # Limit concurrent SSH sessions
|
|
1571
2097
|
threads = [threading.Thread(target=run_command, args=(host, sem,timeout,password), daemon=True) for host in hosts]
|
|
1572
|
-
for thread in threads:
|
|
2098
|
+
for thread, host in zip(threads, hosts):
|
|
1573
2099
|
thread.start()
|
|
2100
|
+
host.thread = thread
|
|
1574
2101
|
time.sleep(__thread_start_delay)
|
|
1575
2102
|
return threads
|
|
1576
2103
|
|
|
@@ -1906,7 +2433,7 @@ def _get_hosts_to_display (hosts, max_num_hosts, hosts_to_display = None, indexO
|
|
|
1906
2433
|
rearrangedHosts.add(host)
|
|
1907
2434
|
return new_hosts_to_display , {'running':len(running_hosts), 'failed':len(failed_hosts), 'finished':len(finished_hosts), 'waiting':len(waiting_hosts)}, rearrangedHosts
|
|
1908
2435
|
|
|
1909
|
-
def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min_char_len =
|
|
2436
|
+
def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min_char_len = DEFAULT_WINDOW_WIDTH, min_line_len = DEFAULT_WINDOW_HEIGHT,single_window=DEFAULT_SINGLE_WINDOW,help_shown = False, config_reason = 'New Configuration'):
|
|
1910
2437
|
global _encoding
|
|
1911
2438
|
_ = config_reason
|
|
1912
2439
|
try:
|
|
@@ -1925,9 +2452,9 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
1925
2452
|
min_line_len_local = max_y-1
|
|
1926
2453
|
# return True if the terminal is too small
|
|
1927
2454
|
if max_x < 2 or max_y < 2:
|
|
1928
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Terminal too small')
|
|
2455
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Terminal too small')
|
|
1929
2456
|
if min_char_len_local < 1 or min_line_len_local < 1:
|
|
1930
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Minimum character or line length too small')
|
|
2457
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Minimum character or line length too small')
|
|
1931
2458
|
# We need to figure out how many hosts we can fit in the terminal
|
|
1932
2459
|
# We will need at least 2 lines per host, one for its name, one for its output
|
|
1933
2460
|
# Each line will be at least 61 characters long (60 for the output, 1 for the borders)
|
|
@@ -1935,10 +2462,10 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
1935
2462
|
max_num_hosts_y = max_y // (min_line_len_local + 1)
|
|
1936
2463
|
max_num_hosts = max_num_hosts_x * max_num_hosts_y
|
|
1937
2464
|
if max_num_hosts < 1:
|
|
1938
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Terminal too small to display any hosts')
|
|
2465
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Terminal too small to display any hosts')
|
|
1939
2466
|
hosts_to_display , host_stats, rearrangedHosts = _get_hosts_to_display(hosts, max_num_hosts)
|
|
1940
2467
|
if len(hosts_to_display) == 0:
|
|
1941
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'No hosts to display')
|
|
2468
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'No hosts to display')
|
|
1942
2469
|
# Now we calculate the actual number of hosts we will display for x and y
|
|
1943
2470
|
optimal_len_x = max(min_char_len_local, 80)
|
|
1944
2471
|
num_hosts_x = min(max(min(max_num_hosts_x, max_x // optimal_len_x),1),len(hosts_to_display))
|
|
@@ -1959,7 +2486,7 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
1959
2486
|
host_window_height = max_y // num_hosts_y
|
|
1960
2487
|
host_window_width = max_x // num_hosts_x
|
|
1961
2488
|
if host_window_height < 1 or host_window_width < 1:
|
|
1962
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Host window too small')
|
|
2489
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Host window too small')
|
|
1963
2490
|
|
|
1964
2491
|
old_stat = ''
|
|
1965
2492
|
old_bottom_stat = ''
|
|
@@ -2020,7 +2547,6 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2020
2547
|
_curses_add_string_to_window(window=help_window,y=12,line='Esc : Clear line', color_pair_list=[-1,-1,1], lead_str='│', box_ansi_color=box_ansi_color)
|
|
2021
2548
|
help_panel = curses.panel.new_panel(help_window)
|
|
2022
2549
|
help_panel.hide()
|
|
2023
|
-
help_shown = False
|
|
2024
2550
|
curses.panel.update_panels()
|
|
2025
2551
|
indexOffset = 0
|
|
2026
2552
|
while host_stats['running'] > 0 or host_stats['waiting'] > 0:
|
|
@@ -2033,7 +2559,7 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2033
2559
|
# with open('keylog.txt','a') as f:
|
|
2034
2560
|
# f.write(str(key)+'\n')
|
|
2035
2561
|
if key == 410 or key == curses.KEY_RESIZE: # 410 is the key code for resize
|
|
2036
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Terminal resize requested')
|
|
2562
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Terminal resize requested')
|
|
2037
2563
|
# if the user pressed ctrl + d and the last line is empty, we will exit by adding 'exit\n' to the last line
|
|
2038
2564
|
elif key == 4 and not __keyPressesIn[-1]:
|
|
2039
2565
|
__keyPressesIn[-1].extend('exit\n')
|
|
@@ -2041,20 +2567,20 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2041
2567
|
elif key == 95 and not __keyPressesIn[-1]: # 95 is the key code for _
|
|
2042
2568
|
# if last line is empty, we will reconfigure the wh to be smaller
|
|
2043
2569
|
if min_line_len != 1:
|
|
2044
|
-
return (lineToDisplay,curserPosition , min_char_len , max(min_line_len -1,1), single_window, 'Decrease line length')
|
|
2570
|
+
return (lineToDisplay,curserPosition , min_char_len , max(min_line_len -1,1), single_window,help_shown, 'Decrease line length')
|
|
2045
2571
|
elif key == 43 and not __keyPressesIn[-1]: # 43 is the key code for +
|
|
2046
2572
|
# if last line is empty, we will reconfigure the wh to be larger
|
|
2047
|
-
return (lineToDisplay,curserPosition , min_char_len , min_line_len +1, single_window, 'Increase line length')
|
|
2573
|
+
return (lineToDisplay,curserPosition , min_char_len , min_line_len +1, single_window,help_shown, 'Increase line length')
|
|
2048
2574
|
elif key == 123 and not __keyPressesIn[-1]: # 123 is the key code for {
|
|
2049
2575
|
# if last line is empty, we will reconfigure the ww to be smaller
|
|
2050
2576
|
if min_char_len != 1:
|
|
2051
|
-
return (lineToDisplay,curserPosition , max(min_char_len -1,1), min_line_len, single_window, 'Decrease character length')
|
|
2577
|
+
return (lineToDisplay,curserPosition , max(min_char_len -1,1), min_line_len, single_window,help_shown, 'Decrease character length')
|
|
2052
2578
|
elif key == 124 and not __keyPressesIn[-1]: # 124 is the key code for |
|
|
2053
2579
|
# if last line is empty, we will toggle the single window mode
|
|
2054
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, not single_window, 'Toggle single window mode')
|
|
2580
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, not single_window,help_shown, 'Toggle single window mode')
|
|
2055
2581
|
elif key == 125 and not __keyPressesIn[-1]: # 125 is the key code for }
|
|
2056
2582
|
# if last line is empty, we will reconfigure the ww to be larger
|
|
2057
|
-
return (lineToDisplay,curserPosition , min_char_len +1, min_line_len, single_window, 'Increase character length')
|
|
2583
|
+
return (lineToDisplay,curserPosition , min_char_len +1, min_line_len, single_window,help_shown, 'Increase character length')
|
|
2058
2584
|
elif key == 60 and not __keyPressesIn[-1]: # 60 is the key code for <
|
|
2059
2585
|
indexOffset = (indexOffset - 1 ) % len(hosts)
|
|
2060
2586
|
elif key == 62 and not __keyPressesIn[-1]: # 62 is the key code for >
|
|
@@ -2089,11 +2615,11 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2089
2615
|
curserPosition = len(__keyPressesIn[lineToDisplay])
|
|
2090
2616
|
elif key == curses.KEY_REFRESH or key == curses.KEY_F5 or key == 18: # 18 is the key code for ctrl + R
|
|
2091
2617
|
# if the key is refresh, we will refresh the screen
|
|
2092
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Refresh requested')
|
|
2618
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Refresh requested')
|
|
2093
2619
|
elif key == curses.KEY_EXIT or key == 27: # 27 is the key code for ESC
|
|
2094
2620
|
# if the key is exit, we will exit the program
|
|
2095
2621
|
return
|
|
2096
|
-
elif key == curses.KEY_HELP or key == 63 or key == curses.KEY_F1: # 63 is the key code for
|
|
2622
|
+
elif key == curses.KEY_HELP or key == 63 or key == curses.KEY_F1 or key == 8: # 63 is the key code for ?, 8 is the key code for backspace
|
|
2097
2623
|
# if the key is help, we will display the help message
|
|
2098
2624
|
if not help_shown:
|
|
2099
2625
|
help_panel.show()
|
|
@@ -2136,7 +2662,7 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2136
2662
|
curserPosition += 1
|
|
2137
2663
|
# reconfigure when the terminal size changes
|
|
2138
2664
|
if org_dim != stdscr.getmaxyx():
|
|
2139
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Terminal resize detected')
|
|
2665
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Terminal resize detected')
|
|
2140
2666
|
# We generate the aggregated stats if user did not input anything
|
|
2141
2667
|
if not __keyPressesIn[lineToDisplay]:
|
|
2142
2668
|
#stats = '┍'+ f" Total: {len(hosts)} Running: {host_stats['running']} Failed: {host_stats['failed']} Finished: {host_stats['finished']} Waiting: {host_stats['waiting']} ww: {min_char_len} wh:{min_line_len} "[:max_x - 2].center(max_x - 2, "━")
|
|
@@ -2205,12 +2731,12 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2205
2731
|
# if the line is visible, we will reprint it
|
|
2206
2732
|
if visibleLowerBound <= lineNumToReprint <= len(host.output):
|
|
2207
2733
|
_curses_add_string_to_window(window=host_window, y=lineNumToReprint + 1, line=host.output[lineNumToReprint], color_pair_list=host.current_color_pair,lead_str='│',keep_top_n_lines=1,box_ansi_color=box_ansi_color,fill_char='')
|
|
2208
|
-
except Exception
|
|
2734
|
+
except Exception:
|
|
2209
2735
|
# import traceback
|
|
2210
2736
|
# print(str(e).strip())
|
|
2211
2737
|
# print(traceback.format_exc().strip())
|
|
2212
2738
|
if org_dim != stdscr.getmaxyx():
|
|
2213
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, 'Terminal resize detected')
|
|
2739
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, 'Terminal resize detected')
|
|
2214
2740
|
if host.lastPrintedUpdateTime != host.lastUpdateTime and host.output_buffer.tell() > 0:
|
|
2215
2741
|
# this means there is still output in the buffer, we will print it
|
|
2216
2742
|
# we will print the output in the window
|
|
@@ -2218,14 +2744,17 @@ def __generate_display(stdscr, hosts, lineToDisplay = -1,curserPosition = 0, min
|
|
|
2218
2744
|
host_window.noutrefresh()
|
|
2219
2745
|
host.lastPrintedUpdateTime = host.lastUpdateTime
|
|
2220
2746
|
hosts_to_display, host_stats,rearrangedHosts = _get_hosts_to_display(hosts, max_num_hosts,hosts_to_display, indexOffset)
|
|
2747
|
+
if help_shown:
|
|
2748
|
+
help_window.touchwin()
|
|
2749
|
+
help_window.noutrefresh()
|
|
2221
2750
|
curses.doupdate()
|
|
2222
2751
|
last_refresh_time = time.perf_counter()
|
|
2223
2752
|
except Exception as e:
|
|
2224
2753
|
import traceback
|
|
2225
|
-
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window, f'Error: {str(e)}',traceback.format_exc())
|
|
2754
|
+
return (lineToDisplay,curserPosition , min_char_len, min_line_len, single_window,help_shown, f'Error: {str(e)}',traceback.format_exc())
|
|
2226
2755
|
return None
|
|
2227
2756
|
|
|
2228
|
-
def curses_print(stdscr, hosts, threads, min_char_len =
|
|
2757
|
+
def curses_print(stdscr, hosts, threads, min_char_len = DEFAULT_WINDOW_WIDTH, min_line_len = DEFAULT_WINDOW_HEIGHT,single_window = DEFAULT_SINGLE_WINDOW):
|
|
2229
2758
|
'''
|
|
2230
2759
|
Print the output of the hosts on the screen
|
|
2231
2760
|
|
|
@@ -2270,9 +2799,9 @@ def curses_print(stdscr, hosts, threads, min_char_len = DEFAULT_CURSES_MINIMUM_C
|
|
|
2270
2799
|
# print if can change color
|
|
2271
2800
|
_curses_add_string_to_window(window=stdscr, y=8, line=f"Real color capability: {curses.can_change_color()}", number_of_char_to_write=stdscr.getmaxyx()[1] - 1)
|
|
2272
2801
|
stdscr.refresh()
|
|
2273
|
-
except:
|
|
2802
|
+
except Exception:
|
|
2274
2803
|
pass
|
|
2275
|
-
params = (-1,0 , min_char_len, min_line_len, single_window,'new config')
|
|
2804
|
+
params = (-1,0 , min_char_len, min_line_len, single_window,False,'new config')
|
|
2276
2805
|
while params:
|
|
2277
2806
|
params = __generate_display(stdscr, hosts, *params)
|
|
2278
2807
|
if not params:
|
|
@@ -2283,39 +2812,317 @@ def curses_print(stdscr, hosts, threads, min_char_len = DEFAULT_CURSES_MINIMUM_C
|
|
|
2283
2812
|
# print the current configuration
|
|
2284
2813
|
stdscr.clear()
|
|
2285
2814
|
try:
|
|
2286
|
-
stdscr.addstr(0, 0, f"{params[
|
|
2287
|
-
if len(params) >
|
|
2815
|
+
stdscr.addstr(0, 0, f"{params[6]}, Reloading Configuration: min_char_len={params[2]}, min_line_len={params[3]}, single_window={params[4]} with window size {stdscr.getmaxyx()} and {len(hosts)} hosts...")
|
|
2816
|
+
if len(params) > 7:
|
|
2288
2817
|
# traceback is available, print it
|
|
2289
2818
|
i = 1
|
|
2290
|
-
for line in params[
|
|
2819
|
+
for line in params[7].split('\n'):
|
|
2291
2820
|
stdscr.addstr(i, 0, line)
|
|
2292
2821
|
i += 1
|
|
2293
2822
|
stdscr.refresh()
|
|
2294
|
-
except:
|
|
2823
|
+
except Exception:
|
|
2295
2824
|
pass
|
|
2296
|
-
params = params[:
|
|
2825
|
+
params = params[:6] + ('new config',)
|
|
2297
2826
|
time.sleep(0.01)
|
|
2298
2827
|
#time.sleep(0.25)
|
|
2299
2828
|
|
|
2300
2829
|
#%% ------------ Generate Output Block ----------------
|
|
2301
|
-
def
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2830
|
+
def can_merge(line_bag1, line_bag2, threshold):
|
|
2831
|
+
if threshold > 0.5:
|
|
2832
|
+
samples = itertools.islice(line_bag1, max(int(len(line_bag1) * (1 - threshold)),1))
|
|
2833
|
+
if not line_bag2.intersection(samples):
|
|
2834
|
+
return False
|
|
2835
|
+
return len(line_bag1.intersection(line_bag2)) >= min(len(line_bag1),len(line_bag2)) * threshold
|
|
2836
|
+
|
|
2837
|
+
def mergeOutput(merging_hostnames,outputs_by_hostname,output,diff_display_threshold,line_length):
|
|
2838
|
+
#indexes = {hostname: 0 for hostname in merging_hostnames}
|
|
2839
|
+
indexes = Counter({hostname: 0 for hostname in merging_hostnames})
|
|
2840
|
+
working_index_keys = set(merging_hostnames)
|
|
2841
|
+
previousBuddies = set()
|
|
2842
|
+
hostnameWrapper = textwrap.TextWrapper(width=line_length -1, tabsize=4, replace_whitespace=False, drop_whitespace=False, break_on_hyphens=False,initial_indent=' ', subsequent_indent='- ')
|
|
2843
|
+
hostnameWrapper.wordsep_simple_re = re.compile(r'([,]+)')
|
|
2844
|
+
diff_display_item_count = max(1,int(max(map(len, outputs_by_hostname.values())) * (1 - diff_display_threshold)))
|
|
2845
|
+
def get_multiset_index_for_hostname(hostname):
|
|
2846
|
+
index = indexes[hostname]
|
|
2847
|
+
tracking_index = min(index + diff_display_item_count,len(outputs_by_hostname[hostname]))
|
|
2848
|
+
tracking_iter = itertools.islice(outputs_by_hostname[hostname], tracking_index)
|
|
2849
|
+
return [deque(outputs_by_hostname[hostname][index:tracking_index],maxlen=diff_display_item_count),tracking_iter]
|
|
2850
|
+
# futuresChainMap = ChainMap()
|
|
2851
|
+
# class futureDict(UserDict):
|
|
2852
|
+
# def __missing__(self, key):
|
|
2853
|
+
# value = get_multiset_index_for_hostname(key)
|
|
2854
|
+
# self[key] = value
|
|
2855
|
+
# # futuresChainMap.maps.append(value[0]._counter)
|
|
2856
|
+
# return value
|
|
2857
|
+
# # def initializeHostnames(self, hostnames):
|
|
2858
|
+
# # entries = {hostname: get_multiset_index_for_hostname(hostname) for hostname in hostnames}
|
|
2859
|
+
# # self.update(entries)
|
|
2860
|
+
# # futuresChainMap.maps.extend(entry[0]._counter for entry in entries.values())
|
|
2861
|
+
def advance(dict,key):
|
|
2862
|
+
try:
|
|
2863
|
+
value = dict[key]
|
|
2864
|
+
value[0].append(next(value[1]))
|
|
2865
|
+
except StopIteration:
|
|
2866
|
+
try:
|
|
2867
|
+
value[0].popleft()
|
|
2868
|
+
except IndexError:
|
|
2869
|
+
pass
|
|
2870
|
+
except KeyError:
|
|
2871
|
+
pass
|
|
2872
|
+
# futures = futureDict()
|
|
2873
|
+
# for hostname in merging_hostnames:
|
|
2874
|
+
# futures[hostname] # ensure it's initialized
|
|
2875
|
+
futures = {hostname: get_multiset_index_for_hostname(hostname) for hostname in merging_hostnames}
|
|
2876
|
+
currentLines = defaultdict(set)
|
|
2877
|
+
color_cap = get_terminal_color_capability()
|
|
2878
|
+
if color_cap == 'None':
|
|
2879
|
+
green_str = ''
|
|
2880
|
+
reset_str = ''
|
|
2881
|
+
else:
|
|
2882
|
+
green_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('green', __DEFAULT_COLOR_PALETTE['green']))
|
|
2883
|
+
reset_str = '\033[0m'
|
|
2884
|
+
for hostname in merging_hostnames:
|
|
2885
|
+
currentLines[outputs_by_hostname[hostname][0]].add(hostname)
|
|
2886
|
+
while indexes:
|
|
2887
|
+
defer = False
|
|
2888
|
+
# sorted_working_hostnames = sorted(working_index_keys, key=lambda hn: indexes[hn])
|
|
2889
|
+
golden_hostname = min(working_index_keys, key=indexes.get)
|
|
2890
|
+
golden_index = indexes[golden_hostname]
|
|
2891
|
+
lineToAdd = outputs_by_hostname[golden_hostname][golden_index]
|
|
2892
|
+
# for hostname, index in sorted_working_indexes[1:]:
|
|
2893
|
+
# if lineToAdd == outputs_by_hostname[hostname][index]:
|
|
2894
|
+
# buddy.add(hostname)
|
|
2895
|
+
# else:
|
|
2896
|
+
# futureLines,tracking_index = futures[hostname]
|
|
2897
|
+
# if lineToAdd in futureLines:
|
|
2898
|
+
# for hn in buddy:
|
|
2899
|
+
# working_indexes.pop(hn,None)
|
|
2900
|
+
# defer = True
|
|
2901
|
+
# break
|
|
2902
|
+
buddy = currentLines[lineToAdd].copy()
|
|
2903
|
+
if len(buddy) < len(working_index_keys):
|
|
2904
|
+
# we need to check the futures then
|
|
2905
|
+
# thisCounter = None
|
|
2906
|
+
# if golden_hostname in futures:
|
|
2907
|
+
# thisCounter = futures[golden_hostname][0]._counter
|
|
2908
|
+
# futuresChainMap.maps.remove(thisCounter)
|
|
2909
|
+
# for hostname in working_index_keys - buddy - set(futures.keys()):
|
|
2910
|
+
# futures[hostname] # ensure it's initialized
|
|
2911
|
+
# futures.initializeHostnames(working_index_keys - buddy - futures.keys())
|
|
2912
|
+
if any(lineToAdd in futures[hostname][0] for hostname in working_index_keys - buddy):
|
|
2913
|
+
defer = True
|
|
2914
|
+
working_index_keys -= buddy
|
|
2915
|
+
# if thisCounter is not None:
|
|
2916
|
+
# futuresChainMap.maps.append(thisCounter)
|
|
2917
|
+
if not defer:
|
|
2918
|
+
if buddy != previousBuddies:
|
|
2919
|
+
hostnameStr = ','.join(compact_hostnames(buddy))
|
|
2920
|
+
hostnameLines = hostnameWrapper.wrap(hostnameStr)
|
|
2921
|
+
# hostnameLines = [line.ljust(line_length) for line in hostnameLines]
|
|
2922
|
+
if color_cap == 'None':
|
|
2923
|
+
hostnameLines[0] = f"■{hostnameLines[0]}"
|
|
2924
|
+
elif len(buddy) < len(merging_hostnames):
|
|
2925
|
+
color = int_to_unique_ansi_color(hash(hostnameStr))
|
|
2926
|
+
hostnameLines[0] = f"{color}■{hostnameLines[0]}"
|
|
2927
|
+
hostnameLines[-1] += reset_str
|
|
2928
|
+
else:
|
|
2929
|
+
hostnameLines[0] = f"{green_str}■{reset_str}{hostnameLines[0]}"
|
|
2930
|
+
output.extend(hostnameLines)
|
|
2931
|
+
previousBuddies = buddy
|
|
2932
|
+
output.append(lineToAdd)
|
|
2933
|
+
currentLines[lineToAdd].difference_update(buddy)
|
|
2934
|
+
if not currentLines[lineToAdd]:
|
|
2935
|
+
del currentLines[lineToAdd]
|
|
2936
|
+
indexes.update(buddy)
|
|
2937
|
+
for hostname in buddy:
|
|
2938
|
+
# currentLines[lineToAdd].remove(hostname)
|
|
2939
|
+
# if not currentLines[lineToAdd]:
|
|
2940
|
+
# del currentLines[lineToAdd]
|
|
2941
|
+
# indexes[hostname] += 1
|
|
2942
|
+
try:
|
|
2943
|
+
currentLines[outputs_by_hostname[hostname][indexes[hostname]]].add(hostname)
|
|
2944
|
+
except IndexError:
|
|
2945
|
+
indexes.pop(hostname, None)
|
|
2946
|
+
futures.pop(hostname, None)
|
|
2947
|
+
# if future:
|
|
2948
|
+
# futuresChainMap.maps.remove(future[0]._counter)
|
|
2949
|
+
continue
|
|
2950
|
+
#advance futures
|
|
2951
|
+
advance(futures, hostname)
|
|
2952
|
+
working_index_keys = set(indexes.keys())
|
|
2953
|
+
|
|
2954
|
+
def mergeOutputs(outputs_by_hostname, merge_groups, remaining_hostnames, diff_display_threshold, line_length):
|
|
2955
|
+
output = []
|
|
2956
|
+
color_cap = get_terminal_color_capability()
|
|
2957
|
+
if color_cap == 'None':
|
|
2958
|
+
color_line = ''
|
|
2959
|
+
color_reset = ''
|
|
2960
|
+
green_str = ''
|
|
2961
|
+
else:
|
|
2962
|
+
color_line = rgb_to_ansi_color_string(*COLOR_PALETTE.get('white', __DEFAULT_COLOR_PALETTE['white']))
|
|
2963
|
+
color_reset = '\033[0m'
|
|
2964
|
+
green_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('green', __DEFAULT_COLOR_PALETTE['green']))
|
|
2965
|
+
output.append(color_line+'─'*(line_length)+color_reset)
|
|
2966
|
+
hostnameWrapper = textwrap.TextWrapper(width=line_length - 1, tabsize=4, replace_whitespace=False, drop_whitespace=False, break_on_hyphens=False,initial_indent=' ', subsequent_indent='- ')
|
|
2967
|
+
hostnameWrapper.wordsep_simple_re = re.compile(r'([,]+)')
|
|
2968
|
+
for merging_hostnames in merge_groups:
|
|
2969
|
+
mergeOutput(merging_hostnames, outputs_by_hostname, output, diff_display_threshold,line_length)
|
|
2970
|
+
output.append(color_line+'─'*(line_length)+color_reset)
|
|
2971
|
+
for hostname in remaining_hostnames:
|
|
2972
|
+
hostnameLines = hostnameWrapper.wrap(','.join(compact_hostnames([hostname])))
|
|
2973
|
+
hostnameLines[0] = f"{green_str}■{color_reset}{hostnameLines[0]}"
|
|
2974
|
+
output.extend(hostnameLines)
|
|
2975
|
+
output.extend(outputs_by_hostname[hostname])
|
|
2976
|
+
output.append(color_line+'─'*(line_length)+color_reset)
|
|
2977
|
+
if output:
|
|
2978
|
+
output.pop()
|
|
2979
|
+
# if output and output[0] and output[0].startswith('├'):
|
|
2980
|
+
# output[0] = '┌' + output[0][1:]
|
|
2981
|
+
return output
|
|
2982
|
+
|
|
2983
|
+
def pre_merge_hosts(hosts):
|
|
2984
|
+
'''Merge hosts with identical outputs.'''
|
|
2985
|
+
output_groups = defaultdict(list)
|
|
2986
|
+
# Group hosts by their output identity
|
|
2987
|
+
for host in hosts:
|
|
2988
|
+
identity = host.get_output_hash()
|
|
2989
|
+
output_groups[identity].append(host)
|
|
2990
|
+
# Create merged hosts
|
|
2991
|
+
merged_hosts = []
|
|
2992
|
+
for group in output_groups.values():
|
|
2993
|
+
group[0].name = ','.join(compact_hostnames(host.name for host in group))
|
|
2994
|
+
merged_hosts.append(group[0])
|
|
2995
|
+
return merged_hosts
|
|
2996
|
+
|
|
2997
|
+
def get_host_raw_output(hosts, terminal_width):
|
|
2998
|
+
outputs_by_hostname = {}
|
|
2999
|
+
line_bag_by_hostname = {}
|
|
3000
|
+
hostnames_by_line_bag_len = {}
|
|
3001
|
+
text_wrapper = textwrap.TextWrapper(width=terminal_width - 1, tabsize=4, replace_whitespace=False, drop_whitespace=False,
|
|
3002
|
+
initial_indent=' ', subsequent_indent='-')
|
|
3003
|
+
max_length = 20
|
|
3004
|
+
color_cap = get_terminal_color_capability()
|
|
3005
|
+
if color_cap == 'None':
|
|
3006
|
+
color_reset_str = ''
|
|
3007
|
+
blue_str = ''
|
|
3008
|
+
cyan_str = ''
|
|
3009
|
+
green_str = ''
|
|
3010
|
+
red_str = ''
|
|
3011
|
+
else:
|
|
3012
|
+
color_reset_str = '\033[0m'
|
|
3013
|
+
blue_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('blue', __DEFAULT_COLOR_PALETTE['blue']))
|
|
3014
|
+
cyan_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('bright_cyan', __DEFAULT_COLOR_PALETTE['bright_cyan']))
|
|
3015
|
+
green_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('bright_green', __DEFAULT_COLOR_PALETTE['bright_green']))
|
|
3016
|
+
red_str = rgb_to_ansi_color_string(*COLOR_PALETTE.get('bright_red', __DEFAULT_COLOR_PALETTE['bright_red']))
|
|
3017
|
+
hosts = pre_merge_hosts(hosts)
|
|
3018
|
+
for host in hosts:
|
|
3019
|
+
max_length = max(max_length, len(max(host.name.split(','), key=len)) + 3)
|
|
3020
|
+
hostPrintOut = [f"{cyan_str}█{color_reset_str} EXECUTED COMMAND:"]
|
|
3021
|
+
for line in host.command.splitlines():
|
|
3022
|
+
hostPrintOut.extend(text_wrapper.wrap(line))
|
|
3023
|
+
# hostPrintOut.extend(itertools.chain.from_iterable(text_wrapper.wrap(line) for line in host['command'].splitlines()))
|
|
3024
|
+
lineBag = {(0,host.command)}
|
|
3025
|
+
prevLine = host.command
|
|
3026
|
+
if host.stdout:
|
|
3027
|
+
hostPrintOut.append(f'{blue_str}▓{color_reset_str} STDOUT:')
|
|
3028
|
+
# for line in host.stdout:
|
|
3029
|
+
# if len(line) < terminal_width - 2:
|
|
3030
|
+
# hostPrintOut.append(f" {line}")
|
|
3031
|
+
# else:
|
|
3032
|
+
# hostPrintOut.extend(text_wrapper.wrap(line))
|
|
3033
|
+
hostPrintOut.extend(f" {line}" for line in host.stdout)
|
|
3034
|
+
max_length = max(max_length, max(map(len, host.stdout)))
|
|
3035
|
+
# hostPrintOut.extend(text_wrapper.wrap(line) for line in host.stdout)
|
|
3036
|
+
lineBag.add((prevLine,1))
|
|
3037
|
+
lineBag.add((1,host.stdout[0]))
|
|
3038
|
+
if len(host.stdout) > 1:
|
|
3039
|
+
lineBag.update(zip(host.stdout, host.stdout[1:]))
|
|
3040
|
+
lineBag.update(host.stdout)
|
|
3041
|
+
prevLine = host.stdout[-1]
|
|
3042
|
+
if host.stderr:
|
|
3043
|
+
if host.stderr[0].strip().startswith('ssh: connect to host ') and host.stderr[0].strip().endswith('Connection refused'):
|
|
3044
|
+
host.stderr[0] = 'SSH not reachable!'
|
|
3045
|
+
elif host.stderr[-1].strip().endswith('Connection timed out'):
|
|
3046
|
+
host.stderr[-1] = 'SSH connection timed out!'
|
|
3047
|
+
elif host.stderr[-1].strip().endswith('No route to host'):
|
|
3048
|
+
host.stderr[-1] = 'Cannot find host!'
|
|
3049
|
+
if host.stderr:
|
|
3050
|
+
hostPrintOut.append(f'{red_str}▒{color_reset_str} STDERR:')
|
|
3051
|
+
# for line in host.stderr:
|
|
3052
|
+
# if len(line) < terminal_width - 2:
|
|
3053
|
+
# hostPrintOut.append(f" {line}")
|
|
3054
|
+
# else:
|
|
3055
|
+
# hostPrintOut.extend(text_wrapper.wrap(line))
|
|
3056
|
+
hostPrintOut.extend(f" {line}" for line in host.stderr)
|
|
3057
|
+
max_length = max(max_length, max(map(len, host.stderr)))
|
|
3058
|
+
lineBag.add((prevLine,2))
|
|
3059
|
+
lineBag.add((2,host.stderr[0]))
|
|
3060
|
+
lineBag.update(host.stderr)
|
|
3061
|
+
if len(host.stderr) > 1:
|
|
3062
|
+
lineBag.update(zip(host.stderr, host.stderr[1:]))
|
|
3063
|
+
prevLine = host.stderr[-1]
|
|
3064
|
+
if host.returncode != 0:
|
|
3065
|
+
codeColor = red_str
|
|
3066
|
+
else:
|
|
3067
|
+
codeColor = green_str
|
|
3068
|
+
hostPrintOut.append(f"{codeColor}░{color_reset_str} RETURN CODE: {host.returncode}")
|
|
3069
|
+
lineBag.add((prevLine,f"{host.returncode}"))
|
|
3070
|
+
outputs_by_hostname[host.name] = hostPrintOut
|
|
3071
|
+
line_bag_by_hostname[host.name] = lineBag
|
|
3072
|
+
hostnames_by_line_bag_len.setdefault(len(lineBag), set()).add(host.name)
|
|
3073
|
+
return outputs_by_hostname, line_bag_by_hostname, hostnames_by_line_bag_len, sorted(hostnames_by_line_bag_len), min(max_length+2,terminal_width)
|
|
3074
|
+
|
|
3075
|
+
def form_merge_groups(hostnames_by_line_bag_len, sorted_hostnames_by_line_bag_len_keys, line_bag_by_hostname, diff_display_threshold):
|
|
3076
|
+
merge_groups = []
|
|
3077
|
+
remaining_hostnames = set()
|
|
3078
|
+
for lbl_i, line_bag_len in enumerate(sorted_hostnames_by_line_bag_len_keys):
|
|
3079
|
+
for this_hostname in hostnames_by_line_bag_len.get(line_bag_len, set()).copy():
|
|
3080
|
+
# if this_hostname not in hostnames_by_line_bag_len.get(line_bag_len, set()):
|
|
3081
|
+
# continue
|
|
3082
|
+
try:
|
|
3083
|
+
this_line_bag = line_bag_by_hostname.pop(this_hostname)
|
|
3084
|
+
hostnames_by_line_bag_len.get(line_bag_len, set()).discard(this_hostname)
|
|
3085
|
+
except KeyError:
|
|
3086
|
+
continue
|
|
3087
|
+
target_threshold = line_bag_len * (2 - diff_display_threshold)
|
|
3088
|
+
merge_group = []
|
|
3089
|
+
for other_line_bag_len in sorted_hostnames_by_line_bag_len_keys[lbl_i:]:
|
|
3090
|
+
if other_line_bag_len > target_threshold:
|
|
3091
|
+
break
|
|
3092
|
+
# if other_line_bag_len < line_bag_len:
|
|
3093
|
+
# continue
|
|
3094
|
+
for other_hostname in hostnames_by_line_bag_len.get(other_line_bag_len, set()).copy():
|
|
3095
|
+
if can_merge(this_line_bag, line_bag_by_hostname[other_hostname], diff_display_threshold):
|
|
3096
|
+
merge_group.append(other_hostname)
|
|
3097
|
+
hostnames_by_line_bag_len[other_line_bag_len].remove(other_hostname)
|
|
3098
|
+
if not hostnames_by_line_bag_len[other_line_bag_len]:
|
|
3099
|
+
del hostnames_by_line_bag_len[other_line_bag_len]
|
|
3100
|
+
del line_bag_by_hostname[other_hostname]
|
|
3101
|
+
if merge_group:
|
|
3102
|
+
merge_group.append(this_hostname)
|
|
3103
|
+
merge_groups.append(merge_group)
|
|
3104
|
+
# del line_bag_by_hostname[this_hostname]
|
|
3105
|
+
else:
|
|
3106
|
+
remaining_hostnames.add(this_hostname)
|
|
3107
|
+
return merge_groups, remaining_hostnames
|
|
3108
|
+
|
|
3109
|
+
def generate_output(hosts, usejson = False, greppable = False,quiet = False,encoding = _encoding,keyPressesIn = [[]]):
|
|
3110
|
+
color_cap = get_terminal_color_capability()
|
|
3111
|
+
if quiet:
|
|
2306
3112
|
# remove hosts with returncode 0
|
|
2307
|
-
hosts = [
|
|
3113
|
+
hosts = [host for host in hosts if host.returncode != 0]
|
|
2308
3114
|
if not hosts:
|
|
2309
3115
|
if usejson:
|
|
2310
3116
|
return '{"Success": true}'
|
|
2311
3117
|
else:
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
3118
|
+
if color_cap == 'None':
|
|
3119
|
+
return 'Success'
|
|
3120
|
+
else:
|
|
3121
|
+
return '\033[32mSuccess\033[0m'
|
|
2315
3122
|
if usejson:
|
|
2316
3123
|
# [print(dict(host)) for host in hosts]
|
|
2317
3124
|
#print(json.dumps([dict(host) for host in hosts],indent=4))
|
|
2318
|
-
rtnStr = json.dumps(hosts,indent=4)
|
|
3125
|
+
rtnStr = json.dumps([dict(host) for host in hosts],indent=4)
|
|
2319
3126
|
elif greppable:
|
|
2320
3127
|
# transform hosts to a 2d list
|
|
2321
3128
|
rtnStr = '*'*80+'\n'
|
|
@@ -2323,58 +3130,55 @@ def generate_output(hosts, usejson = False, greppable = False):
|
|
|
2323
3130
|
for host in hosts:
|
|
2324
3131
|
#header = f"{host['name']} | rc: {host['returncode']} | "
|
|
2325
3132
|
hostAdded = False
|
|
2326
|
-
for line in host
|
|
2327
|
-
rtnList.append([host
|
|
3133
|
+
for line in host.stdout:
|
|
3134
|
+
rtnList.append([host.name,f"rc: {host.returncode}",'stdout',line])
|
|
2328
3135
|
hostAdded = True
|
|
2329
|
-
for line in host
|
|
2330
|
-
rtnList.append([host
|
|
3136
|
+
for line in host.stderr:
|
|
3137
|
+
rtnList.append([host.name,f"rc: {host.returncode}",'stderr',line])
|
|
2331
3138
|
hostAdded = True
|
|
2332
3139
|
if not hostAdded:
|
|
2333
|
-
rtnList.append([host
|
|
3140
|
+
rtnList.append([host.name,f"rc: {host.returncode}",'N/A','<EMPTY>'])
|
|
2334
3141
|
rtnList.append(['','','',''])
|
|
2335
3142
|
rtnStr += pretty_format_table(rtnList)
|
|
2336
3143
|
rtnStr += '*'*80+'\n'
|
|
2337
|
-
if
|
|
2338
|
-
CMDsOut = [''.join(cmd).encode(encoding=
|
|
3144
|
+
if keyPressesIn[-1]:
|
|
3145
|
+
CMDsOut = [''.join(cmd).encode(encoding=encoding,errors='backslashreplace').decode(encoding=encoding,errors='backslashreplace').replace('\\n', '↵') for cmd in keyPressesIn if cmd]
|
|
2339
3146
|
rtnStr += 'User Inputs: '+ '\nUser Inputs: '.join(CMDsOut)
|
|
2340
3147
|
#rtnStr += '\n'
|
|
2341
3148
|
else:
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
3149
|
+
try:
|
|
3150
|
+
diff_display_threshold = float(DEFAULT_DIFF_DISPLAY_THRESHOLD)
|
|
3151
|
+
if diff_display_threshold < 0 or diff_display_threshold > 1:
|
|
3152
|
+
raise ValueError
|
|
3153
|
+
except Exception:
|
|
3154
|
+
eprint("Warning: diff_display_threshold should be a float between 0 and 1. Setting to default value of 0.9")
|
|
3155
|
+
diff_display_threshold = 0.9
|
|
3156
|
+
|
|
3157
|
+
color_reset_str = '' if color_cap == 'None' else '\033[0m'
|
|
3158
|
+
white_str = '' if color_cap == 'None' else rgb_to_ansi_color_string(*COLOR_PALETTE.get('white', __DEFAULT_COLOR_PALETTE['white']))
|
|
3159
|
+
terminal_length = get_terminal_size()[0]
|
|
3160
|
+
outputs_by_hostname, line_bag_by_hostname, hostnames_by_line_bag_len, sorted_hostnames_by_line_bag_len_keys, line_length = get_host_raw_output(hosts,terminal_length)
|
|
3161
|
+
merge_groups ,remaining_hostnames = form_merge_groups(hostnames_by_line_bag_len, sorted_hostnames_by_line_bag_len_keys, line_bag_by_hostname, diff_display_threshold)
|
|
3162
|
+
outputs = mergeOutputs(outputs_by_hostname, merge_groups,remaining_hostnames, diff_display_threshold,line_length)
|
|
3163
|
+
if keyPressesIn[-1]:
|
|
3164
|
+
CMDsOut = [''.join(cmd).encode(encoding=encoding,errors='backslashreplace').decode(encoding=encoding,errors='backslashreplace').replace('\\n', '↵') for cmd in keyPressesIn if cmd]
|
|
3165
|
+
outputs.append(color_reset_str + "░ User Inputs:".ljust(line_length,'─'))
|
|
3166
|
+
cmdOut = []
|
|
3167
|
+
for line in CMDsOut:
|
|
3168
|
+
cmdOut.extend(textwrap.wrap(line, width=line_length-1, tabsize=4, replace_whitespace=False, drop_whitespace=False,
|
|
3169
|
+
initial_indent=' ', subsequent_indent='-'))
|
|
3170
|
+
outputs.extend(cmdOut)
|
|
3171
|
+
keyPressesIn[-1].clear()
|
|
3172
|
+
if not outputs:
|
|
3173
|
+
if quiet:
|
|
3174
|
+
if color_cap == 'None':
|
|
3175
|
+
return 'Success'
|
|
3176
|
+
else:
|
|
3177
|
+
return '\033[32mSuccess\033[0m'
|
|
2363
3178
|
else:
|
|
2364
|
-
rtnStr
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
rtnStr += '*'*80+'\n'
|
|
2368
|
-
if __keyPressesIn[-1]:
|
|
2369
|
-
CMDsOut = [''.join(cmd).encode(encoding=_encoding,errors='backslashreplace').decode(encoding=_encoding,errors='backslashreplace').replace('\\n', '↵') for cmd in __keyPressesIn if cmd]
|
|
2370
|
-
#rtnStr += f"Key presses: {''.join(__keyPressesIn).encode('unicode_escape').decode()}\n"
|
|
2371
|
-
#rtnStr += f"Key presses: {__keyPressesIn}\n"
|
|
2372
|
-
rtnStr += "User Inputs: \n "
|
|
2373
|
-
rtnStr += '\n '.join(CMDsOut)
|
|
2374
|
-
rtnStr += '\n'
|
|
2375
|
-
__keyPressesIn[-1].clear()
|
|
2376
|
-
if __global_suppress_printout and not outputs:
|
|
2377
|
-
rtnStr += 'Success'
|
|
3179
|
+
rtnStr = ''
|
|
3180
|
+
else:
|
|
3181
|
+
rtnStr = '\n'.join(outputs + [white_str + '─' * (line_length) + color_reset_str])
|
|
2378
3182
|
return rtnStr
|
|
2379
3183
|
|
|
2380
3184
|
def print_output(hosts,usejson = False,quiet = False,greppable = False):
|
|
@@ -2389,15 +3193,20 @@ def print_output(hosts,usejson = False,quiet = False,greppable = False):
|
|
|
2389
3193
|
Returns:
|
|
2390
3194
|
str: The pretty output generated
|
|
2391
3195
|
'''
|
|
2392
|
-
|
|
3196
|
+
global __global_suppress_printout
|
|
3197
|
+
global _encoding
|
|
3198
|
+
global __keyPressesIn
|
|
3199
|
+
for host in hosts:
|
|
3200
|
+
host.output.clear()
|
|
3201
|
+
rtnStr = generate_output(hosts,usejson,greppable,quiet=__global_suppress_printout,encoding=_encoding,keyPressesIn=__keyPressesIn)
|
|
2393
3202
|
if not quiet:
|
|
2394
3203
|
print(rtnStr)
|
|
2395
3204
|
return rtnStr
|
|
2396
3205
|
|
|
2397
3206
|
#%% ------------ Run / Process Hosts Block ----------------
|
|
2398
3207
|
def processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinished, no_watch, json, called, greppable,
|
|
2399
|
-
unavailableHosts:dict,willUpdateUnreachableHosts,
|
|
2400
|
-
|
|
3208
|
+
unavailableHosts:dict,willUpdateUnreachableHosts,window_width = DEFAULT_WINDOW_WIDTH,
|
|
3209
|
+
window_height = DEFAULT_WINDOW_HEIGHT,single_window = DEFAULT_SINGLE_WINDOW,
|
|
2401
3210
|
unavailable_host_expiry = DEFAULT_UNAVAILABLE_HOST_EXPIRY):
|
|
2402
3211
|
global __globalUnavailableHosts
|
|
2403
3212
|
global _no_env
|
|
@@ -2415,7 +3224,16 @@ def processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinishe
|
|
|
2415
3224
|
if total_sleeped > 0.1:
|
|
2416
3225
|
break
|
|
2417
3226
|
if any([host.returncode is None for host in hosts]):
|
|
2418
|
-
|
|
3227
|
+
try:
|
|
3228
|
+
curses.wrapper(curses_print, hosts, threads, min_char_len = window_width, min_line_len = window_height, single_window = single_window)
|
|
3229
|
+
except Exception:
|
|
3230
|
+
try:
|
|
3231
|
+
os.environ['TERM'] = 'xterm-256color'
|
|
3232
|
+
curses.wrapper(curses_print, hosts, threads, min_char_len = window_width, min_line_len = window_height, single_window = single_window)
|
|
3233
|
+
except Exception as e:
|
|
3234
|
+
eprint(f"Curses print error: {e}")
|
|
3235
|
+
import traceback
|
|
3236
|
+
print(traceback.format_exc())
|
|
2419
3237
|
if not returnUnfinished:
|
|
2420
3238
|
# wait until all hosts have a return code
|
|
2421
3239
|
while any([host.returncode is None for host in hosts]):
|
|
@@ -2424,56 +3242,58 @@ def processRunOnHosts(timeout, password, max_connections, hosts, returnUnfinishe
|
|
|
2424
3242
|
sleep_interval *= 1.1
|
|
2425
3243
|
for thread in threads:
|
|
2426
3244
|
thread.join(timeout=3)
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
3245
|
+
# update the unavailable hosts and global unavailable hosts
|
|
3246
|
+
if willUpdateUnreachableHosts:
|
|
3247
|
+
availableHosts = set()
|
|
3248
|
+
for host in hosts:
|
|
3249
|
+
if host.stderr and ('No route to host' in host.stderr[0].strip() or 'Connection timed out' in host.stderr[0].strip() or (host.stderr[-1].strip().startswith('Timeout!') and host.returncode == 124)):
|
|
3250
|
+
unavailableHosts[host.name] = int(time.monotonic() + unavailable_host_expiry)
|
|
3251
|
+
__globalUnavailableHosts[host.name] = int(time.monotonic() + unavailable_host_expiry)
|
|
3252
|
+
else:
|
|
3253
|
+
availableHosts.add(host.name)
|
|
3254
|
+
if host.name in unavailableHosts:
|
|
3255
|
+
del unavailableHosts[host.name]
|
|
3256
|
+
if host.name in __globalUnavailableHosts:
|
|
3257
|
+
del __globalUnavailableHosts[host.name]
|
|
3258
|
+
if __DEBUG_MODE:
|
|
3259
|
+
print(f'Unreachable hosts: {unavailableHosts}')
|
|
3260
|
+
try:
|
|
3261
|
+
# check for the old content, only update if the new content is different
|
|
3262
|
+
if not os.path.exists(os.path.join(tempfile.gettempdir(),f'__{getpass.getuser()}_multiSSH3_UNAVAILABLE_HOSTS.csv')):
|
|
3263
|
+
with open(os.path.join(tempfile.gettempdir(),f'__{getpass.getuser()}_multiSSH3_UNAVAILABLE_HOSTS.csv'),'w') as f:
|
|
3264
|
+
f.writelines(f'{host},{expTime}' for host,expTime in unavailableHosts.items())
|
|
3265
|
+
else:
|
|
3266
|
+
oldDic = {}
|
|
3267
|
+
try:
|
|
3268
|
+
with open(os.path.join(tempfile.gettempdir(),f'__{getpass.getuser()}_multiSSH3_UNAVAILABLE_HOSTS.csv'),'r') as f:
|
|
3269
|
+
for line in f:
|
|
3270
|
+
line = line.strip()
|
|
3271
|
+
if line and ',' in line and len(line.split(',')) >= 2 and line.split(',')[0] and line.split(',')[1].isdigit():
|
|
3272
|
+
hostname = line.split(',')[0]
|
|
3273
|
+
expireTime = int(line.split(',')[1])
|
|
3274
|
+
if expireTime < time.monotonic() and hostname not in availableHosts:
|
|
3275
|
+
oldDic[hostname] = expireTime
|
|
3276
|
+
except Exception:
|
|
3277
|
+
pass
|
|
3278
|
+
# add new entries
|
|
3279
|
+
oldDic.update(unavailableHosts)
|
|
3280
|
+
with open(os.path.join(tempfile.gettempdir(),getpass.getuser()+'__multiSSH3_UNAVAILABLE_HOSTS.csv.new'),'w') as f:
|
|
3281
|
+
for key, value in oldDic.items():
|
|
3282
|
+
f.write(f'{key},{value}\n')
|
|
3283
|
+
os.replace(os.path.join(tempfile.gettempdir(),getpass.getuser()+'__multiSSH3_UNAVAILABLE_HOSTS.csv.new'),os.path.join(tempfile.gettempdir(),f'__{getpass.getuser()}_multiSSH3_UNAVAILABLE_HOSTS.csv'))
|
|
3284
|
+
except Exception as e:
|
|
3285
|
+
eprint(f'Error writing to temporary file: {e!r}')
|
|
3286
|
+
import traceback
|
|
3287
|
+
eprint(traceback.format_exc())
|
|
3288
|
+
if not called:
|
|
3289
|
+
print_output(hosts,json,greppable=greppable)
|
|
3290
|
+
else:
|
|
3291
|
+
__running_threads.update(threads)
|
|
2471
3292
|
# print the output, if the output of multiple hosts are the same, we aggragate them
|
|
2472
|
-
|
|
2473
|
-
print_output(hosts,json,greppable=greppable)
|
|
3293
|
+
|
|
2474
3294
|
|
|
2475
3295
|
#%% ------------ Stringfy Block ----------------
|
|
2476
|
-
|
|
3296
|
+
|
|
2477
3297
|
def formHostStr(host) -> str:
|
|
2478
3298
|
"""
|
|
2479
3299
|
Forms a comma-separated string of hosts.
|
|
@@ -2493,8 +3313,7 @@ def formHostStr(host) -> str:
|
|
|
2493
3313
|
if 'local_shell' in host:
|
|
2494
3314
|
host.remove('local_shell')
|
|
2495
3315
|
host.add('localhost')
|
|
2496
|
-
|
|
2497
|
-
return host
|
|
3316
|
+
return ','.join(compact_hostnames(host))
|
|
2498
3317
|
|
|
2499
3318
|
@cache_decorator
|
|
2500
3319
|
def __formCommandArgStr(oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
@@ -2504,37 +3323,66 @@ def __formCommandArgStr(oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT
|
|
|
2504
3323
|
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,skip_hosts = DEFAULT_SKIP_HOSTS,
|
|
2505
3324
|
file_sync = False, error_only = DEFAULT_ERROR_ONLY, identity_file = DEFAULT_IDENTITY_FILE,
|
|
2506
3325
|
copy_id = False, unavailable_host_expiry = DEFAULT_UNAVAILABLE_HOST_EXPIRY, no_history = DEFAULT_NO_HISTORY,
|
|
2507
|
-
history_file = DEFAULT_HISTORY_FILE, env_file =
|
|
3326
|
+
history_file = DEFAULT_HISTORY_FILE, env_file = '', env_files = DEFAULT_ENV_FILES,
|
|
2508
3327
|
repeat = DEFAULT_REPEAT,interval = DEFAULT_INTERVAL,
|
|
2509
3328
|
shortend = False) -> str:
|
|
2510
3329
|
argsList = []
|
|
2511
|
-
if oneonone:
|
|
2512
|
-
|
|
2513
|
-
if
|
|
2514
|
-
|
|
2515
|
-
if
|
|
2516
|
-
|
|
2517
|
-
if
|
|
2518
|
-
|
|
2519
|
-
if
|
|
2520
|
-
|
|
2521
|
-
if
|
|
2522
|
-
|
|
2523
|
-
if
|
|
2524
|
-
|
|
2525
|
-
if
|
|
2526
|
-
|
|
2527
|
-
if
|
|
2528
|
-
|
|
2529
|
-
if
|
|
2530
|
-
|
|
2531
|
-
if
|
|
2532
|
-
|
|
2533
|
-
if
|
|
2534
|
-
|
|
2535
|
-
if
|
|
2536
|
-
|
|
2537
|
-
if
|
|
3330
|
+
if oneonone:
|
|
3331
|
+
argsList.append('--oneonone' if not shortend else '-11')
|
|
3332
|
+
if timeout and timeout != DEFAULT_TIMEOUT:
|
|
3333
|
+
argsList.append(f'--timeout={timeout}' if not shortend else f'-t={timeout}')
|
|
3334
|
+
if repeat and repeat != DEFAULT_REPEAT:
|
|
3335
|
+
argsList.append(f'--repeat={repeat}' if not shortend else f'-r={repeat}')
|
|
3336
|
+
if interval and interval != DEFAULT_INTERVAL:
|
|
3337
|
+
argsList.append(f'--interval={interval}' if not shortend else f'-i={interval}')
|
|
3338
|
+
if password and password != DEFAULT_PASSWORD:
|
|
3339
|
+
argsList.append(f'--password="{password}"' if not shortend else f'-p="{password}"')
|
|
3340
|
+
if identity_file and identity_file != DEFAULT_IDENTITY_FILE:
|
|
3341
|
+
argsList.append(f'--key="{identity_file}"' if not shortend else f'-k="{identity_file}"')
|
|
3342
|
+
if copy_id:
|
|
3343
|
+
argsList.append('--copy_id' if not shortend else '-ci')
|
|
3344
|
+
if no_watch:
|
|
3345
|
+
argsList.append('--no_watch' if not shortend else '-q')
|
|
3346
|
+
if json:
|
|
3347
|
+
argsList.append('--json' if not shortend else '-j')
|
|
3348
|
+
if max_connections and max_connections != DEFAULT_MAX_CONNECTIONS:
|
|
3349
|
+
argsList.append(f'--max_connections={max_connections}' if not shortend else f'-m={max_connections}')
|
|
3350
|
+
if files:
|
|
3351
|
+
argsList.extend([f'--file="{file}"' for file in files] if not shortend else [f'-f="{file}"' for file in files])
|
|
3352
|
+
if ipmi:
|
|
3353
|
+
argsList.append('--ipmi')
|
|
3354
|
+
if interface_ip_prefix and interface_ip_prefix != DEFAULT_INTERFACE_IP_PREFIX:
|
|
3355
|
+
argsList.append(f'--interface_ip_prefix="{interface_ip_prefix}"' if not shortend else f'-pre="{interface_ip_prefix}"')
|
|
3356
|
+
if scp:
|
|
3357
|
+
argsList.append('--scp')
|
|
3358
|
+
if gather_mode:
|
|
3359
|
+
argsList.append('--gather_mode' if not shortend else '-gm')
|
|
3360
|
+
if username and username != DEFAULT_USERNAME:
|
|
3361
|
+
argsList.append(f'--username="{username}"' if not shortend else f'-u="{username}"')
|
|
3362
|
+
if extraargs and extraargs != DEFAULT_EXTRA_ARGS:
|
|
3363
|
+
argsList.append(f'--extraargs="{extraargs}"' if not shortend else f'-ea="{extraargs}"')
|
|
3364
|
+
if skipUnreachable:
|
|
3365
|
+
argsList.append('--skip_unreachable' if not shortend else '-su')
|
|
3366
|
+
if unavailable_host_expiry and unavailable_host_expiry != DEFAULT_UNAVAILABLE_HOST_EXPIRY:
|
|
3367
|
+
argsList.append(f'--unavailable_host_expiry={unavailable_host_expiry}' if not shortend else f'-uhe={unavailable_host_expiry}')
|
|
3368
|
+
if no_env:
|
|
3369
|
+
argsList.append('--no_env')
|
|
3370
|
+
if env_file:
|
|
3371
|
+
argsList.append(f'--env_file="{env_file}"' if not shortend else f'-ef="{env_file}"')
|
|
3372
|
+
if env_files:
|
|
3373
|
+
argsList.extend([f'--env_files="{ef}"' for ef in env_files] if not shortend else [f'-efs="{ef}"' for ef in env_files])
|
|
3374
|
+
if no_history:
|
|
3375
|
+
argsList.append('--no_history' if not shortend else '-nh')
|
|
3376
|
+
if history_file and history_file != DEFAULT_HISTORY_FILE:
|
|
3377
|
+
argsList.append(f'--history_file="{history_file}"' if not shortend else f'-hf="{history_file}"')
|
|
3378
|
+
if greppable:
|
|
3379
|
+
argsList.append('--greppable' if not shortend else '-g')
|
|
3380
|
+
if error_only:
|
|
3381
|
+
argsList.append('--error_only' if not shortend else '-eo')
|
|
3382
|
+
if skip_hosts and skip_hosts != DEFAULT_SKIP_HOSTS:
|
|
3383
|
+
argsList.append(f'--skip_hosts="{skip_hosts}"' if not shortend else f'-sh="{skip_hosts}"')
|
|
3384
|
+
if file_sync:
|
|
3385
|
+
argsList.append('--file_sync' if not shortend else '-fs')
|
|
2538
3386
|
return ' '.join(argsList)
|
|
2539
3387
|
|
|
2540
3388
|
def getStrCommand(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
@@ -2542,18 +3390,18 @@ def getStrCommand(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAULT_ONE_O
|
|
|
2542
3390
|
files = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = _DEFAULT_RETURN_UNFINISHED,
|
|
2543
3391
|
scp=DEFAULT_SCP,gather_mode = False,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
|
|
2544
3392
|
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=_DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=_DEFAULT_NO_START,
|
|
2545
|
-
skip_hosts = DEFAULT_SKIP_HOSTS,
|
|
3393
|
+
skip_hosts = DEFAULT_SKIP_HOSTS, window_width = DEFAULT_WINDOW_WIDTH, window_height = DEFAULT_WINDOW_HEIGHT,
|
|
2546
3394
|
single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY, identity_file = DEFAULT_IDENTITY_FILE,
|
|
2547
3395
|
copy_id = False, unavailable_host_expiry = DEFAULT_UNAVAILABLE_HOST_EXPIRY,no_history = DEFAULT_NO_HISTORY,
|
|
2548
|
-
history_file = DEFAULT_HISTORY_FILE, env_file =
|
|
3396
|
+
history_file = DEFAULT_HISTORY_FILE, env_file = '', env_files = [],
|
|
2549
3397
|
repeat = DEFAULT_REPEAT,interval = DEFAULT_INTERVAL,
|
|
2550
|
-
shortend = False,tabSeperated = False):
|
|
3398
|
+
shortend = False,tabSeperated = False,**kwargs) -> str:
|
|
2551
3399
|
_ = called
|
|
2552
3400
|
_ = returnUnfinished
|
|
2553
3401
|
_ = willUpdateUnreachableHosts
|
|
2554
3402
|
_ = no_start
|
|
2555
|
-
_ =
|
|
2556
|
-
_ =
|
|
3403
|
+
_ = window_width
|
|
3404
|
+
_ = window_height
|
|
2557
3405
|
_ = single_window
|
|
2558
3406
|
hosts = hosts if isinstance(hosts,str) else frozenset(hosts)
|
|
2559
3407
|
hostStr = formHostStr(hosts)
|
|
@@ -2565,10 +3413,10 @@ def getStrCommand(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAULT_ONE_O
|
|
|
2565
3413
|
no_env=no_env, greppable=greppable,skip_hosts = skip_hosts,
|
|
2566
3414
|
file_sync = file_sync,error_only = error_only, identity_file = identity_file,
|
|
2567
3415
|
copy_id = copy_id, unavailable_host_expiry =unavailable_host_expiry,no_history = no_history,
|
|
2568
|
-
history_file = history_file, env_file = env_file,
|
|
3416
|
+
history_file = history_file, env_file = env_file, env_files = env_files,
|
|
2569
3417
|
repeat = repeat,interval = interval,
|
|
2570
3418
|
shortend = shortend)
|
|
2571
|
-
commands = [command.replace('"', '\\"').replace('\n', '\\n').replace('\t', '\\t') for command in commands]
|
|
3419
|
+
commands = [command.replace('"', '\\"').replace('\n', '\\n').replace('\t', '\\t') for command in format_commands(commands)]
|
|
2572
3420
|
commandStr = '"' + '" "'.join(commands) + '"' if commands else ''
|
|
2573
3421
|
filePath = os.path.abspath(__file__)
|
|
2574
3422
|
programName = filePath if filePath else 'mssh'
|
|
@@ -2616,56 +3464,56 @@ def record_command_history(kwargs):
|
|
|
2616
3464
|
#%% ------------ Main Block ----------------
|
|
2617
3465
|
def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAULT_ONE_ON_ONE, timeout = DEFAULT_TIMEOUT,password = DEFAULT_PASSWORD,
|
|
2618
3466
|
no_watch = DEFAULT_NO_WATCH,json = DEFAULT_JSON_MODE,called = _DEFAULT_CALLED,max_connections=DEFAULT_MAX_CONNECTIONS,
|
|
2619
|
-
|
|
3467
|
+
file = None,ipmi = DEFAULT_IPMI,interface_ip_prefix = DEFAULT_INTERFACE_IP_PREFIX,returnUnfinished = _DEFAULT_RETURN_UNFINISHED,
|
|
2620
3468
|
scp=DEFAULT_SCP,gather_mode = False,username=DEFAULT_USERNAME,extraargs=DEFAULT_EXTRA_ARGS,skipUnreachable=DEFAULT_SKIP_UNREACHABLE,
|
|
2621
3469
|
no_env=DEFAULT_NO_ENV,greppable=DEFAULT_GREPPABLE_MODE,willUpdateUnreachableHosts=_DEFAULT_UPDATE_UNREACHABLE_HOSTS,no_start=_DEFAULT_NO_START,
|
|
2622
|
-
skip_hosts = DEFAULT_SKIP_HOSTS,
|
|
3470
|
+
skip_hosts = DEFAULT_SKIP_HOSTS, window_width = DEFAULT_WINDOW_WIDTH, window_height = DEFAULT_WINDOW_HEIGHT,
|
|
2623
3471
|
single_window = DEFAULT_SINGLE_WINDOW,file_sync = False,error_only = DEFAULT_ERROR_ONLY,quiet = False,identity_file = DEFAULT_IDENTITY_FILE,
|
|
2624
3472
|
copy_id = False, unavailable_host_expiry = DEFAULT_UNAVAILABLE_HOST_EXPIRY,no_history = True,
|
|
2625
|
-
history_file = DEFAULT_HISTORY_FILE
|
|
3473
|
+
history_file = DEFAULT_HISTORY_FILE,**kwargs
|
|
2626
3474
|
):
|
|
2627
|
-
|
|
2628
|
-
Run
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
hosts (str
|
|
2632
|
-
commands (list):
|
|
2633
|
-
oneonone (bool
|
|
2634
|
-
timeout (int
|
|
2635
|
-
password (str
|
|
2636
|
-
no_watch (bool
|
|
2637
|
-
json (bool
|
|
2638
|
-
called (bool
|
|
2639
|
-
max_connections (int
|
|
2640
|
-
|
|
2641
|
-
ipmi (bool
|
|
2642
|
-
interface_ip_prefix (str
|
|
2643
|
-
returnUnfinished (bool
|
|
2644
|
-
scp (bool
|
|
2645
|
-
gather_mode (bool
|
|
2646
|
-
username (str
|
|
2647
|
-
extraargs (str
|
|
2648
|
-
skipUnreachable (bool
|
|
2649
|
-
no_env (bool
|
|
2650
|
-
greppable (bool
|
|
2651
|
-
willUpdateUnreachableHosts (bool
|
|
2652
|
-
no_start (bool
|
|
2653
|
-
skip_hosts (str
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
single_window (bool
|
|
2657
|
-
file_sync (bool
|
|
2658
|
-
error_only (bool
|
|
2659
|
-
quiet (bool
|
|
2660
|
-
identity_file (str
|
|
2661
|
-
copy_id (bool
|
|
2662
|
-
unavailable_host_expiry (int
|
|
2663
|
-
no_history (bool
|
|
2664
|
-
history_file (str
|
|
3475
|
+
"""
|
|
3476
|
+
Run commands on multiple hosts via SSH or IPMI.
|
|
3477
|
+
|
|
3478
|
+
Parameters:
|
|
3479
|
+
hosts (str or iterable): Hosts to run the command on. Can be a string (comma/space-separated) or iterable. Default: DEFAULT_HOSTS.
|
|
3480
|
+
commands (list or None): List of commands to run on the hosts. If files are used, defines the destination. Default: None.
|
|
3481
|
+
oneonone (bool): If True, run each command on the corresponding host (1:1 mapping). Default: DEFAULT_ONE_ON_ONE.
|
|
3482
|
+
timeout (int): Timeout for each command in seconds. Default: DEFAULT_TIMEOUT.
|
|
3483
|
+
password (str): Password for SSH/IPMI authentication. Default: DEFAULT_PASSWORD.
|
|
3484
|
+
no_watch (bool): If True, do not use curses TUI; just print output. Default: DEFAULT_NO_WATCH.
|
|
3485
|
+
json (bool): If True, output results in JSON format. Default: DEFAULT_JSON_MODE.
|
|
3486
|
+
called (bool): If True, function is called programmatically (not CLI). Default: _DEFAULT_CALLED.
|
|
3487
|
+
max_connections (int): Maximum concurrent SSH sessions. Default: 4 * os.cpu_count().
|
|
3488
|
+
file (list or None): Files to copy to hosts. Default: None.
|
|
3489
|
+
ipmi (bool): Use IPMI instead of SSH. Default: DEFAULT_IPMI.
|
|
3490
|
+
interface_ip_prefix (str or None): Override IP prefix for host connection. Default: DEFAULT_INTERFACE_IP_PREFIX.
|
|
3491
|
+
returnUnfinished (bool): If True, return hosts even if not finished. Default: _DEFAULT_RETURN_UNFINISHED.
|
|
3492
|
+
scp (bool): Use scp for file transfer (instead of rsync). Default: DEFAULT_SCP.
|
|
3493
|
+
gather_mode (bool): Gather files from hosts (pull mode). Default: False.
|
|
3494
|
+
username (str or None): Username for SSH/IPMI. Default: DEFAULT_USERNAME.
|
|
3495
|
+
extraargs (str or list or None): Extra args for SSH/SCP/rsync. Default: DEFAULT_EXTRA_ARGS.
|
|
3496
|
+
skipUnreachable (bool): Skip hosts marked as unreachable. Default: DEFAULT_SKIP_UNREACHABLE.
|
|
3497
|
+
no_env (bool): Do not load environment variables from shell. Default: DEFAULT_NO_ENV.
|
|
3498
|
+
greppable (bool): Output in greppable table format. Default: DEFAULT_GREPPABLE_MODE.
|
|
3499
|
+
willUpdateUnreachableHosts (bool): Update global unreachable hosts file. Default: _DEFAULT_UPDATE_UNREACHABLE_HOSTS.
|
|
3500
|
+
no_start (bool): If True, return Host objects without running commands. Default: _DEFAULT_NO_START.
|
|
3501
|
+
skip_hosts (str or None): Hosts to skip. Default: DEFAULT_SKIP_HOSTS.
|
|
3502
|
+
window_width (int): Minimum width per curses window. Default: DEFAULT_WINDOW_WIDTH.
|
|
3503
|
+
window_height (int): Minimum height per curses window. Default: DEFAULT_WINDOW_HEIGHT.
|
|
3504
|
+
single_window (bool): Use a single curses window for all hosts. Default: DEFAULT_SINGLE_WINDOW.
|
|
3505
|
+
file_sync (bool): Enable file sync mode (sync directories). Default: DEFAULT_FILE_SYNC.
|
|
3506
|
+
error_only (bool): Only print error output. Default: DEFAULT_ERROR_ONLY.
|
|
3507
|
+
quiet (bool): Suppress all output (overrides other output options). Default: False.
|
|
3508
|
+
identity_file (str or None): SSH identity file. Default: DEFAULT_IDENTITY_FILE.
|
|
3509
|
+
copy_id (bool): Use ssh-copy-id to copy public key to hosts. Default: False.
|
|
3510
|
+
unavailable_host_expiry (int): Seconds to keep hosts marked as unavailable. Default: DEFAULT_UNAVAILABLE_HOST_EXPIRY.
|
|
3511
|
+
no_history (bool): Do not record command history. Default: True.
|
|
3512
|
+
history_file (str): File to store command history. Default: DEFAULT_HISTORY_FILE.
|
|
2665
3513
|
|
|
2666
3514
|
Returns:
|
|
2667
|
-
list:
|
|
2668
|
-
|
|
3515
|
+
list: List of Host objects representing each host/command run.
|
|
3516
|
+
"""
|
|
2669
3517
|
global __globalUnavailableHosts
|
|
2670
3518
|
global __global_suppress_printout
|
|
2671
3519
|
global _no_env
|
|
@@ -2694,7 +3542,7 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2694
3542
|
if line and ',' in line and len(line.split(',')) >= 2 and line.split(',')[0] and line.split(',')[1].isdigit():
|
|
2695
3543
|
hostname = line.split(',')[0]
|
|
2696
3544
|
expireTime = int(line.split(',')[1])
|
|
2697
|
-
if expireTime
|
|
3545
|
+
if expireTime > time.monotonic():
|
|
2698
3546
|
__globalUnavailableHosts[hostname] = expireTime
|
|
2699
3547
|
readed = True
|
|
2700
3548
|
if readed and not __global_suppress_printout:
|
|
@@ -2703,7 +3551,7 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2703
3551
|
eprint(f"Warning: Unable to read the unavailable hosts from the file {os.path.join(tempfile.gettempdir(),f'__{getpass.getuser()}_multiSSH3_UNAVAILABLE_HOSTS.csv')!r}")
|
|
2704
3552
|
eprint(str(e))
|
|
2705
3553
|
elif '__multiSSH3_UNAVAILABLE_HOSTS' in readEnvFromFile():
|
|
2706
|
-
__globalUnavailableHosts.update({host: int(time.monotonic()) for host in readEnvFromFile()['__multiSSH3_UNAVAILABLE_HOSTS'].split(',') if host})
|
|
3554
|
+
__globalUnavailableHosts.update({host: int(time.monotonic()+ unavailable_host_expiry) for host in readEnvFromFile()['__multiSSH3_UNAVAILABLE_HOSTS'].split(',') if host})
|
|
2707
3555
|
if not max_connections:
|
|
2708
3556
|
max_connections = 4 * os.cpu_count()
|
|
2709
3557
|
elif max_connections == 0:
|
|
@@ -2717,15 +3565,7 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2717
3565
|
if max_connections > __max_connections_nofile_limit_supported * 2:
|
|
2718
3566
|
# we need to throttle thread start to avoid hitting the nofile limit
|
|
2719
3567
|
__thread_start_delay = 0.001
|
|
2720
|
-
|
|
2721
|
-
commands = []
|
|
2722
|
-
else:
|
|
2723
|
-
commands = [commands] if isinstance(commands,str) else commands
|
|
2724
|
-
# reformat commands into a list of strings, join the iterables if they are not strings
|
|
2725
|
-
try:
|
|
2726
|
-
commands = [' '.join(command) if not isinstance(command,str) else command for command in commands]
|
|
2727
|
-
except:
|
|
2728
|
-
eprint(f"Warning: commands should ideally be a list of strings. Now mssh had failed to convert {commands!r} to a list of strings. Continuing anyway but expect failures.")
|
|
3568
|
+
commands = format_commands(commands)
|
|
2729
3569
|
#verify_ssh_config()
|
|
2730
3570
|
# load global unavailable hosts only if the function is called (so using --repeat will not load the unavailable hosts again)
|
|
2731
3571
|
if called:
|
|
@@ -2777,13 +3617,14 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2777
3617
|
skipHostsDic = expand_hostnames(skipHostStr.split(','))
|
|
2778
3618
|
skipHostSet = set(skipHostsDic).union(skipHostsDic.values())
|
|
2779
3619
|
if skipHostSet:
|
|
2780
|
-
eprint(f"Skipping hosts: \"{' '.join(
|
|
3620
|
+
eprint(f"Skipping hosts: \"{' '.join(compact_hostnames(skipHostSet))}\"")
|
|
2781
3621
|
if copy_id:
|
|
2782
3622
|
if 'ssh-copy-id' in _binPaths:
|
|
2783
3623
|
# we will copy the id to the hosts
|
|
2784
3624
|
hosts = []
|
|
2785
3625
|
for host in targetHostDic:
|
|
2786
|
-
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
3626
|
+
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
3627
|
+
continue
|
|
2787
3628
|
command = f"{_binPaths['ssh-copy-id']} "
|
|
2788
3629
|
if identity_file:
|
|
2789
3630
|
command = f"{command}-i {identity_file} "
|
|
@@ -2800,39 +3641,39 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2800
3641
|
processRunOnHosts(timeout=timeout, password=password, max_connections=max_connections, hosts=hosts,
|
|
2801
3642
|
returnUnfinished=returnUnfinished, no_watch=no_watch, json=json, called=called, greppable=greppable,
|
|
2802
3643
|
unavailableHosts=unavailableHosts,willUpdateUnreachableHosts=willUpdateUnreachableHosts,
|
|
2803
|
-
|
|
3644
|
+
window_width = window_width, window_height = window_height,
|
|
2804
3645
|
single_window=single_window,unavailable_host_expiry=unavailable_host_expiry)
|
|
2805
3646
|
else:
|
|
2806
3647
|
eprint(f"Warning: ssh-copy-id not found in {_binPaths} , skipping copy id to the hosts")
|
|
2807
3648
|
if not commands:
|
|
2808
3649
|
_exit_with_code(0, "Copy id finished, no commands to run")
|
|
2809
|
-
if
|
|
3650
|
+
if file and not commands:
|
|
2810
3651
|
# if files are specified but not target dir, we default to file sync mode
|
|
2811
3652
|
file_sync = True
|
|
2812
3653
|
if file_sync:
|
|
2813
3654
|
# set the files to the union of files and commands
|
|
2814
|
-
|
|
2815
|
-
if
|
|
3655
|
+
file = set(file+commands) if file else set(commands)
|
|
3656
|
+
if file:
|
|
2816
3657
|
# try to resolve files first (like * etc)
|
|
2817
3658
|
if not gather_mode:
|
|
2818
3659
|
pathSet = set()
|
|
2819
|
-
for file in
|
|
3660
|
+
for file in file:
|
|
2820
3661
|
try:
|
|
2821
3662
|
pathSet.update(glob.glob(file,include_hidden=True,recursive=True))
|
|
2822
|
-
except:
|
|
3663
|
+
except Exception:
|
|
2823
3664
|
pathSet.update(glob.glob(file,recursive=True))
|
|
2824
3665
|
if not pathSet:
|
|
2825
|
-
_exit_with_code(66, f'No source files at {
|
|
3666
|
+
_exit_with_code(66, f'No source files at {file!r} are found after resolving globs!')
|
|
2826
3667
|
else:
|
|
2827
|
-
pathSet = set(
|
|
3668
|
+
pathSet = set(file)
|
|
2828
3669
|
if file_sync:
|
|
2829
3670
|
# use abosolute path for file sync
|
|
2830
3671
|
commands = [os.path.abspath(file) for file in pathSet]
|
|
2831
|
-
|
|
3672
|
+
file = []
|
|
2832
3673
|
else:
|
|
2833
|
-
|
|
3674
|
+
file = list(pathSet)
|
|
2834
3675
|
if __DEBUG_MODE:
|
|
2835
|
-
eprint(f"Files: {
|
|
3676
|
+
eprint(f"Files: {file!r}")
|
|
2836
3677
|
if oneonone:
|
|
2837
3678
|
hosts = []
|
|
2838
3679
|
if len(commands) != len(set(targetHostDic) - set(skipHostSet)):
|
|
@@ -2844,22 +3685,24 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2844
3685
|
eprint('-'*80)
|
|
2845
3686
|
eprint("Running in one on one mode")
|
|
2846
3687
|
for host, command in zip(targetHostDic, commands):
|
|
2847
|
-
if not ipmi and skipUnreachable and host in unavailableHosts:
|
|
3688
|
+
if not ipmi and skipUnreachable and host in unavailableHosts and unavailableHosts[host] > time.monotonic():
|
|
2848
3689
|
eprint(f"Skipping unavailable host: {host}")
|
|
2849
3690
|
continue
|
|
2850
|
-
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
3691
|
+
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
3692
|
+
continue
|
|
2851
3693
|
if file_sync:
|
|
2852
3694
|
hosts.append(Host(host, os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs,gatherMode=gather_mode,identity_file=identity_file,ip = targetHostDic[host]))
|
|
2853
3695
|
else:
|
|
2854
|
-
hosts.append(Host(host, command, files =
|
|
3696
|
+
hosts.append(Host(host, command, files = file,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs,gatherMode=gather_mode,identity_file=identity_file,ip=targetHostDic[host]))
|
|
2855
3697
|
if not __global_suppress_printout:
|
|
2856
3698
|
eprint(f"Running command: {command!r} on host: {host!r}")
|
|
2857
|
-
if not __global_suppress_printout:
|
|
3699
|
+
if not __global_suppress_printout:
|
|
3700
|
+
eprint('-'*80)
|
|
2858
3701
|
if not no_start:
|
|
2859
3702
|
processRunOnHosts(timeout=timeout, password=password, max_connections=max_connections, hosts=hosts,
|
|
2860
3703
|
returnUnfinished=returnUnfinished, no_watch=no_watch, json=json, called=called, greppable=greppable,
|
|
2861
3704
|
unavailableHosts=unavailableHosts,willUpdateUnreachableHosts=willUpdateUnreachableHosts,
|
|
2862
|
-
|
|
3705
|
+
window_width = window_width, window_height = window_height,
|
|
2863
3706
|
single_window=single_window,unavailable_host_expiry=unavailable_host_expiry)
|
|
2864
3707
|
return hosts
|
|
2865
3708
|
else:
|
|
@@ -2868,41 +3711,45 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2868
3711
|
# run in interactive mode ssh mode
|
|
2869
3712
|
hosts = []
|
|
2870
3713
|
for host in targetHostDic:
|
|
2871
|
-
if not ipmi and skipUnreachable and host in unavailableHosts:
|
|
2872
|
-
if not __global_suppress_printout:
|
|
3714
|
+
if not ipmi and skipUnreachable and host in unavailableHosts and unavailableHosts[host] > time.monotonic():
|
|
3715
|
+
if not __global_suppress_printout:
|
|
3716
|
+
print(f"Skipping unavailable host: {host}")
|
|
3717
|
+
continue
|
|
3718
|
+
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
2873
3719
|
continue
|
|
2874
|
-
if host in skipHostSet or targetHostDic[host] in skipHostSet: continue
|
|
2875
3720
|
if file_sync:
|
|
2876
|
-
eprint(
|
|
3721
|
+
eprint("Error: file sync mode need to be specified with at least one path to sync.")
|
|
2877
3722
|
return []
|
|
2878
|
-
elif
|
|
2879
|
-
eprint(
|
|
3723
|
+
elif file:
|
|
3724
|
+
eprint("Error: files need to be specified with at least one path to sync")
|
|
2880
3725
|
else:
|
|
2881
|
-
hosts.append(Host(host, '', files =
|
|
3726
|
+
hosts.append(Host(host, '', files = file,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs,identity_file=identity_file,ip=targetHostDic[host]))
|
|
2882
3727
|
if not __global_suppress_printout:
|
|
2883
3728
|
eprint('-'*80)
|
|
2884
3729
|
eprint(f"Running in interactive mode on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
|
|
2885
3730
|
eprint('-'*80)
|
|
2886
3731
|
if no_start:
|
|
2887
|
-
eprint(
|
|
3732
|
+
eprint("Warning: no_start is set, the command will not be started. As we are in interactive mode, no action will be done.")
|
|
2888
3733
|
else:
|
|
2889
3734
|
processRunOnHosts(timeout=timeout, password=password, max_connections=max_connections, hosts=hosts,
|
|
2890
3735
|
returnUnfinished=returnUnfinished, no_watch=no_watch, json=json, called=called, greppable=greppable,
|
|
2891
3736
|
unavailableHosts=unavailableHosts,willUpdateUnreachableHosts=willUpdateUnreachableHosts,
|
|
2892
|
-
|
|
3737
|
+
window_width = window_width, window_height = window_height,
|
|
2893
3738
|
single_window=single_window,unavailable_host_expiry=unavailable_host_expiry)
|
|
2894
3739
|
return hosts
|
|
2895
3740
|
for command in commands:
|
|
2896
3741
|
hosts = []
|
|
2897
3742
|
for host in targetHostDic:
|
|
2898
|
-
if not ipmi and skipUnreachable and host in unavailableHosts:
|
|
2899
|
-
if not __global_suppress_printout:
|
|
3743
|
+
if not ipmi and skipUnreachable and host in unavailableHosts and unavailableHosts[host] > time.monotonic():
|
|
3744
|
+
if not __global_suppress_printout:
|
|
3745
|
+
print(f"Skipping unavailable host: {host}")
|
|
3746
|
+
continue
|
|
3747
|
+
if host in skipHostSet or targetHostDic[host] in skipHostSet:
|
|
2900
3748
|
continue
|
|
2901
|
-
if host in skipHostSet or targetHostDic[host] in skipHostSet: continue
|
|
2902
3749
|
if file_sync:
|
|
2903
3750
|
hosts.append(Host(host, os.path.dirname(command)+os.path.sep, files = [command],ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs,gatherMode=gather_mode,identity_file=identity_file,ip=targetHostDic[host]))
|
|
2904
3751
|
else:
|
|
2905
|
-
hosts.append(Host(host, command, files =
|
|
3752
|
+
hosts.append(Host(host, command, files = file,ipmi=ipmi,interface_ip_prefix=interface_ip_prefix,scp=scp,extraargs=extraargs,gatherMode=gather_mode,identity_file=identity_file,ip=targetHostDic[host]))
|
|
2906
3753
|
if not __global_suppress_printout and len(commands) > 1:
|
|
2907
3754
|
eprint('-'*80)
|
|
2908
3755
|
eprint(f"Running command: {command} on hosts: {hostStr}" + (f"; skipping: {skipHostStr}" if skipHostStr else ''))
|
|
@@ -2911,7 +3758,7 @@ def run_command_on_hosts(hosts = DEFAULT_HOSTS,commands = None,oneonone = DEFAUL
|
|
|
2911
3758
|
processRunOnHosts(timeout=timeout, password=password, max_connections=max_connections, hosts=hosts,
|
|
2912
3759
|
returnUnfinished=returnUnfinished, no_watch=no_watch, json=json, called=called, greppable=greppable,
|
|
2913
3760
|
unavailableHosts=unavailableHosts,willUpdateUnreachableHosts=willUpdateUnreachableHosts,
|
|
2914
|
-
|
|
3761
|
+
window_width = window_width, window_height = window_height,
|
|
2915
3762
|
single_window=single_window,unavailable_host_expiry=unavailable_host_expiry)
|
|
2916
3763
|
allHosts += hosts
|
|
2917
3764
|
return allHosts
|
|
@@ -2933,8 +3780,8 @@ def generate_default_config(args):
|
|
|
2933
3780
|
'DEFAULT_HOSTS': args.hosts,
|
|
2934
3781
|
'DEFAULT_USERNAME': args.username,
|
|
2935
3782
|
'DEFAULT_PASSWORD': args.password,
|
|
2936
|
-
'DEFAULT_IDENTITY_FILE': args.
|
|
2937
|
-
'
|
|
3783
|
+
'DEFAULT_IDENTITY_FILE': args.identity_file if args.identity_file and not os.path.isdir(args.identity_file) else DEFAULT_IDENTITY_FILE,
|
|
3784
|
+
'DEFAULT_SSH_KEY_SEARCH_PATH': args.identity_file if args.identity_file and os.path.isdir(args.identity_file) else DEFAULT_SSH_KEY_SEARCH_PATH,
|
|
2938
3785
|
'DEFAULT_USE_KEY': args.use_key,
|
|
2939
3786
|
'DEFAULT_EXTRA_ARGS': args.extraargs,
|
|
2940
3787
|
'DEFAULT_ONE_ON_ONE': args.oneonone,
|
|
@@ -2948,15 +3795,17 @@ def generate_default_config(args):
|
|
|
2948
3795
|
'DEFAULT_IPMI': args.ipmi,
|
|
2949
3796
|
'DEFAULT_IPMI_INTERFACE_IP_PREFIX': args.ipmi_interface_ip_prefix,
|
|
2950
3797
|
'DEFAULT_INTERFACE_IP_PREFIX': args.interface_ip_prefix,
|
|
3798
|
+
'DEFAULT_IPMI_USERNAME': args.ipmi_username,
|
|
3799
|
+
'DEFAULT_IPMI_PASSWORD': args.ipmi_password,
|
|
2951
3800
|
'DEFAULT_NO_WATCH': args.no_watch,
|
|
2952
|
-
'
|
|
2953
|
-
'
|
|
3801
|
+
'DEFAULT_WINDOW_WIDTH': args.window_width,
|
|
3802
|
+
'DEFAULT_WINDOW_HEIGHT': args.window_height,
|
|
2954
3803
|
'DEFAULT_SINGLE_WINDOW': args.single_window,
|
|
2955
3804
|
'DEFAULT_ERROR_ONLY': args.error_only,
|
|
2956
3805
|
'DEFAULT_NO_OUTPUT': args.no_output,
|
|
2957
3806
|
'DEFAULT_RETURN_ZERO': args.return_zero,
|
|
2958
3807
|
'DEFAULT_NO_ENV': args.no_env,
|
|
2959
|
-
'
|
|
3808
|
+
'DEFAULT_ENV_FILES': args.env_files,
|
|
2960
3809
|
'DEFAULT_NO_HISTORY': args.no_history,
|
|
2961
3810
|
'DEFAULT_HISTORY_FILE': args.history_file,
|
|
2962
3811
|
'DEFAULT_MAX_CONNECTIONS': args.max_connections if args.max_connections != 4 * os.cpu_count() else None,
|
|
@@ -2966,8 +3815,10 @@ def generate_default_config(args):
|
|
|
2966
3815
|
'DEFAULT_SKIP_UNREACHABLE': args.skip_unreachable,
|
|
2967
3816
|
'DEFAULT_SKIP_HOSTS': args.skip_hosts,
|
|
2968
3817
|
'DEFAULT_ENCODING': args.encoding,
|
|
3818
|
+
'DEFAULT_DIFF_DISPLAY_THRESHOLD': args.diff_display_threshold,
|
|
2969
3819
|
'SSH_STRICT_HOST_KEY_CHECKING': SSH_STRICT_HOST_KEY_CHECKING,
|
|
2970
3820
|
'ERROR_MESSAGES_TO_IGNORE': ERROR_MESSAGES_TO_IGNORE,
|
|
3821
|
+
'FORCE_TRUECOLOR': args.force_truecolor,
|
|
2971
3822
|
}
|
|
2972
3823
|
|
|
2973
3824
|
def write_default_config(args,CONFIG_FILE = None):
|
|
@@ -2980,9 +3831,9 @@ def write_default_config(args,CONFIG_FILE = None):
|
|
|
2980
3831
|
backup = True
|
|
2981
3832
|
if os.path.exists(CONFIG_FILE):
|
|
2982
3833
|
eprint(f"Warning: {CONFIG_FILE!r} already exists, what to do? (o/b/n)")
|
|
2983
|
-
eprint(
|
|
3834
|
+
eprint("o: Overwrite the file")
|
|
2984
3835
|
eprint(f"b: Rename the current config file at {CONFIG_FILE!r}.bak forcefully and write the new config file (default)")
|
|
2985
|
-
eprint(
|
|
3836
|
+
eprint("n: Do nothing")
|
|
2986
3837
|
inStr = input_with_timeout_and_countdown(10)
|
|
2987
3838
|
if (not inStr) or inStr.lower().strip().startswith('b'):
|
|
2988
3839
|
backup = True
|
|
@@ -3005,26 +3856,26 @@ def write_default_config(args,CONFIG_FILE = None):
|
|
|
3005
3856
|
eprint(f"Config file written to {CONFIG_FILE!r}")
|
|
3006
3857
|
except Exception as e:
|
|
3007
3858
|
eprint(f"Error: Unable to write to the config file: {e!r}")
|
|
3008
|
-
eprint(
|
|
3859
|
+
eprint('Printing the config file to stdout:')
|
|
3009
3860
|
print(json.dumps(__configs_from_file, indent=4))
|
|
3010
3861
|
|
|
3011
3862
|
#%% ------------ Argument Processing -----------------
|
|
3012
3863
|
def get_parser():
|
|
3013
3864
|
global _binPaths
|
|
3014
|
-
parser = argparse.ArgumentParser(description=
|
|
3015
|
-
epilog=f'Found bins: {list(_binPaths.values())}\n Missing bins: {_binCalled - set(_binPaths.keys())}')
|
|
3865
|
+
parser = argparse.ArgumentParser(description='Run a command on multiple hosts, Use #HOST# or #HOSTNAME# to replace the host name in the command.',
|
|
3866
|
+
epilog=f'Found bins: {list(_binPaths.values())}\n Missing bins: {_binCalled - set(_binPaths.keys())}\n Terminal color capability: {get_terminal_color_capability()}\nConfig file chain: {CONFIG_FILE_CHAIN!r}',)
|
|
3016
3867
|
parser.add_argument('hosts', metavar='hosts', type=str, nargs='?', help=f'Hosts to run the command on, use "," to seperate hosts. (default: {DEFAULT_HOSTS})',default=DEFAULT_HOSTS)
|
|
3017
3868
|
parser.add_argument('commands', metavar='commands', type=str, nargs='*',default=None,help='the command to run on the hosts / the destination of the files #HOST# or #HOSTNAME# will be replaced with the host name.')
|
|
3018
3869
|
parser.add_argument('-u','--username', type=str,help=f'The general username to use to connect to the hosts. Will get overwrote by individual username@host if specified. (default: {DEFAULT_USERNAME})',default=DEFAULT_USERNAME)
|
|
3019
3870
|
parser.add_argument('-p', '--password', type=str,help=f'The password to use to connect to the hosts, (default: {DEFAULT_PASSWORD})',default=DEFAULT_PASSWORD)
|
|
3020
|
-
parser.add_argument('-k','--key','--identity',nargs='?', type=str,help=f'The identity file to use to connect to the hosts. Implies --use_key. Specify a folder for program to search for a key. Use option without value to use {
|
|
3871
|
+
parser.add_argument('-k','--identity_file','--key','--identity',nargs='?', type=str,help=f'The identity file to use to connect to the hosts. Implies --use_key. Specify a folder for program to search for a key. Use option without value to use {DEFAULT_SSH_KEY_SEARCH_PATH} (default: {DEFAULT_IDENTITY_FILE})',const=DEFAULT_SSH_KEY_SEARCH_PATH,default=DEFAULT_IDENTITY_FILE)
|
|
3021
3872
|
parser.add_argument('-uk','--use_key', action='store_true', help=f'Attempt to use public key file to connect to the hosts. (default: {DEFAULT_USE_KEY})', default=DEFAULT_USE_KEY)
|
|
3022
3873
|
parser.add_argument('-ea','--extraargs',type=str,help=f'Extra arguments to pass to the ssh / rsync / scp command. Put in one string for multiple arguments.Use "=" ! Ex. -ea="--delete" (default: {DEFAULT_EXTRA_ARGS})',default=DEFAULT_EXTRA_ARGS)
|
|
3023
3874
|
parser.add_argument("-11",'--oneonone', action='store_true', help=f"Run one corresponding command on each host. (default: {DEFAULT_ONE_ON_ONE})", default=DEFAULT_ONE_ON_ONE)
|
|
3024
3875
|
parser.add_argument("-f","--file", action='append', help="The file to be copied to the hosts. Use -f multiple times to copy multiple files")
|
|
3025
|
-
parser.add_argument('-s','-fs','--file_sync', action='
|
|
3876
|
+
parser.add_argument('-s','-fs','--file_sync',nargs='?', action='append', help=f'Operate in file sync mode, sync path in <COMMANDS> from this machine to <HOSTS>. Treat --file <FILE> and <COMMANDS> both as source and source and destination will be the same in this mode. Infer destination from source path. (default: {DEFAULT_FILE_SYNC})',const=True, default=[DEFAULT_FILE_SYNC])
|
|
3026
3877
|
parser.add_argument('-W','--scp', action='store_true', help=f'Use scp for copying files instead of rsync. Need to use this on windows. (default: {DEFAULT_SCP})', default=DEFAULT_SCP)
|
|
3027
|
-
parser.add_argument('-G','-gm','--gather_mode', action='store_true', help=
|
|
3878
|
+
parser.add_argument('-G','-gm','--gather_mode', action='store_true', help='Gather files from the hosts instead of sending files to the hosts. Will send remote files specified in <FILE> to local path specified in <COMMANDS> (default: False)', default=False)
|
|
3028
3879
|
#parser.add_argument("-d",'-c',"--destination", type=str, help="The destination of the files. Same as specify with commands. Added for compatibility. Use #HOST# or #HOSTNAME# to replace the host name in the destination")
|
|
3029
3880
|
parser.add_argument("-t","--timeout", type=int, help=f"Timeout for each command in seconds. Set default value via DEFAULT_CLI_TIMEOUT in config file. Use 0 for disabling timeout. (default: {DEFAULT_CLI_TIMEOUT})", default=DEFAULT_CLI_TIMEOUT)
|
|
3030
3881
|
parser.add_argument('-T','--use_script_timeout',action='store_true', help=f'Use shortened timeout suitable to use in a script. Set value via DEFAULT_TIMEOUT field in config file. (current: {DEFAULT_TIMEOUT})', default=False)
|
|
@@ -3033,37 +3884,45 @@ def get_parser():
|
|
|
3033
3884
|
parser.add_argument('-M',"--ipmi", action='store_true', help=f"Use ipmitool to run the command. (default: {DEFAULT_IPMI})", default=DEFAULT_IPMI)
|
|
3034
3885
|
parser.add_argument("-mpre","--ipmi_interface_ip_prefix", type=str, help=f"The prefix of the IPMI interfaces (default: {DEFAULT_IPMI_INTERFACE_IP_PREFIX})", default=DEFAULT_IPMI_INTERFACE_IP_PREFIX)
|
|
3035
3886
|
parser.add_argument("-pre","--interface_ip_prefix", type=str, help=f"The prefix of the for the interfaces (default: {DEFAULT_INTERFACE_IP_PREFIX})", default=DEFAULT_INTERFACE_IP_PREFIX)
|
|
3036
|
-
parser.add_argument('-
|
|
3037
|
-
parser.add_argument(
|
|
3038
|
-
parser.add_argument("-
|
|
3887
|
+
parser.add_argument('-iu','--ipmi_username', type=str,help=f'The username to use to connect to the hosts via ipmi. (default: {DEFAULT_IPMI_USERNAME})',default=DEFAULT_IPMI_USERNAME)
|
|
3888
|
+
parser.add_argument('-ip','--ipmi_password', type=str,help=f'The password to use to connect to the hosts via ipmi. (default: {DEFAULT_IPMI_PASSWORD})',default=DEFAULT_IPMI_PASSWORD)
|
|
3889
|
+
parser.add_argument('-S',"-q","-nw","--no_watch", action='store_true', help=f"Quiet mode, no curses watch, only print the output. (default: {DEFAULT_NO_WATCH})", default=DEFAULT_NO_WATCH)
|
|
3890
|
+
parser.add_argument("-ww",'--window_width', type=int, help=f"The minimum character length of the curses window. (default: {DEFAULT_WINDOW_WIDTH})", default=DEFAULT_WINDOW_WIDTH)
|
|
3891
|
+
parser.add_argument("-wh",'--window_height', type=int, help=f"The minimum line height of the curses window. (default: {DEFAULT_WINDOW_HEIGHT})", default=DEFAULT_WINDOW_HEIGHT)
|
|
3039
3892
|
parser.add_argument('-B','-sw','--single_window', action='store_true', help=f'Use a single window for all hosts. (default: {DEFAULT_SINGLE_WINDOW})', default=DEFAULT_SINGLE_WINDOW)
|
|
3040
3893
|
parser.add_argument('-R','-eo','--error_only', action='store_true', help=f'Only print the error output. (default: {DEFAULT_ERROR_ONLY})', default=DEFAULT_ERROR_ONLY)
|
|
3041
|
-
parser.add_argument('-Q',"-no","--no_output", action='store_true', help=f"Do not print the output. (default: {DEFAULT_NO_OUTPUT})", default=DEFAULT_NO_OUTPUT)
|
|
3894
|
+
parser.add_argument('-Q',"-no","--no_output","--quiet", action='store_true', help=f"Do not print the output. (default: {DEFAULT_NO_OUTPUT})", default=DEFAULT_NO_OUTPUT)
|
|
3042
3895
|
parser.add_argument('-Z','-rz','--return_zero', action='store_true', help=f"Return 0 even if there are errors. (default: {DEFAULT_RETURN_ZERO})", default=DEFAULT_RETURN_ZERO)
|
|
3043
3896
|
parser.add_argument('-C','--no_env', action='store_true', help=f'Do not load the command line environment variables. (default: {DEFAULT_NO_ENV})', default=DEFAULT_NO_ENV)
|
|
3044
|
-
parser.add_argument("--env_file", type=str, help=
|
|
3045
|
-
parser.add_argument(
|
|
3897
|
+
parser.add_argument('-ef',"--env_file", type=str, help="Replace the env file look up chain with this env_file. ( Still work with --no_env ) (default: None)", default='')
|
|
3898
|
+
parser.add_argument('-efs',"--env_files", action='append', help=f"The files to load the mssh file based environment variables from. Can specify multiple. Load first to last. ( Still work with --no_env ) (default: {DEFAULT_ENV_FILES})")
|
|
3899
|
+
parser.add_argument("-m","--max_connections", type=int, help="Max number of connections to use (default: 4 * cpu_count)", default=DEFAULT_MAX_CONNECTIONS)
|
|
3046
3900
|
parser.add_argument("-j","--json", action='store_true', help=F"Output in json format. (default: {DEFAULT_JSON_MODE})", default=DEFAULT_JSON_MODE)
|
|
3047
3901
|
parser.add_argument('-w',"--success_hosts", action='store_true', help=f"Output the hosts that succeeded in summary as well. (default: {DEFAULT_PRINT_SUCCESS_HOSTS})", default=DEFAULT_PRINT_SUCCESS_HOSTS)
|
|
3048
3902
|
parser.add_argument('-P',"-g","--greppable",'--table', action='store_true', help=f"Output in greppable table. (default: {DEFAULT_GREPPABLE_MODE})", default=DEFAULT_GREPPABLE_MODE)
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3903
|
+
su_group = parser.add_mutually_exclusive_group()
|
|
3904
|
+
su_group.add_argument('-x',"-su","--skip_unreachable", action='store_true', help=f"Skip unreachable hosts. Note: Timedout Hosts are considered unreachable. Note: multiple command sequence will still auto skip unreachable hosts. (default: {DEFAULT_SKIP_UNREACHABLE})", default=DEFAULT_SKIP_UNREACHABLE)
|
|
3905
|
+
su_group.add_argument('-a',"-nsu","--no_skip_unreachable",dest = 'skip_unreachable', action='store_false', help=f"Do not skip unreachable hosts. Note: Timedout Hosts are considered unreachable. Note: multiple command sequence will still auto skip unreachable hosts. (default: {not DEFAULT_SKIP_UNREACHABLE})", default=not DEFAULT_SKIP_UNREACHABLE)
|
|
3052
3906
|
parser.add_argument('-uhe','--unavailable_host_expiry', type=int, help=f"Time in seconds to expire the unavailable hosts (default: {DEFAULT_UNAVAILABLE_HOST_EXPIRY})", default=DEFAULT_UNAVAILABLE_HOST_EXPIRY)
|
|
3053
3907
|
parser.add_argument('-X',"-sh","--skip_hosts", type=str, help=f"Skip the hosts in the list. (default: {DEFAULT_SKIP_HOSTS if DEFAULT_SKIP_HOSTS else 'None'})", default=DEFAULT_SKIP_HOSTS)
|
|
3054
|
-
parser.add_argument('--generate_config_file', action='store_true', help=
|
|
3055
|
-
parser.add_argument('--config_file', type=str,nargs='?', help=
|
|
3056
|
-
parser.add_argument('--store_config_file',type = str,nargs='?',help=
|
|
3908
|
+
parser.add_argument('--generate_config_file', action='store_true', help='Store / generate the default config file from command line argument and current config at --config_file / stdout')
|
|
3909
|
+
parser.add_argument('--config_file', type=str,nargs='?', help='Additional config file to use, will pioritize over config chains. When using with store_config_file, will store the resulting config file at this location. Use without a path will use multiSSH3.config.json',const='multiSSH3.config.json',default=None)
|
|
3910
|
+
parser.add_argument('--store_config_file',type = str,nargs='?',help='Store the default config file from command line argument and current config. Same as --store_config_file --config_file=<path>',const='multiSSH3.config.json')
|
|
3057
3911
|
parser.add_argument('--debug', action='store_true', help='Print debug information')
|
|
3058
3912
|
parser.add_argument('-ci','--copy_id', action='store_true', help='Copy the ssh id to the hosts')
|
|
3059
3913
|
parser.add_argument('-I','-nh','--no_history', action='store_true', help=f'Do not record the command to history. Default: {DEFAULT_NO_HISTORY}', default=DEFAULT_NO_HISTORY)
|
|
3060
3914
|
parser.add_argument('-hf','--history_file', type=str, help=f'The file to store the history. (default: {DEFAULT_HISTORY_FILE})', default=DEFAULT_HISTORY_FILE)
|
|
3061
3915
|
parser.add_argument('--script', action='store_true', help='Run the command in script mode, short for -SCRIPT or --no_watch --skip_unreachable --no_env --no_history --greppable --error_only')
|
|
3062
3916
|
parser.add_argument('-e','--encoding', type=str, help=f'The encoding to use for the output. (default: {DEFAULT_ENCODING})', default=DEFAULT_ENCODING)
|
|
3917
|
+
parser.add_argument('-dt','--diff_display_threshold', type=float, help=f'The threshold of lines to display the diff when files differ. {{0-1}} Set to 0 to always display the diff. Set to 1 to disable diff. (Only merge same) (default: {DEFAULT_DIFF_DISPLAY_THRESHOLD})', default=DEFAULT_DIFF_DISPLAY_THRESHOLD)
|
|
3918
|
+
parser.add_argument('--force_truecolor', action='store_true', help=f'Force truecolor output even when not in a truecolor terminal. (default: {FORCE_TRUECOLOR})', default=FORCE_TRUECOLOR)
|
|
3919
|
+
parser.add_argument('--add_control_master_config', action='store_true', help='Add ControlMaster configuration to ~/.ssh/config to speed up multiple connections to the same host.')
|
|
3063
3920
|
parser.add_argument("-V","--version", action='version', version=f'%(prog)s {version} @ {COMMIT_DATE} with [ {", ".join(_binPaths.keys())} ] by {AUTHOR} ({AUTHOR_EMAIL})')
|
|
3064
3921
|
return parser
|
|
3065
3922
|
|
|
3066
3923
|
def process_args(args = None):
|
|
3924
|
+
global DEFAULT_IPMI_USERNAME
|
|
3925
|
+
global DEFAULT_IPMI_PASSWORD
|
|
3067
3926
|
parser = get_parser()
|
|
3068
3927
|
# We handle the signal
|
|
3069
3928
|
signal.signal(signal.SIGINT, signal_handler)
|
|
@@ -3087,7 +3946,18 @@ def process_args(args = None):
|
|
|
3087
3946
|
args.no_history = True
|
|
3088
3947
|
args.greppable = True
|
|
3089
3948
|
args.error_only = True
|
|
3090
|
-
|
|
3949
|
+
|
|
3950
|
+
if args.file_sync:
|
|
3951
|
+
for path in args.file_sync:
|
|
3952
|
+
if path and isinstance(path, str):
|
|
3953
|
+
if args.file:
|
|
3954
|
+
if path not in args.file:
|
|
3955
|
+
args.file.append(path)
|
|
3956
|
+
else:
|
|
3957
|
+
args.file = [path]
|
|
3958
|
+
args.file_sync = any(args.file_sync)
|
|
3959
|
+
else:
|
|
3960
|
+
args.file_sync = False
|
|
3091
3961
|
if args.unavailable_host_expiry <= 0:
|
|
3092
3962
|
eprint(f"Warning: The unavailable host expiry time {args.unavailable_host_expiry} is less than 0, setting it to 10 seconds.")
|
|
3093
3963
|
args.unavailable_host_expiry = 10
|
|
@@ -3106,7 +3976,7 @@ def process_config_file(args):
|
|
|
3106
3976
|
else:
|
|
3107
3977
|
configFileToWriteTo = args.config_file
|
|
3108
3978
|
write_default_config(args,configFileToWriteTo)
|
|
3109
|
-
if not args.commands:
|
|
3979
|
+
if not args.commands and not args.file:
|
|
3110
3980
|
if configFileToWriteTo:
|
|
3111
3981
|
with open(configFileToWriteTo,'r') as f:
|
|
3112
3982
|
eprint(f"Config file content: \n{f.read()}")
|
|
@@ -3123,10 +3993,10 @@ def process_config_file(args):
|
|
|
3123
3993
|
|
|
3124
3994
|
def process_commands(args):
|
|
3125
3995
|
if not args.file and len(args.commands) > 1 and all([len(command.split()) == 1 for command in args.commands]):
|
|
3126
|
-
eprint(
|
|
3996
|
+
eprint("Multiple one word command detected, what to do? (1/m/n)")
|
|
3127
3997
|
eprint(f"1: Run 1 command [{' '.join(args.commands)}] on all hosts ( default )")
|
|
3128
3998
|
eprint(f"m: Run multiple commands [{', '.join(args.commands)}] on all hosts")
|
|
3129
|
-
eprint(
|
|
3999
|
+
eprint("n: Exit")
|
|
3130
4000
|
inStr = input_with_timeout_and_countdown(3)
|
|
3131
4001
|
if (not inStr) or inStr.lower().strip().startswith('1'):
|
|
3132
4002
|
args.commands = [" ".join(args.commands)]
|
|
@@ -3138,32 +4008,74 @@ def process_commands(args):
|
|
|
3138
4008
|
return args
|
|
3139
4009
|
|
|
3140
4010
|
def process_keys(args):
|
|
3141
|
-
if args.
|
|
3142
|
-
if not args.
|
|
3143
|
-
args.
|
|
4011
|
+
if args.identity_file or args.use_key:
|
|
4012
|
+
if not args.identity_file:
|
|
4013
|
+
args.identity_file = find_ssh_key_file()
|
|
3144
4014
|
else:
|
|
3145
|
-
if os.path.isdir(os.path.expanduser(args.
|
|
3146
|
-
args.
|
|
3147
|
-
elif not os.path.exists(args.
|
|
3148
|
-
eprint(f"Warning: Identity file {args.
|
|
4015
|
+
if os.path.isdir(os.path.expanduser(args.identity_file)):
|
|
4016
|
+
args.identity_file = find_ssh_key_file(args.identity_file)
|
|
4017
|
+
elif not os.path.exists(args.identity_file):
|
|
4018
|
+
eprint(f"Warning: Identity file {args.identity_file!r} not found. Passing to ssh anyway. Proceed with caution.")
|
|
3149
4019
|
return args
|
|
3150
4020
|
|
|
4021
|
+
def process_control_master_config(args):
|
|
4022
|
+
global __control_master_string
|
|
4023
|
+
if args.add_control_master_config:
|
|
4024
|
+
try:
|
|
4025
|
+
if not os.path.exists(os.path.expanduser('~/.ssh')):
|
|
4026
|
+
os.makedirs(os.path.expanduser('~/.ssh'),mode=0o700)
|
|
4027
|
+
ssh_config_file = os.path.expanduser('~/.ssh/config')
|
|
4028
|
+
if not os.path.exists(ssh_config_file):
|
|
4029
|
+
with open(ssh_config_file,'w') as f:
|
|
4030
|
+
f.write(__control_master_string)
|
|
4031
|
+
os.chmod(ssh_config_file,0o644)
|
|
4032
|
+
else:
|
|
4033
|
+
with open(ssh_config_file,'r') as f:
|
|
4034
|
+
ssh_config_content = f.readlines()
|
|
4035
|
+
if set(map(str.strip,ssh_config_content)).issuperset(set(map(str.strip,__control_master_string.splitlines()))):
|
|
4036
|
+
eprint("ControlMaster configuration already exists in ~/.ssh/config, skipping adding:")
|
|
4037
|
+
eprint(__control_master_string)
|
|
4038
|
+
else:
|
|
4039
|
+
with open(ssh_config_file,'a') as f:
|
|
4040
|
+
f.write('\n# Added by multiSSH3.py to speed up subsequent connections\n')
|
|
4041
|
+
f.write(__control_master_string)
|
|
4042
|
+
eprint("ControlMaster configuration added to ~/.ssh/config.")
|
|
4043
|
+
except Exception as e:
|
|
4044
|
+
eprint(f"Error adding ControlMaster configuration: {e}")
|
|
4045
|
+
import traceback
|
|
4046
|
+
traceback.print_exc()
|
|
4047
|
+
if not args.commands and not args.file:
|
|
4048
|
+
_exit_with_code(0, "Done configuring ControlMaster.")
|
|
4049
|
+
return args
|
|
3151
4050
|
|
|
3152
4051
|
def set_global_with_args(args):
|
|
3153
4052
|
global _emo
|
|
3154
4053
|
global __ipmiiInterfaceIPPrefix
|
|
3155
|
-
global
|
|
4054
|
+
global _env_files
|
|
3156
4055
|
global __DEBUG_MODE
|
|
3157
4056
|
global __configs_from_file
|
|
3158
4057
|
global _encoding
|
|
3159
4058
|
global __returnZero
|
|
4059
|
+
global DEFAULT_IPMI_USERNAME
|
|
4060
|
+
global DEFAULT_IPMI_PASSWORD
|
|
4061
|
+
global DEFAULT_DIFF_DISPLAY_THRESHOLD
|
|
4062
|
+
global FORCE_TRUECOLOR
|
|
3160
4063
|
_emo = False
|
|
3161
4064
|
__ipmiiInterfaceIPPrefix = args.ipmi_interface_ip_prefix
|
|
3162
|
-
|
|
4065
|
+
if args.env_file:
|
|
4066
|
+
_env_files = [args.env_file]
|
|
4067
|
+
else:
|
|
4068
|
+
_env_files = DEFAULT_ENV_FILES + args.env_files if args.env_files else DEFAULT_ENV_FILES
|
|
3163
4069
|
__DEBUG_MODE = args.debug
|
|
3164
4070
|
_encoding = args.encoding
|
|
3165
4071
|
if args.return_zero:
|
|
3166
4072
|
__returnZero = True
|
|
4073
|
+
if args.ipmi_username:
|
|
4074
|
+
DEFAULT_IPMI_USERNAME = args.ipmi_username
|
|
4075
|
+
if args.ipmi_password:
|
|
4076
|
+
DEFAULT_IPMI_PASSWORD = args.ipmi_password
|
|
4077
|
+
DEFAULT_DIFF_DISPLAY_THRESHOLD = args.diff_display_threshold
|
|
4078
|
+
FORCE_TRUECOLOR = args.force_truecolor
|
|
3167
4079
|
|
|
3168
4080
|
#%% ------------ Wrapper Block ----------------
|
|
3169
4081
|
def main():
|
|
@@ -3174,6 +4086,7 @@ def main():
|
|
|
3174
4086
|
args = process_config_file(args)
|
|
3175
4087
|
args = process_commands(args)
|
|
3176
4088
|
args = process_keys(args)
|
|
4089
|
+
args = process_control_master_config(args)
|
|
3177
4090
|
set_global_with_args(args)
|
|
3178
4091
|
|
|
3179
4092
|
if args.use_script_timeout:
|
|
@@ -3184,16 +4097,7 @@ def main():
|
|
|
3184
4097
|
if args.no_output:
|
|
3185
4098
|
__global_suppress_printout = True
|
|
3186
4099
|
if not __global_suppress_printout:
|
|
3187
|
-
cmdStr = getStrCommand(args
|
|
3188
|
-
oneonone=args.oneonone,timeout=args.timeout,password=args.password,
|
|
3189
|
-
no_watch=args.no_watch,json=args.json,called=args.no_output,max_connections=args.max_connections,
|
|
3190
|
-
files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,gather_mode = args.gather_mode,username=args.username,
|
|
3191
|
-
extraargs=args.extraargs,skipUnreachable=args.skip_unreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skip_hosts,
|
|
3192
|
-
curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only,identity_file=args.key,
|
|
3193
|
-
copy_id=args.copy_id,unavailable_host_expiry=args.unavailable_host_expiry,no_history=args.no_history,
|
|
3194
|
-
history_file = args.history_file,
|
|
3195
|
-
env_file = args.env_file,
|
|
3196
|
-
repeat = args.repeat,interval = args.interval)
|
|
4100
|
+
cmdStr = getStrCommand(**vars(args))
|
|
3197
4101
|
eprint('> ' + cmdStr)
|
|
3198
4102
|
if args.error_only:
|
|
3199
4103
|
__global_suppress_printout = True
|
|
@@ -3203,16 +4107,9 @@ def main():
|
|
|
3203
4107
|
eprint(f"Sleeping for {args.interval} seconds")
|
|
3204
4108
|
time.sleep(args.interval)
|
|
3205
4109
|
|
|
3206
|
-
if not __global_suppress_printout:
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
no_watch=args.no_watch,json=args.json,called=args.no_output,max_connections=args.max_connections,
|
|
3210
|
-
files=args.file,file_sync=args.file_sync,ipmi=args.ipmi,interface_ip_prefix=args.interface_ip_prefix,scp=args.scp,gather_mode = args.gather_mode,username=args.username,
|
|
3211
|
-
extraargs=args.extraargs,skipUnreachable=args.skip_unreachable,no_env=args.no_env,greppable=args.greppable,skip_hosts = args.skip_hosts,
|
|
3212
|
-
curses_min_char_len = args.window_width, curses_min_line_len = args.window_height,single_window=args.single_window,error_only=args.error_only,identity_file=args.key,
|
|
3213
|
-
copy_id=args.copy_id,unavailable_host_expiry=args.unavailable_host_expiry,no_history=args.no_history,
|
|
3214
|
-
history_file = args.history_file,
|
|
3215
|
-
)
|
|
4110
|
+
if not __global_suppress_printout:
|
|
4111
|
+
eprint(f"Running the {i+1}/{args.repeat} time") if args.repeat > 1 else None
|
|
4112
|
+
hosts = run_command_on_hosts(**vars(args),called=False)
|
|
3216
4113
|
#print('*'*80)
|
|
3217
4114
|
#if not __global_suppress_printout: eprint('-'*80)
|
|
3218
4115
|
succeededHosts = set()
|
|
@@ -3227,12 +4124,13 @@ def main():
|
|
|
3227
4124
|
if __mainReturnCode > 0:
|
|
3228
4125
|
if not __global_suppress_printout:
|
|
3229
4126
|
eprint(f'Complete. Failed hosts (Return Code not 0) count: {__mainReturnCode}')
|
|
3230
|
-
eprint(f'failed_hosts: {",".join(
|
|
4127
|
+
eprint(f'failed_hosts: {",".join(compact_hostnames(__failedHosts))}')
|
|
3231
4128
|
else:
|
|
3232
|
-
if not __global_suppress_printout:
|
|
4129
|
+
if not __global_suppress_printout:
|
|
4130
|
+
eprint('Complete. All hosts returned 0.')
|
|
3233
4131
|
|
|
3234
4132
|
if args.success_hosts and not __global_suppress_printout:
|
|
3235
|
-
eprint(f'succeeded_hosts: {",".join(
|
|
4133
|
+
eprint(f'succeeded_hosts: {",".join(compact_hostnames(succeededHosts))}')
|
|
3236
4134
|
|
|
3237
4135
|
if threading.active_count() > 1 and not __global_suppress_printout:
|
|
3238
4136
|
eprint(f'Remaining active thread: {threading.active_count()}')
|