rsh-tools 3.0.1__py3-none-any.whl

rsh/__init__.py

@@ -0,0 +1,3 @@
+"""rsh - ssh helpers: sr (root shell), sExec/pExec (run a command on hosts), inventory plugins."""
+
+__version__ = '3.0.1'

rsh/build_inventory.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""rshInventory - collect hosts from the enabled inventory plugins and write inventory.yaml."""
+
+import os
+import sys
+import json
+import logging
+import re
+
+import yaml
+
+import rsh.config
+from rsh.plugins import PLUGINS
+
+logger = logging.getLogger('rshInventory')
+
+
+def natural_sort_key(name):
+    """Token-aware natural order: 'web2' < 'web10'; the domain part (e.g. '.aws') is ignored."""
+    left = str(name).split('.', 1)[0]
+    tokens = []
+    for rough in re.split(r'[^A-Za-z0-9]+', left):
+        for part in re.findall(r'[A-Za-z]+|\d+', rough):
+            tokens.append((1, int(part)) if part.isdigit() else (0, part.lower()))
+    return tokens
+
+
+def group_name(*parts):
+    name = '_'.join(part.replace('-', '_') for part in parts)
+    return re.sub(r'\s+', '_', name)
+
+
+def collect_records(cfg):
+    """Run every enabled plugin, return the merged list of normalized records."""
+    records = []
+    for key, plugin_class in PLUGINS.items():
+        if not (cfg.get(key) or {}).get('enable'):
+            logger.debug("plugin '%s' disabled", key)
+            continue
+        logger.debug("running plugin '%s'", key)
+        records.extend(plugin_class(cfg).instances_info())
+    for key in cfg:
+        if key.endswith('Inventory') and key not in PLUGINS:
+            logger.warning("config section '%s' has no matching inventory plugin, known: %s", key, sorted(PLUGINS))
+    return records
+
+
+def build_inventory(records, cfg):
+    """Turn plugin records into the {'hosts': ..., 'groups': ...} structure."""
+    hosts = {}
+    groups = {}
+
+    def add_to_group(name, host):
+        groups.setdefault(name, [])
+        if host not in groups[name]:
+            groups[name].append(host)
+
+    for record in records:
+        host = record.get('host')
+        if not host:
+            logger.error("record without 'host': %s", record)
+            sys.exit(1)
+        module = record.get('rshInventoryModule')
+        settings = cfg.get(module) or {}
+        if host not in hosts:
+            hosts[host] = {'sshHost': record['sshHost']}
+            for field in settings.get('hostInfoFields', []):
+                hosts[host][field] = record.get(field, 'unknown')
+        skip_tags = settings.get('skipTagsForMakeGroups', [])
+        for tag in record.get('tags', []):
+            if tag['Key'] in skip_tags:
+                continue
+            add_to_group(group_name('tag', tag['Key'], tag['Value']), host)
+        add_to_group(group_name('dc', record.get('hosting', 'unknown'), record['dc']), host)
+
+    hosts = {name: hosts[name] for name in sorted(hosts, key=natural_sort_key)}
+    groups = {name: sorted(groups[name], key=natural_sort_key) for name in sorted(groups, key=natural_sort_key)}
+    return {'hosts': hosts, 'groups': groups}
+
+
+def write_inventory(inventory, path):
+    path = os.path.expanduser(path)
+    with open(path, 'w') as f:
+        yaml.dump(inventory, f, default_flow_style=False, sort_keys=False)
+    logger.info("inventory written to '%s': %d hosts, %d groups", path, len(inventory['hosts']), len(inventory['groups']))
+
+
+def main():
+    cfg = rsh.config.cfg
+    logger.debug("cfg=%s", json.dumps(cfg, indent=4))
+    records = collect_records(cfg)
+    logger.debug("records=%s", json.dumps(records, indent=4))
+    inventory = build_inventory(records, cfg)
+    logger.debug("inventory=%s", inventory)
+    write_inventory(inventory, cfg['inventoryFilePath'])
+
+
+if __name__ == "__main__":
+    main()

rsh/compgen.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""rshCompgen - word lists for bash completion, and the completion script itself."""
+
+import argparse
+import logging
+import sys
+from importlib import resources
+
+import rsh.config
+from rsh.inventory import load_inventory
+
+logger = logging.getLogger('rshCompgen')
+
+
+def completion_script():
+    """Bash completion for sr/sExec/pExec; install with: eval "$(rshCompgen completion)"."""
+    return resources.files('rsh').joinpath('completion.bash').read_text()
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(description='bash completion helper for sr, sExec and pExec')
+    parser.add_argument(
+        'what',
+        choices=['hosts', 'groups', 'completion'],
+        help="'hosts' or 'groups' print inventory names, 'completion' prints the bash completion script",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    if args.what == 'completion':
+        sys.stdout.write(completion_script())
+        return
+    inventory = load_inventory(rsh.config.cfg)
+    print(' '.join(inventory[args.what].keys()))
+
+
+if __name__ == "__main__":
+    main()

rsh/completion.bash

@@ -0,0 +1,10 @@
+# bash completion for rsh tools; install with:  eval "$(rshCompgen completion)"
+_rsh_completion_hosts(){
+    COMPREPLY=( $(compgen -W "$(rshCompgen hosts)" -- "$2") )
+}
+_rsh_completion_groups(){
+    COMPREPLY=( $(compgen -W "$(rshCompgen groups)" -- "$2") )
+}
+complete -F _rsh_completion_hosts sr
+complete -F _rsh_completion_groups sExec
+complete -F _rsh_completion_groups pExec

rsh/config.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+from os.path import expanduser
+import sys
+import yaml
+import re
+import logging
+from deepmerge import always_merger
+
+PROGRAM_NAME = 'rsh'
+HOME_DIR = expanduser("~")
+DEFAULT_CONFIG_FILES = [
+    '/etc/' + PROGRAM_NAME + '/config.yaml',
+    HOME_DIR + '/.' + PROGRAM_NAME + '.yaml',
+    './.config.yaml',
+]
+cfg = {
+    #'logFile': HOME_DIR + '/log/' + PROGRAM_NAME + '/' + PROGRAM_NAME + '.log',
+    'logFile': 'stdout',
+    'logLevel': 'info',
+    'gpgFile': HOME_DIR + '/.pass.gpg',
+    'inventoryFilePath': HOME_DIR + '/inventory.yaml',
+    'sensitiveKeys': [
+        'application_key',
+        'application_secret',
+        'consumer_key',
+    ],
+    'awsInventory': {
+        'enable': True,
+        'regions': [],
+        'sshHostField': 'host',             # field for 'sshHost', variants: host, PublicDnsName, PublicIpAddress, PrivateDnsName, PrivateIpAddress
+        'skipTagsForMakeGroups': [          # these tag 'key' will skip when creating groups
+            'Name',
+            'Description',
+            'description',
+        ],
+        'skipNotRunningInstance': True,     # skipping not running instance
+        'hostInfoFields': [
+            'PrivateIpAddress',
+            'PublicIpAddress',
+        ],
+        'nameSuffix': '.aws',               # suffix for instance names
+    },
+    'ovhInventory': {
+        'enable': True,
+        'api': {
+            'endpoint': 'ovh-ca',           # Endpoint of API OVH
+            'application_key': 'xxx',       # Application Key
+            'application_secret': 'yyy',    # Application Secret
+            'consumer_key': 'zzz',          # Consumer Key
+        },
+        'sshHostField': 'host',             # field for 'sshHost', variants: host, customName, publicIp, internalName
+        'skipTagsForMakeGroups': [],        # these tag 'key' will skip when creating groups
+        'hostInfoFields': [
+            'publicIp',
+            'internalName', # internal name
+            'dc',
+        ],
+    },
+    'gceInventory': {
+        'enable': True,
+        'skipNotRunningInstance': True,     # skipping not running instance
+        'sshHostField': 'host',             # field for 'sshHost', variants: host, PrivateIpAddress, PublicIpAddress
+        'hostInfoFields': [
+            'PrivateIpAddress',
+            'PublicIpAddress',
+            'dc',
+        ],
+        'skipTagsForMakeGroups': [          # these tag 'key' will skip when creating groups
+            'name',
+        ],
+        'nameSuffix': '.gc',               # suffix for instance names
+    },
+    'sr': {
+        'ssh': {
+            'command': 'ssh -Y -o ConnectTimeout=5 -o StrictHostKeyChecking=no',
+        },
+        'pexpect': {
+            'timeout': 120,
+        },
+        'useInventorySshHost': True, # use sshHost from inventory for connect to host
+        'escalatePrivilegesCommands': [],   # commands are tried on the host in the listed order until one gives a root shell,
+                                            # e.g. ['su -m', 'sudo -s -E'];
+                                            # must stay empty here: deepmerge appends lists, so a default value would be added to the user's list
+        'escalatePrivilegesCommand': 'sudo -s -E', # deprecated, used only when 'escalatePrivilegesCommands' is empty; variants: 'su -m', 'sudo -s', 'sudo -s -E'
+        'postLoginCommand': None # command will run after login, used to set up the environment
+    },
+}
+
+# load config files {{
+def load_config_files(cfg, paths):
+    """Deep-merge every existing YAML mapping from paths into cfg, in order."""
+    for path in paths:
+        if not os.path.isfile(path):
+            continue
+        try:
+            with open(path, 'r') as f:
+                loaded = yaml.load(f, Loader=yaml.Loader)
+        except Exception as e:
+            logging.warning("skipping config file '%s': %s", path, e)
+            continue
+        if not isinstance(loaded, dict):
+            logging.warning("skipping config file '%s': top level is not a mapping", path)
+            continue
+        cfg = always_merger.merge(cfg, loaded)
+    return cfg
+
+cfg = load_config_files(cfg, DEFAULT_CONFIG_FILES)
+cfg['logDir'] = os.path.dirname(cfg['logFile']) or '.'
+# }}
+
+# logging {{
+def parse_log_level(name):
+    if re.match(r"^(warn|warning)$", str(name), re.IGNORECASE):
+        return logging.WARNING
+    if re.match(r"^debug$", str(name), re.IGNORECASE):
+        return logging.DEBUG
+    return logging.INFO
+
+
+def find_values(data, keys):
+    """Collect values of the given keys anywhere in a nested dict/list structure."""
+    if isinstance(keys, str):
+        keys = [keys]
+    found = []
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if key in keys:
+                found.append(value)
+            else:
+                found.extend(find_values(value, keys))
+    elif isinstance(data, list):
+        for item in data:
+            found.extend(find_values(item, keys))
+    return found
+
+
+class SensitiveFormatter(logging.Formatter):
+    """Masks sensitive config values (api keys etc.) in log output."""
+
+    def __init__(self, fmt, datefmt, sensitive_values):
+        super().__init__(fmt, datefmt=datefmt)
+        values = [str(value) for value in sensitive_values if value]
+        self.pattern = re.compile('|'.join(re.escape(value) for value in values)) if values else None
+
+    def format(self, record):
+        message = super().format(record)
+        if self.pattern is None:
+            return message
+        return self.pattern.sub('[CLASSIFIED]', message)
+
+
+LOG_FORMAT = '%(asctime)s\t%(name)s\t%(funcName)s\t%(levelname)s\t%(message)s'
+LOG_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
+
+
+def setup_logging(cfg):
+    level = parse_log_level(cfg['logLevel'])
+    if level > logging.DEBUG:
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+        logging.getLogger("requests").setLevel(logging.WARNING)
+    if cfg['logFile'] == 'stdout':
+        logging.basicConfig(level=level, format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT)
+    else:
+        os.makedirs(cfg['logDir'], exist_ok=True)
+        logging.basicConfig(filename=cfg['logFile'], level=level, format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT)
+    formatter = SensitiveFormatter(LOG_FORMAT, LOG_DATE_FORMAT, find_values(cfg, cfg['sensitiveKeys']))
+    for handler in logging.root.handlers:
+        handler.setFormatter(formatter)
+
+setup_logging(cfg)
+# }}

rsh/exec_functions.py

@@ -0,0 +1,161 @@
+# exec_functions.py
+
+import sys
+import os
+import json
+import argparse
+import time
+import logging
+from braceexpand import braceexpand
+from fabric import Connection
+import rsh.config
+from rsh.inventory import load_inventory as _load_inventory, resolve_hosts as _resolve_hosts
+
+logger = logging.getLogger(__name__)
+logging.getLogger("paramiko").setLevel(logging.WARNING)
+
+
+def parse_arguments():
+    """
+    Parse CLI arguments.
+    Returns a tuple: (hosts_or_groups: list[str], command: str, delay: float)
+    """
+    prog_name = os.path.basename(sys.argv[0])
+    parser = argparse.ArgumentParser(
+        description='Execute commands via SSH on remote servers.',
+        usage=(
+            f"[{prog_name}] -g GROUP -c COMMAND  "
+            f"or  [{prog_name}] GROUP COMMAND...  "
+            f"with optional --delay SECONDS"
+        ),
+    )
+
+    parser.add_argument(
+        '-g', '--groups', dest='hosts_or_groups', nargs='+',
+        help='Hosts or groups (supports brace expansions)'
+    )
+    parser.add_argument(
+        '-c', '--command', nargs=argparse.REMAINDER,
+        help='Command to execute'
+    )
+    parser.add_argument(
+        'fallback', nargs=argparse.REMAINDER,
+        help='Fallback: GROUP COMMAND ...'
+    )
+    parser.add_argument(
+        '-d', '--delay', type=float, default=0.0,
+        help='Delay (in seconds) between hosts when running sequentially'
+    )
+
+    args = parser.parse_args()
+
+    if args.hosts_or_groups and args.command:
+        hosts_or_groups_raw, command_parts = args.hosts_or_groups, args.command
+    elif args.fallback:
+        if len(args.fallback) < 2:
+            parser.print_usage()
+            sys.exit(1)
+        hosts_or_groups_raw, command_parts = args.fallback[0], args.fallback[1:]
+    else:
+        parser.print_usage()
+        sys.exit(1)
+
+    hosts_or_groups = (
+        list(braceexpand(hosts_or_groups_raw))
+        if isinstance(hosts_or_groups_raw, str)
+        else sum([list(braceexpand(x)) for x in hosts_or_groups_raw], [])
+    )
+
+    return hosts_or_groups, ' '.join(command_parts), float(args.delay)
+
+
+def load_inventory():
+    """Load inventory file defined in rsh.config.cfg (see rsh.inventory)."""
+    return _load_inventory(rsh.config.cfg)
+
+
+def resolve_hosts(hosts_or_groups, inventory):
+    """Resolve group names to ssh addresses (see rsh.inventory)."""
+    return _resolve_hosts(hosts_or_groups, inventory)
+
+
+def run_command_sequential(hosts, command, delay=0.0):
+    """
+    Run a command sequentially on each host, streaming stdout/stderr immediately.
+    Parameter `delay` adds a pause (in seconds) between hosts.
+    """
+    logger.debug(f"hosts={json.dumps(hosts)}")
+    logger.debug(f"command='{command}'")
+    logger.debug(f"delay={delay}")
+
+    for idx, host in enumerate(hosts, 1):
+        header = f"[{host}] ({idx}/{len(hosts)})"
+        print(header)
+        print("-" * len(header))
+        try:
+            conn = Connection(host)
+            result = conn.sudo(command, pty=True, hide=False, warn=True)
+            print(f"\n[{host}] exit status: {result.return_code}")
+        except Exception as e:
+            logger.error(f"[{host}] Command execution failed: {e}")
+            print(f"[{host}] ERROR: {e}")
+        finally:
+            if len(hosts) > 1:
+                print("===\n")
+
+            # Optional delay between hosts
+            if idx < len(hosts) and delay > 0:
+                print(f"Sleeping {delay} seconds before next host...\n")
+                try:
+                    time.sleep(delay)
+                except KeyboardInterrupt:
+                    print("Sleep interrupted by user, continuing...\n")
+
+
+def run_command_parallel_dedup(GroupClass, hosts, command):
+    """
+    Run a command in parallel across hosts and deduplicate identical outputs.
+    Prints each unique output once alongside the list of hosts that produced it.
+    """
+    logger.debug(f"hosts={json.dumps(hosts)}")
+    logger.debug(f"command='{command}'")
+
+    try:
+        group = GroupClass(*hosts)
+        results = group.sudo(command, warn=True, hide=True, pty=True)
+
+        buckets = {}     # output -> list of hosts
+        exit_codes = {}  # output -> representative exit code
+
+        for conn, res in results.items():
+            out = (res.stdout or "").strip()
+            err = (res.stderr or "").strip()
+            combined = out if out else err
+            key = combined
+
+            if key not in buckets:
+                buckets[key] = []
+                exit_codes[key] = res.return_code
+            buckets[key].append(conn.host)
+
+        def sort_key(item):
+            _, host_list = item
+            return (-len(host_list), ", ".join(sorted(host_list)))
+
+        for output, host_list in sorted(buckets.items(), key=sort_key):
+            host_line = ", ".join(sorted(host_list))
+            header = f"[{host_line}]"
+            print(header)
+            print("-" * len(header))
+
+            print(output if output else "(no output)")
+            print(f"\nexit status (representative): {exit_codes[output]}")
+            print("===\n")
+
+    except Exception as e:
+        try:
+            for i in e:
+                logger.error(f"Command execution failed: {i}")
+        except Exception:
+            logger.error(f"Command execution failed: {e}")
+        sys.exit(1)

rsh/execute.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""sExec / pExec - run a command over ssh on hosts or inventory groups, sequentially or in parallel."""
+
+import sys
+import os
+import json
+import argparse
+import time
+import logging
+from braceexpand import braceexpand
+from fabric import Connection, ThreadingGroup
+import rsh.config
+from rsh.inventory import load_inventory, resolve_hosts
+
+logger = logging.getLogger(__name__)
+logging.getLogger("paramiko").setLevel(logging.WARNING)
+
+
+def parse_arguments():
+    """
+    Parse CLI arguments.
+    Returns a tuple: (hosts_or_groups: list[str], command: str, delay: float)
+    """
+    prog_name = os.path.basename(sys.argv[0])
+    parser = argparse.ArgumentParser(
+        description='Execute commands via SSH on remote servers.',
+        usage=(
+            f"[{prog_name}] -g GROUP -c COMMAND  "
+            f"or  [{prog_name}] GROUP COMMAND...  "
+            f"with optional --delay SECONDS"
+        ),
+    )
+
+    parser.add_argument(
+        '-g', '--groups', dest='hosts_or_groups', nargs='+',
+        help='Hosts or groups (supports brace expansions)'
+    )
+    parser.add_argument(
+        '-c', '--command', nargs=argparse.REMAINDER,
+        help='Command to execute'
+    )
+    parser.add_argument(
+        'fallback', nargs=argparse.REMAINDER,
+        help='Fallback: GROUP COMMAND ...'
+    )
+    parser.add_argument(
+        '-d', '--delay', type=float, default=0.0,
+        help='Delay (in seconds) between hosts when running sequentially'
+    )
+
+    args = parser.parse_args()
+
+    if args.hosts_or_groups and args.command:
+        hosts_or_groups_raw, command_parts = args.hosts_or_groups, args.command
+    elif args.fallback:
+        if len(args.fallback) < 2:
+            parser.print_usage()
+            sys.exit(1)
+        hosts_or_groups_raw, command_parts = args.fallback[0], args.fallback[1:]
+    else:
+        parser.print_usage()
+        sys.exit(1)
+
+    hosts_or_groups = (
+        list(braceexpand(hosts_or_groups_raw))
+        if isinstance(hosts_or_groups_raw, str)
+        else sum([list(braceexpand(x)) for x in hosts_or_groups_raw], [])
+    )
+
+    return hosts_or_groups, ' '.join(command_parts), float(args.delay)
+
+
+def resolve_targets():
+    """Parse CLI args and resolve them against the inventory: (hosts, command, delay)."""
+    hosts_or_groups, command, delay = parse_arguments()
+    inventory = load_inventory(rsh.config.cfg)
+    return resolve_hosts(hosts_or_groups, inventory), command, delay
+
+
+def run_command_sequential(hosts, command, delay=0.0):
+    """
+    Run a command sequentially on each host, streaming stdout/stderr immediately.
+    Parameter `delay` adds a pause (in seconds) between hosts.
+    """
+    logger.debug(f"hosts={json.dumps(hosts)}")
+    logger.debug(f"command='{command}'")
+    logger.debug(f"delay={delay}")
+
+    for idx, host in enumerate(hosts, 1):
+        header = f"[{host}] ({idx}/{len(hosts)})"
+        print(header)
+        print("-" * len(header))
+        try:
+            conn = Connection(host)
+            result = conn.sudo(command, pty=True, hide=False, warn=True)
+            print(f"\n[{host}] exit status: {result.return_code}")
+        except Exception as e:
+            logger.error(f"[{host}] Command execution failed: {e}")
+            print(f"[{host}] ERROR: {e}")
+        finally:
+            if len(hosts) > 1:
+                print("===\n")
+
+            # Optional delay between hosts
+            if idx < len(hosts) and delay > 0:
+                print(f"Sleeping {delay} seconds before next host...\n")
+                try:
+                    time.sleep(delay)
+                except KeyboardInterrupt:
+                    print("Sleep interrupted by user, continuing...\n")
+
+
+def run_command_parallel_dedup(GroupClass, hosts, command):
+    """
+    Run a command in parallel across hosts and deduplicate identical outputs.
+    Prints each unique output once alongside the list of hosts that produced it.
+    """
+    logger.debug(f"hosts={json.dumps(hosts)}")
+    logger.debug(f"command='{command}'")
+
+    try:
+        group = GroupClass(*hosts)
+        results = group.sudo(command, warn=True, hide=True, pty=True)
+
+        buckets = {}     # output -> list of hosts
+        exit_codes = {}  # output -> representative exit code
+
+        for conn, res in results.items():
+            out = (res.stdout or "").strip()
+            err = (res.stderr or "").strip()
+            combined = out if out else err
+            key = combined
+
+            if key not in buckets:
+                buckets[key] = []
+                exit_codes[key] = res.return_code
+            buckets[key].append(conn.host)
+
+        def sort_key(item):
+            _, host_list = item
+            return (-len(host_list), ", ".join(sorted(host_list)))
+
+        for output, host_list in sorted(buckets.items(), key=sort_key):
+            host_line = ", ".join(sorted(host_list))
+            header = f"[{host_line}]"
+            print(header)
+            print("-" * len(header))
+
+            print(output if output else "(no output)")
+            print(f"\nexit status (representative): {exit_codes[output]}")
+            print("===\n")
+
+    except Exception as e:
+        try:
+            for i in e:
+                logger.error(f"Command execution failed: {i}")
+        except Exception:
+            logger.error(f"Command execution failed: {e}")
+        sys.exit(1)
+
+
+def main_sequential():
+    """Entry point for sExec."""
+    hosts, command, delay = resolve_targets()
+    run_command_sequential(hosts, command, delay=delay)
+
+
+def main_parallel():
+    """Entry point for pExec."""
+    hosts, command, _ = resolve_targets()
+    run_command_parallel_dedup(ThreadingGroup, hosts, command)