automagix 4.0.0.dev1__py3-none-any.whl

automagix/__init__.py

@@ -0,0 +1,95 @@
+# PYTHON_ARGCOMPLETE_OK
+import argparse
+import os
+import subprocess
+import sys
+from time import time, strftime, gmtime
+
+from .batch_runner import get_script_and_batch_items, run_batch_items
+from .config import init_logger, CONFIG, LOG, VERSION, arguments, MAGIC_SELECTION_INT
+from .helpers import empty_queued_input_data, selector
+from .parallel_runner import run_parallel_screens
+from .progress_bar import setup_scroll_area, destroy_scroll_area
+
+
+def run_startup_script():
+    if not CONFIG.get('startup_script'):
+        return
+
+    cmds = [os.path.expandvars(CONFIG["startup_script"])]
+    cmds.extend(sys.argv)
+    subprocess.run(cmds)
+
+
+def setup(args: argparse.Namespace) -> float:
+    """Setup logger and print version information"""
+    init_logger(name=CONFIG['logger'], debug=args.debug)
+    starttime = time()
+
+    LOG.info(f'Automagix Version {VERSION}')
+    LOG.info(f'Started at: {strftime("%a, %d %b %Y %H:%M:%S UTC", gmtime(starttime))}')
+
+    configfile = CONFIG.get('config_file')
+    if configfile:
+        LOG.info(f'Using configuration from: {configfile}')
+    else:
+        LOG.warning('Configuration file not found or not configured. Using defaults.')
+
+    return starttime
+
+
+def check_screen():
+    p = subprocess.run('screen -v', shell=True, stdout=subprocess.PIPE)
+    screen_version = p.stdout.decode()
+    if p.returncode != 0 or 'command not found' in screen_version:
+        raise Exception('No GNU screen version found')
+    if 'FAU' in screen_version:
+        LOG.error(
+            'Parallel processing only supported for the "GNU" version of screen. You have the "FAU" version.\n'
+            'On MacOS you can try to install the GNU version via Homebrew: `brew install screen`.'
+        )
+        raise Exception('No supported GNU screen version found')
+
+
+def main():
+    if os.getenv('AUTOMAGIX_SHELL'):
+        print('You are running Automagix from an interactive shell of an already running Automagix!')
+        empty_queued_input_data()
+        answer = input('Do you really want to proceed? Then type "yes" and ENTER.\n')
+        if answer != 'yes':
+            sys.exit(0)
+
+    args = arguments()
+    run_startup_script()
+
+    starttime = setup(args=args)
+
+    script, batch_items = get_script_and_batch_items(args=args)
+
+    if args.jump_to == MAGIC_SELECTION_INT:
+        # next(iter(pi.items())) is here needed, because the pipeline items are all dictionaries with only one key.
+        pipeline_items = [next(iter(pi.items())) for pi in script['pipeline']]
+        args.jump_to = selector(
+            entries=[
+                (i, f'[{key}]: {cmd}')
+                for i, (key, cmd) in enumerate(pipeline_items)
+            ],
+            message='Please choose index of desired start command:'
+        )
+
+    if args.vars_file and args.parallel:
+        check_screen()
+        run_parallel_screens(script=script, batch_items=batch_items, args=args)
+        sys.exit(0)
+
+    try:
+        if CONFIG['progress_bar']:
+            setup_scroll_area()
+
+        run_batch_items(script=script, batch_items=batch_items, args=args)
+    finally:
+        if CONFIG['progress_bar']:
+            destroy_scroll_area()
+
+    if 'AUTOMAGIX_TIME' in os.environ:
+        LOG.info(f'The Automagix script took {round(time() - starttime)}s!')

automagix/automagix.py

@@ -0,0 +1,185 @@
+import os
+import sys
+from argparse import Namespace
+from collections import OrderedDict
+from functools import cached_property
+
+from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS, ReloadFromFile
+from .config import get_script
+from .environment import PipelineEnvironment
+
+
+class Automagix:
+    def __init__(
+            self,
+            script: dict,
+            variables: dict,
+            config: dict,
+            script_fields: OrderedDict,
+            cmd_args: Namespace,
+            batch_index: int,
+    ):
+        self.script = script
+        self.script_fields = script_fields
+        self.env = PipelineEnvironment(
+            config=config,
+            script=script,
+            variables=variables,
+            batch_index=batch_index,
+            cmd_args=cmd_args,
+        )
+
+        self._command_lists: dict = {}
+
+    @cached_property
+    def cmd_class(self) -> type:
+        if self.env.config.get('bundlewrap'):
+            from .bundlewrap import BWCommand, AutomagixBwRepo
+
+            self.env.config['bw_repo'] = AutomagixBwRepo(repo_path=os.environ.get('BW_REPO_PATH', '.'))
+            return BWCommand
+        else:
+            return Command
+
+    def command_list(self, pipeline: str) -> list:
+        if pipeline == 'main':
+            pipeline = 'pipeline'
+        if not self._command_lists.get(pipeline):
+            self._command_lists[pipeline] = self.build_command_list(pipeline=pipeline)
+        return self._command_lists[pipeline]
+
+    def get_command_position(self, index: int, pipeline: str) -> int:
+        if pipeline == 'always':
+            return index
+        if pipeline in ['pipeline', 'main']:
+            return index + len(self.command_list('always'))
+        if pipeline == 'cleanup':
+            return index + len(self.command_list('always')) + len(self.command_list('main'))
+
+    def set_command_count(self):
+        if not self.env.command_count:
+            self.env.command_count = len(
+                self.command_list('always') + self.command_list('main') + self.command_list('cleanup')
+            )
+
+    def build_command_list(self, pipeline: str) -> list[Command]:
+        command_list = []
+        for index, cmd in enumerate(self.script.get(pipeline, [])):
+            new_cmd = self.cmd_class(
+                cmd=cmd,
+                index=index,
+                env=self.env,
+                pipeline=pipeline,
+                position=self.get_command_position(index=index, pipeline=pipeline),
+            )
+            command_list.append(new_cmd)
+            if new_cmd.assignment_var and new_cmd.assignment_var not in self.env.vars:
+                self.env.vars[new_cmd.assignment_var] = f'{{{new_cmd.assignment_var}}}'
+        return command_list
+
+    def reload_script(self):
+        self.script = get_script(args=self.env.cmd_args)
+        self._command_lists = {}  # Clear cache
+
+    def print_main_data(self):
+        print('\n')
+        self.env.LOG.info(' ------ Overview ------')
+        for field_key, field_value in self.script_fields.items():
+            print()
+            self.env.LOG.info(f'{field_value}:')
+            for key, value in self.script.get(field_key, {}).items():
+                self.env.LOG.info(f" {key}: {value}")
+
+    def print_command_line_steps(self, command_list: list[Command]):
+        print()
+        self.env.LOG.info('Commandline Steps:')
+        for cmd in command_list:
+            self.env.LOG.info(f"({cmd.index}) [{cmd.orig_key}]: {cmd.get_resolved_value(dummy=True)}")
+        print()
+
+    def check_possibly_dangerous_vars(self):
+        warn = 0
+        for key, value in self.env.vars.items():
+            if isinstance(value, str) and value.lower() == 'false':
+                self.env.LOG.warning(
+                    f'[vars:{key}] This variable is a string with value "{value}".'
+                    ' Be aware that this becomes `True` for conditions and other boolean operations without conversion,'
+                    ' because any non-empty string in Python is evaluated as `True`.'
+                )
+                warn += 1
+
+        if warn:
+            answer = self.env.interact(question='Do you want to proceed? Then type "yes" and ENTER.\n')
+            if answer != 'yes':
+                sys.exit(0)
+
+    def _execute_command_list(self, name: str, start_index: int, treat_as_main: bool):
+        try:
+            steps = self.script.get('_steps')
+            for cmd in self.command_list(name)[start_index:]:
+                if treat_as_main:
+                    if steps and (self.script['_exclude'] == (cmd.index in steps)):
+                        # Case 1: exclude is True  and index is in steps => skip
+                        # Case 2: exclude is False and index is in steps => execute
+                        print()
+                        self.env.LOG.notice(f'({cmd.index}) Not selected for execution: skip')
+                        continue
+                    cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force)
+                else:
+                    cmd.execute()
+        except ReloadFromFile as exc:
+            print()
+            self.env.LOG.info(f'Reload script from file and retry => ({exc.index})')
+            self.reload_script()
+            self._execute_command_list(name=name, start_index=exc.index, treat_as_main=treat_as_main)
+
+    def execute_pipeline(self, name: str):
+        if not self.command_list(name):
+            return
+
+        if name == 'main':
+            treat_as_main = True
+            start_index = self.env.cmd_args.jump_to
+        else:
+            treat_as_main = False
+            start_index = 0
+
+        print()
+        self.env.LOG.info('------------------------------')
+        self.env.LOG.info(f' --- Start {name.upper()} pipeline ---')
+
+        self._execute_command_list(name=name, start_index=start_index, treat_as_main=treat_as_main)
+
+        print()
+        self.env.LOG.info(f' --- End {name.upper()} pipeline ---')
+        self.env.LOG.info('------------------------------\n')
+
+    def run(self):
+        print('\n')
+        self.env.LOG.info('//////////////////////////////////////////////////////////////////////')
+        self.env.LOG.info(f"---- {self.script['name']} ----")
+        self.env.LOG.info('//////////////////////////////////////////////////////////////////////')
+
+        PERSISTENT_VARS.clear()
+
+        self.execute_pipeline(name='always')
+
+        self.print_main_data()
+        self.print_command_line_steps(command_list=self.command_list('main'))
+        self.check_possibly_dangerous_vars()
+        if self.env.cmd_args.print_overview:
+            sys.exit(0)
+
+        try:
+            self.execute_pipeline(name='main')
+        except (AbortException, SkipBatchItemException):
+            self.env.LOG.debug('Abort requested. Cleaning up.')
+            self.execute_pipeline(name='cleanup')
+            self.env.LOG.debug('Clean up done. Exiting.')
+            raise
+
+        self.execute_pipeline(name='cleanup')
+
+        self.env.LOG.info('---------------------------------------------------------------')
+        self.env.LOG.info('Automagix finished: Congratulations and have a N.I.C.E. day :-)')
+        self.env.LOG.info('---------------------------------------------------------------')

automagix/automagix_test.py

@@ -0,0 +1,34 @@
+from automagix.command import Command
+from tests.test_environment import testauto, environment
+
+len_always = len(testauto.script.get('always', []))
+len_main = len(testauto.script.get('pipeline', []))
+len_cleanup = len(testauto.script.get('cleanup', []))
+
+
+def test__automagix__command_list():
+    assert testauto.command_list('main') == testauto.command_list('pipeline')
+    cmd = Command(
+        cmd={'local': "echo 'Print this always :-)'"},
+        index=0,
+        pipeline='always',
+        env=environment,
+        position=0,
+    )
+    first_cleanup = testauto.command_list('always')[0]
+    assert isinstance(first_cleanup, Command)
+    assert vars(first_cleanup) == vars(cmd)
+
+
+def test__automagix__get_command_position():
+    assert testauto.get_command_position(index=4, pipeline='always') == 4
+    assert testauto.get_command_position(index=3, pipeline='always') != 4
+    assert testauto.get_command_position(index=3, pipeline='main') == len_always + 3
+    assert testauto.get_command_position(index=3, pipeline='pipeline') == len_always + 3
+    assert testauto.get_command_position(index=2, pipeline='cleanup') == len_always + len_main + 2
+
+
+def test__automagix__set_command_count():
+    testauto.env.command_count = None
+    testauto.set_command_count()
+    assert testauto.env.command_count == len_always + len_main + len_cleanup

automagix/bash_completion.py

@@ -0,0 +1,65 @@
+import locale
+import subprocess
+from argparse import Action, Namespace
+
+from argcomplete import warn
+
+from .helpers import read_yaml, search_script
+
+
+def _call(*args, **kwargs):
+    try:
+        return subprocess.check_output(*args, **kwargs).decode(locale.getpreferredencoding()).splitlines()
+    except subprocess.CalledProcessError:
+        return []
+
+
+class ScriptFileCompleter:
+    """
+    Scriptfile completer
+    """
+
+    def __init__(self, script_dir: str):
+        self.script_dir = script_dir
+
+    def __call__(self, prefix: str, **kwargs):
+        completion = []
+        try:
+            completion.extend(self.find_dirs_and_files(root_path='.', prefix=prefix))
+            completion.extend(self.find_dirs_and_files(root_path=self.script_dir, prefix=prefix))
+        except Exception as exc:
+            warn(f'Shell completion failed: {repr(exc)}')
+        return completion
+
+    def find_dirs_and_files(self, root_path: str, prefix: str) -> list:
+        completion = []
+        pre_len = len(root_path) + 1
+        directories = _call(["bash", "-c", f"compgen -A directory -- '{root_path}/{prefix}'"])
+        completion += [f'{d[pre_len:]}/' for d in directories]
+        for ext in ['yaml', 'yml']:
+            files = _call(["bash", "-c", f"compgen -A file -X '!*.{ext}' -- '{root_path}/{prefix}'"])
+            completion += [f[pre_len:] for f in files]
+        return completion
+
+
+class ScriptFieldCompleter:
+    def __init__(self, script_dir: str):
+        self.script_dir = script_dir
+
+    def __call__(self, action: Action, parsed_args: Namespace, **kwargs):
+        try:
+            if parsed_args.scriptfile is None:
+                return []
+
+            s_file = search_script(name=parsed_args.scriptfile, script_dir=self.script_dir, non_interactive=True)
+            if not s_file:
+                warn('Script not found or multiple options. Cannot complete.')
+                return []
+
+            script = read_yaml(s_file)
+            completion = [f'{key}=' for key in script.get(action.dest, {}).keys()]
+
+            return completion
+        except Exception as exc:
+            warn(f'Shell completion failed: {repr(exc)}')
+            return []

automagix/batch_runner.py

@@ -0,0 +1,70 @@
+import sys
+from argparse import Namespace
+from copy import deepcopy
+from csv import DictReader
+from typing import Callable
+
+from .automagix import Automagix
+from .command import SkipBatchItemException, AbortException
+from .config import CONFIG, get_script, LOG, update_script_from_row, collect_vars, SCRIPT_FIELDS
+
+
+def get_script_and_batch_items(args: Namespace) -> (dict, list):
+    script = get_script(args=args)
+
+    # Empty item means: there is nothing to update, take the script as it is
+    batch_items: list[dict] = [{}]
+    if args.vars_file:
+        with open(args.vars_file) as csvfile:
+            batch_items = list(DictReader(filter(lambda row: row[0] != '#', csvfile)))
+
+    return script, batch_items
+
+
+def create_automagix_list(script: dict, batch_items: list, args: Namespace) -> list[Automagix]:
+    automagix_list = []
+    for i, row in enumerate(batch_items, start=1):
+        script_copy = deepcopy(script)
+        script_copy['_batch_mode'] = len(batch_items) > 1
+        script_copy['_batch_items_count'] = len(batch_items)
+
+        update_script_from_row(row=row, script=script_copy, index=i)
+
+        variables = collect_vars(script_copy)
+
+        auto = Automagix(
+            script=script_copy,
+            variables=variables,
+            config=CONFIG,
+            script_fields=SCRIPT_FIELDS,
+            cmd_args=args,
+            batch_index=i,
+        )
+        automagix_list.append(auto)
+    return automagix_list
+
+
+def run_automagix_list(automagix_list: list[Automagix], send_status_callback: Callable = None):
+    for auto in automagix_list:
+        auto.set_command_count()
+        auto.env.attach_logger()
+        auto.env.reinit_logger()
+        if send_status_callback:
+            auto.env.send_status = send_status_callback
+        try:
+            auto.run()
+        except SkipBatchItemException as exc:
+            LOG.info(str(exc))
+            LOG.notice('=====> Jumping to next batch item.')
+            continue
+        except AbortException as exc:
+            sys.exit(int(exc))
+        except KeyboardInterrupt:
+            print()
+            LOG.warning('Aborted by user. Exiting.')
+            sys.exit(130)
+
+
+def run_batch_items(script: dict, batch_items: list, args: Namespace):
+    automagix_list = create_automagix_list(script=script, batch_items=batch_items, args=args)
+    run_automagix_list(automagix_list=automagix_list)

automagix/bundlewrap.py

@@ -0,0 +1,75 @@
+from bundlewrap.exceptions import NoSuchNode, NoSuchGroup
+from bundlewrap.group import Group
+from bundlewrap.node import Node
+from bundlewrap.repo import Repository
+
+from .command import Command, PA
+
+
+class AutomagixBwRepo(Repository):
+    def reload(self):
+        self.__init__(repo_path=self.path)
+
+
+class BWCommand(Command):
+    def _generate_python_vars(self):
+        locale_vars = {'AUTOMAGIX_BW_REPO': self.env.config['bw_repo']}
+        for key, value in self.env.systems.items():
+            if not value.startswith('hostname!'):
+                try:
+                    self.env.config['bw_repo'].get_node(value)
+                except NoSuchNode:
+                    try:
+                        self.env.config['bw_repo'].get_group(value)
+                    except NoSuchGroup:
+                        self.env.LOG.warning(f'"{value}" is neither a BW node nor a BW group')
+        locale_vars['VARS'] = self.env.vars
+        locale_vars['NODES'] = BWNodesWrapper(repo=self.env.config['bw_repo'], systems=self.env.systems)
+        return locale_vars
+
+    def _remote_action(self) -> int:
+        bw_repo: Repository = self.env.config['bw_repo']
+        system = self.get_system()
+        if system.startswith('hostname!'):
+            return self._remote_action_on_hostname(hostname=system.replace('hostname!', ''))
+        try:
+            node: Node = bw_repo.get_node(system)
+            return self._remote_action_on_hostname(hostname=node.hostname)
+        except NoSuchNode as exc:
+            try:
+                group: Group = bw_repo.get_group(system)
+            except NoSuchGroup:
+                raise exc
+            print()
+            self.env.LOG.info(f' --- Executing command for all nodes in BW group >{group.name}< ---')
+            for node in group.nodes:
+                print()
+                self.env.LOG.info(f'- {node.name} -')
+                self._remote_bw_group_action(node=node)
+            return 0
+
+    def _remote_bw_group_action(self, node: Node):
+        return_code = self._remote_action_on_hostname(hostname=node.hostname)
+        if return_code != 0:
+            self.env.LOG.error(f'Command ({self.index}) on {node.name} failed with return code {return_code}.')
+            if self.env.cmd_args.force:
+                return
+
+            err_answer = self._ask_user(
+                question='[PF] What should I do?',
+                allowed_options=[PA.proceed, PA.terminal, PA.variables, PA.retry, PA.abort],
+            )
+            # _ask_user handles are answers but PA.retry, PA.skip, PA.proceed
+            # PA.skip is not in the allowed options'
+            # PA.proceed means 'proceed' so we can just go on
+            if err_answer == PA.retry.answer:
+                return self._remote_bw_group_action(node=node)
+
+
+class BWNodesWrapper:
+    def __init__(self, repo: Repository, systems: dict):
+        self._repo = repo
+        self._systems = systems
+
+    def __getattr__(self, name):
+        return self._repo.get_node(self._systems[name])

automagix/colors.py

@@ -0,0 +1,42 @@
+BLUE = '\033[34m'
+CYAN = '\033[36m'
+GREEN = '\033[32m'
+RED = '\033[31m'
+YELLOW = '\033[33m'
+
+BOLD = '\033[1m'
+ITALIC = '\033[3m'
+
+RESET = '\033[0m'
+
+
+def blue(text):
+    return f'{BLUE}{text}{RESET}'
+
+
+def cyan(text):
+    return f'{CYAN}{text}{RESET}'
+
+
+def green(text):
+    return f'{GREEN}{text}{RESET}'
+
+
+def red(text):
+    return f'{RED}{text}{RESET}'
+
+
+def yellow(text):
+    return f'{YELLOW}{text}{RESET}'
+
+
+def bold(text):
+    return f'{BOLD}{text}{RESET}'
+
+
+def italic(text):
+    return f'{ITALIC}{text}{RESET}'
+
+
+def nocolor(text):
+    return f'{RESET}{text}'