ae-shell 0.3.16__tar.gz → 0.3.18__tar.gz

{ae_shell-0.3.16/ae_shell.egg-info → ae_shell-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ae_shell
-Version: 0.3.16
+Version: 0.3.18
 Summary: ae namespace module portion shell: shell execution and environment helpers
 Home-page: https://gitlab.com/ae-group/ae_shell
 Author: AndiEcker
@@ -64,13 +64,13 @@ Dynamic: summary
 
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project ae.ae v0.3.110 -->
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project aedev.namespace_root_tpls v0.3.33 -->
-# shell 0.3.16
+# shell 0.3.18
 
 [![GitLab develop](https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/develop?logo=python)](
     https://gitlab.com/ae-group/ae_shell)
 [![LatestPyPIrelease](
-    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.16?logo=python)](
-    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.16)
+    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.18?logo=python)](
+    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.18)
 [![PyPIVersions](https://img.shields.io/pypi/v/ae_shell)](
     https://pypi.org/project/ae-shell/#history)
 

{ae_shell-0.3.16 → ae_shell-0.3.18}/README.md

@@ -1,12 +1,12 @@
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project ae.ae v0.3.110 -->
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project aedev.namespace_root_tpls v0.3.33 -->
-# shell 0.3.16
+# shell 0.3.18
 
 [![GitLab develop](https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/develop?logo=python)](
     https://gitlab.com/ae-group/ae_shell)
 [![LatestPyPIrelease](
-    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.16?logo=python)](
-    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.16)
+    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.18?logo=python)](
+    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.18)
 [![PyPIVersions](https://img.shields.io/pypi/v/ae_shell)](
     https://pypi.org/project/ae-shell/#history)
 

ae_shell-0.3.18/ae/shell.py

@@ -0,0 +1,334 @@
+"""
+shell execution and environment helpers
+=======================================
+
+this module is designed to provide a comprehensive set of constants and helper functions
+to manage shell printouts, OS environment variables and to execute shell commands.
+
+* :func:`debug_or_verbose`: checks if the application is running in debug or verbose mode.
+* :func:`get_domain_user_var`: retrieves an OS environment variable value for a specific domain and/or user.
+* :func:`hint`: provides a hint message for console printouts based on the provided arguments.
+* :func:`in_os_env`: context manager to temporarily add environment variables from the ``.env`` files onto `os.environ`.
+* :func:`mask_token`: hide/mask tokens in a text block, to prevent to show them in logs and printouts.
+* :func:`output_line_split`: decode and split the specified shell/console output streams into line chunks.
+* :func:`output_zero_split`: decode and split the specified shell/console output streams separated by `NUL` (\\0) chars.
+* :func:`run_cmd`: execute command in the current working directory of the OS console/shell.
+* :func:`run_logged_cmd`: extended version of :func:`run_cmd` with logging and error checking.
+  after a command is executed and handles application shutdown/termination gracefully.
+
+* :data:`STDERR_BEG_MARKER`: marker used in the console/shell printouts for the beginning of merged-in `stderr` output.
+* :data:`STDERR_END_MARKER`: marker used in the console/shell printouts for the end of merged-in `stderr` output.
+"""
+import os
+import subprocess
+
+from collections.abc import Callable, Iterator, MutableMapping
+from contextlib import contextmanager
+from typing import Any, Sequence, cast, overload
+
+from ae.base import UNSET, UnsetType, dummy_function, env_str, norm_name                    # type: ignore
+from ae.system import load_env_var_defaults, os_env_venv                                    # type: ignore
+from ae.core import main_app_instance, AppBase                                              # type: ignore
+from ae.console import MAIN_SECTION_NAME, ConsoleApp                                        # type: ignore
+
+
+__version__ = '0.3.18'
+
+
+STDERR_BEG_MARKER = 'vvv   STDERR   vvv'  #: :paramref:`ae.shell.run_cmd.output_lines` begin `stderr` lines marker
+STDERR_END_MARKER = '^^^   STDERR   ^^^'  #: end `stderr` lines marker in :paramref:`ae.shell.run_cmd.output_lines`
+
+
+def debug_or_verbose(app_obj: ConsoleApp | UnsetType | None = None) -> bool:
+    """ determine if the current app runs in debug|verbose mode, while preventing early .get_option() call an app init.
+
+    :param app_obj:             optional ConsoleApp instance (def=main_app_instance()).
+    :return:                    a boolean False when the main app debug level is :data:`~ae.core.DEBUG_LEVEL_DISABLED`
+                                and the app option --more_verbose is not specified (in cfg-file or at the command line),
+                                else True.
+
+    .. note:: the return value on app startup/initialization, before the command line parsing, is always True.
+
+    .. hint::
+        the debug mode can be activated via the :class:`~ae.console.ConsoleApp` option `debug_level`, specified either
+        in a config file or via the command line options. the verbose mode get activated via the `more_verbose`  option.
+    """
+    if app_obj is None:
+        app_obj = main_app_instance()
+
+    return bool(
+        not isinstance(app_obj, AppBase)                    # prevent exception in early app startup and unit test runs
+        or app_obj.debug                                    # main_app.debug_level > DEBUG_LEVEL_DISABLED
+        or not isinstance(app_obj, ConsoleApp)              # return True for pure ae.core.AppBase instances
+        or not getattr(app_obj, '_parsed_arguments', None)  # ConsoleApp._parsed_arguments args Namespace not created
+        or app_obj.get_option('more_verbose'))              # optional ConsoleApp instance verbose option
+
+
+def get_domain_user_var(variable_name: str, domain: str = "", user: str = "") -> Any:
+    """ determine the value of an OS environment variable for a specific domain and/or username.
+
+    :param variable_name:       name of the config variable.
+    :param domain:              name of the domain.
+    :param user:                name/id of the user to get a user-specific value of.
+    :return:                    domain/user-specific value of the specified config variable.
+    """
+    parts = (MAIN_SECTION_NAME, variable_name.lower(), f'AT_{norm_name(domain)}'.lower(), norm_name(user).lower())
+    value = None
+    if domain:
+        if user:
+            value = env_str('_'.join(parts), convert_name=True)
+        if value is None:
+            value = env_str('_'.join(parts[:-1]), convert_name=True)
+    elif user:
+        value = env_str('_'.join(parts[:2] + parts[-1:]), convert_name=True)
+
+    if value is None:
+        value = env_str('_'.join(parts[:2]), convert_name=True)
+
+    return value
+
+
+def hint(command: str, action: Callable | str, message_suffix: str = "") -> str:
+    """ return hint string in debug/verbose mode, to be appended onto a shell/console output.
+
+    :param command:             shell command.
+    :param action:              shell command action function/method.
+    :param message_suffix:      extra message text, added to the end of the returned console output string.
+    :return:                    in debug/verbose mode return a string with leading line feed to be sent
+                                to console output, else return an empty string.
+    """
+    if not isinstance(action, str):
+        action = action.__name__
+    return f"{os.linesep}      (run: {command} {action}{message_suffix})" if debug_or_verbose() else ""
+
+
+@contextmanager
+def in_os_env(start_dir: str = "") -> Iterator[MutableMapping[str, str]]:
+    """ temporarily add environment variables from the ``.env`` files that not exist in :attr:`os.environ` to it.
+
+    :param start_dir:           path to the folder where the first dotenv file (with the highest priority) is stored.
+    :return:                    yielding the OS env variables that got added to os.environ in this temporary context.
+    """
+    loaded_env_vars = load_env_var_defaults(start_dir, os.environ)
+    try:
+        yield loaded_env_vars
+    finally:
+        for var_name in loaded_env_vars:
+            os.environ.pop(var_name)
+
+
+@overload
+def mask_token(text: str) -> str: ...       # type: ignore[overload-overlap]
+
+
+@overload
+def mask_token(text: Sequence[str]) -> list[str]: ...
+
+
+def mask_token(text: str | Sequence[str]) -> str | list[str]:
+    """ hide most parts of any Codeberg/GitHub/GitHub URL tokens found in the specified text/-lines.
+
+    :param text:                text, specified either as str object or as a list of str objects (lines),
+                                each str/line get searched for URL tokens, to hide/mask the most part of them.
+    :return:                    text with masked URL tokens (only leaving the first/last 3 token characters unmasked).
+
+    .. note:: see also :func:`ae.base.mask_url` of a more generic way to hide passwords and tokens in URLs.
+    """
+    if is_str_arg := isinstance(text, str):
+        lines = [text]
+    else:
+        lines = list(text)  # copy to not change text list content
+
+    url_beg = 'https://'  # PDV_REPO_HOST_PROTOCOL
+    url_beg_len = len(url_beg)
+    for tok_beg, tok_end in ((':', '@codeberg.org'), ('glpat-', '@gitlab.com'), ('ghp_', '@github.com')):
+        for idx, line in enumerate(lines):
+            beg = -1
+            while ((beg := line.find(url_beg, beg + 1)) != -1 and
+                   (beg := line.find(tok_beg, beg + url_beg_len)) != -1 and
+                   (end := line.find(tok_end, beg)) != -1):
+                line = line[:beg + 3] + "***-masked-token-***" + line[end - 3:]
+            lines[idx] = line
+
+    return lines[0] if is_str_arg else lines
+
+
+def output_line_split(output: bytes) -> list[str]:
+    """ decode and split the specified shell/console output streams into line chunks.
+
+    :param output:              captured `stdout`/`stderr` output from an executed OS shell command.
+    :return:                    list of non-empty shell/console output lines, decoded into string.
+    """
+    return [line for line in output.decode().splitlines() if line.strip()]
+
+
+def output_zero_split(output: bytes) -> list[str]:
+    """ decode and split the specified shell/console output streams separated by `NUL` (\\0) characters.
+
+    :param output:              captured `stdout`/`stderr` output from an executed OS shell command (e.g. `env -0`).
+    :return:                    list of non-empty shell/console output chunks, decoded into string.
+    """
+    return [line for line in output.decode().split('\0') if line.strip()]
+
+
+def run_cmd(*cmd_args: str,
+            output_lines: list[str] | None = None,
+            app_obj: AppBase | UnsetType | None = None,
+            decoder_splitter: Callable[[bytes], list[str]] = output_line_split,
+            **run_kwargs) -> int:
+    """ run command in the current working directory, capturing/logging errors and optionally returning console output.
+
+    :param cmd_args:            command line string or args sequence of command name and arguments
+                                to be run/executed on the console/shell via :func:`subprocess.run`.
+    :param output_lines:        specify a list to be extended with the lines printed on the console/shell `stdout`
+                                stream, and to also hide this output on the console. if and how the `stderr` stream get
+                                also either hidden, captured or to be redirected to this list can be controlled via the
+                                value passed in the kwarg :paramref:`~subprocess.Popen.stderr`: if this argument is a
+                                list, and you specified the argument value :data:`subprocess.PIPE`, then the `stderr`
+                                output will get added at the end of this list (enclosed between the list items
+                                :data:`STDERR_BEG_MARKER` and :data:`STDERR_END_MARKER`). if this argument is a list,
+                                and you specified the argument value :data:`subprocess.STDOUT` then the `stderr` outputs
+                                will get merged without any markers into this list in the order they get printed.
+                                specify :data:`subprocess.DEVNULL` in :paramref:`~subprocess.Popen.stderr` to hide any
+                                `stderr` output onto the console/shell as well as in this list argument.
+                                specifying `None` as :paramref:`~subprocess.Popen.stderr` argument value
+                                (the default argument) then the `stderr` output will be printed only on the console.
+    :param app_obj:             optional :class:`~ae.core.AppBase`/:class:`~ae.console.ConsoleApp` instance, used for
+                                logging. if not specified or None and if :func:`~ae.core.main_app_instance()` returns
+                                None then the Python :func:`print` function is used.
+                                specify :data:`~ae.base.UNSET` to suppress any printing/logging output.
+    :param decoder_splitter:    callable to decode and split the output from the `stdout`/`stderr` streams into a list
+                                of string chunks/lines, to be added to and returned by :paramref:`.output_lines`.
+    :param run_kwargs:          kwargs to be passed onto :func:`subprocess.run`, most of them unchanged, like e.g.
+                                :paramref:`~subprocess.run.input`. some of them, will get adopted/changed before
+                                they get passed onto :func:`subprocess.run`:
+
+                                * :paramref:`~subprocess.run.check`: will get passed onto :func:`subprocess.run`
+                                  as `True` if not specified in this kwarg (in order to catch and log the
+                                  :class:`subprocess.CalledProcessError` exception in debug mode).
+                                * :paramref:`~subprocess.Popen.env`: shell environment variables to be used instead of
+                                  the currently set OS shell variable values.
+                                  only if the value of this argument does not get specified or has the value `UNSET`,
+                                  then an isolated dict copy of :attr:`os.environ` will get passed onto
+                                  :func:`subprocess.run` in order to avoid potential runtime errors, caused by a parent
+                                  process (or a concurrent thread) if it modifies :attr:`os.environ` during the creation
+                                  of this subprocess. additionally, with the convertion of the special :class:`_Environ`
+                                  object into a standard dict, the execution will result slightly faster and avoids
+                                  any special method overrides interfering with the child process creation. it also is
+                                  preventing rare errors with multithread-processes that try to change OS env variable
+                                  (e.g. Conda in relation with pip could lead to raise a `RuntimeError: dictionary
+                                  changed size during iteration` exception). specify `None` as the :paramref:`.env`
+                                  value in order to use the original/unisolated :attr:`os.environ` object.
+                                * :paramref:`~subprocess.Popen.stdout`: will be passed to :func:`subprocess.run`
+                                  as :data:`subprocess.PIPE` (instead of None) if a list instance got specified
+                                  as the :paramref:`.output_lines` argument.
+
+    :return:                    return code of the executed command or 126 if execution raised any other exception.
+    """
+    masked_args = mask_token(cmd_args)
+
+    if app_obj is None:
+        app_obj = main_app_instance()
+    print_out = dummy_function if app_obj is UNSET else app_obj.print_out if isinstance(app_obj, AppBase) else print
+    debug_out = dummy_function if app_obj is UNSET else app_obj.debug_out if isinstance(app_obj, AppBase) else print
+
+    run_kwargs.setdefault('check', True)
+    if run_kwargs.get('env', UNSET) is UNSET:
+        run_kwargs['env'] = os.environ.copy()
+    if isinstance(output_lines, list):
+        run_kwargs.setdefault('stdout', subprocess.PIPE)
+
+    debug_out(f"    . executing {masked_args} at {os.getcwd()=} in {os_env_venv()=}")
+
+    result: subprocess.CompletedProcess | subprocess.CalledProcessError         # having: stdout/stderr/returncode
+    try:
+        result = subprocess.run(cmd_args, **run_kwargs)                         # pylint: disable=subprocess-run-check
+    except subprocess.CalledProcessError as exc:
+        debug_out(f"****  subprocess.run({masked_args}) returned non-zero exit code {exc.returncode}; {exc=}")
+        result = exc
+    except Exception as exc:                                         # pylint: disable=broad-except
+        print_out(f"****  subprocess.run({masked_args}) raised exception {exc}")
+        return (126, )[0]       # put return/exit code into tuple for global code search
+
+    if isinstance(output_lines, list):
+        if result.stdout:
+            output_lines.extend(decoder_splitter(result.stdout))
+        if result.stderr and run_kwargs.get('stderr', None) == subprocess.PIPE:
+            output_lines.append(STDERR_BEG_MARKER)
+            output_lines.extend(decoder_splitter(result.stderr))
+            output_lines.append(STDERR_END_MARKER)
+
+    return result.returncode
+
+
+def run_logged_cmd(err_code: int, *cmd_args: str,
+                   output_lines: list[str] | None = None,
+                   exit_on_err: bool | str = True,
+                   app_obj: ConsoleApp | UnsetType | None = None,
+                   decoder_splitter: Callable[[bytes], list[str]] = output_line_split,
+                   **run_kwargs) -> int:
+    """ run command in the current working directory, optionally capturing/logging console output and exiting on error.
+
+    :param err_code:            error code to be passed onto the console as exit code if the command set an error code
+                                and the value of the :paramref:`.exit_on_err` argument is `True` or a nonempty string.
+    :param cmd_args:            command line string or a sequence of command name and separate line arguments
+                                to be run/executed on the console/shell.
+    :param output_lines:        optional list extended with the lines printed to `stdout`/`stderr` on execution.
+                                see the :paramref:`~run_cmd.output_lines` argument of :func:`run_cmd` for more details.
+    :param exit_on_err:         specifying `True` (the default) or a nonempty string will shut-down/quit/exit the
+                                currently running app (the Python interpreter) if the executed command set an
+                                error code. a nonempty string will get printed/logged to `stdout` before the exit.
+                                pass `False` or an empty string to not exit the app if a command error occurred.
+    :param app_obj:             optional :class:`~ae.console.ConsoleApp` instance, used for logging and error checking
+                                (with the option to ignore errors if the app got started with the `--force` option).
+                                if not specified or None and if :func:`~ae.core.main_app_instance()` returns
+                                None then the Python :func:`print` function is used for logging.
+                                specify :data:`~ae.base.UNSET` to suppress any printing/logging output.
+    :param decoder_splitter:    callable to decode and split the output from the `stdout`/`stderr` streams into a list
+                                of string chunks/lines, to be added to and returned by :paramref:`.output_lines`.
+    :param run_kwargs:          extra kwargs to be passed onto :func:`run_cmd` and :func:`subprocess.run`. some kwargs
+                                like e.g. :paramref:`~subprocess.run.input` and :paramref:`~subprocess.Popen.shell`
+                                will get passed unchanged onto :func:`run_cmd`, others like
+                                :paramref:`~run_cmd.env` or :paramref:`~run_cmd.stderr`, will be first processed by
+                                this function and/or :func:`run_cmd` before they get passed onto :func:`subprocess.run`:
+
+                                * :paramref:`~subprocess.Popen.stderr`: controls how the output onto `stderr` will get
+                                  captured, redirected and/or returned. if this argument is not specified then the value
+                                  :data:`subprocess.DEVNULL` will get passed onto :func:`run_cmd` and
+                                  :func:`subprocess.run`, which is suppressing any output sent onto `stderr` on the
+                                  console as well as any addition of it to the list argument specified in
+                                  :paramref:`.output_lines`. if you specify the value :data:`subprocess.PIPE` or
+                                  :data:`subprocess.STDOUT` together with a list in the :paramref:`.output_lines`
+                                  argument, then any output onto `stderr` will get added to this list.
+                                  see also the description of the :paramref:`~run_cmd.output_lines` parameter of
+                                  :func:`run_cmd` for more details on the supported values of this parameter.
+
+    :return:                    0 on success, or if an error occurred the error number set by the executed command.
+    """
+    if output_lines is None:
+        output_lines = []
+        output_len = 0
+    else:
+        output_len = len(output_lines)
+
+    if app_obj is None:
+        app_obj = cast(ConsoleApp, main_app_instance())  # calls ConsoleApp./app_obj.chk() method
+
+    run_kwargs.setdefault('stderr', subprocess.DEVNULL)
+
+    sh_err = run_cmd(*cmd_args, output_lines=output_lines, app_obj=app_obj, decoder_splitter=decoder_splitter,
+                     **run_kwargs)
+
+    if isinstance(app_obj, ConsoleApp) and (app_obj.debug or sh_err and exit_on_err):
+        for line in output_lines[output_len:]:
+            if app_obj.verbose or not line.startswith("LOG:  "):  # if verbose show mypy's endless (stderr) log entries
+                app_obj.po(" " * 6 + line)
+        command = mask_token(cmd_args)
+        if sh_err == 0:
+            app_obj.dpo(f"    = successfully executed {command=}")
+        else:
+            if isinstance(exit_on_err, str):
+                app_obj.po(f"      {exit_on_err}")
+            app_obj.chk(err_code, not bool(exit_on_err), f"run_logged_cmd({err_code}, {command!r}) error {sh_err}")
+
+    return sh_err

{ae_shell-0.3.16 → ae_shell-0.3.18/ae_shell.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ae_shell
-Version: 0.3.16
+Version: 0.3.18
 Summary: ae namespace module portion shell: shell execution and environment helpers
 Home-page: https://gitlab.com/ae-group/ae_shell
 Author: AndiEcker
@@ -64,13 +64,13 @@ Dynamic: summary
 
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project ae.ae v0.3.110 -->
 <!-- THIS FILE IS EXCLUSIVELY MAINTAINED by the project aedev.namespace_root_tpls v0.3.33 -->
-# shell 0.3.16
+# shell 0.3.18
 
 [![GitLab develop](https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/develop?logo=python)](
     https://gitlab.com/ae-group/ae_shell)
 [![LatestPyPIrelease](
-    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.16?logo=python)](
-    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.16)
+    https://img.shields.io/gitlab/pipeline/ae-group/ae_shell/release0.3.18?logo=python)](
+    https://gitlab.com/ae-group/ae_shell/-/tree/release0.3.18)
 [![PyPIVersions](https://img.shields.io/pypi/v/ae_shell)](
     https://pypi.org/project/ae-shell/#history)
 

{ae_shell-0.3.16 → ae_shell-0.3.18}/setup.py

@@ -77,7 +77,7 @@ setup_kwargs: dict[str, Any] = {
     },
     'python_requires': '>=3.12',
     'url': 'https://gitlab.com/ae-group/ae_shell',
-    'version': '0.3.16',
+    'version': '0.3.18',
     'zip_safe': True,
 }
 

{ae_shell-0.3.16 → ae_shell-0.3.18}/tests/test_shell.py

@@ -6,14 +6,15 @@ import subprocess
 from unittest.mock import PropertyMock, patch
 
 from ae.base import UNSET, camel_to_snake, norm_name, os_path_join, write_file
-from ae.system import load_env_var_defaults, active_venv
+from ae.system import load_env_var_defaults, os_env_venv
 from ae.core import DEBUG_LEVEL_DISABLED, DEBUG_LEVEL_ENABLED, DEBUG_LEVEL_VERBOSE
 from ae.console import MAIN_SECTION_NAME, ConsoleApp
 
 
 from ae.shell import (
     STDERR_BEG_MARKER, STDERR_END_MARKER,
-    debug_or_verbose, get_domain_user_var, hint, in_os_env, mask_token, sh_exec, sh_exit_if_exec_err)
+    debug_or_verbose, get_domain_user_var, hint, in_os_env, mask_token,
+    output_line_split, output_zero_split, run_cmd, run_logged_cmd)
 
 
 class TestHelpers:
@@ -257,68 +258,92 @@ STDERR_LINE = b'std___err'
 
 
 class TestShellExecutions:
-    def test_sh_exec_catch_any_exception(self, capsys):
+    def test_output_line_split(self):
+        assert output_line_split(STDOUT_LINE) == [STDOUT_LINE.decode()]
+
+        assert output_line_split(STDOUT_LINE + b"\n" + STDERR_LINE + b"\n") == [STDOUT_LINE.decode(),
+                                                                                STDERR_LINE.decode()]
+
+        assert output_line_split(STDOUT_LINE + b"\n\0" + STDERR_LINE + b"\0") == [
+            STDOUT_LINE.decode(), "\0" + STDERR_LINE.decode() + "\0"]
+
+        assert output_line_split(STDOUT_LINE + b"\r" + STDERR_LINE + b"\r\n\r\0\r") == [
+            STDOUT_LINE.decode(), STDERR_LINE.decode(), "\0"]
+
+    def test_output_zero_split(self):
+        assert output_zero_split(STDOUT_LINE) == [STDOUT_LINE.decode()]
+
+        assert output_zero_split(STDOUT_LINE + b"\0" + STDERR_LINE + b"\0") == [STDOUT_LINE.decode(),
+                                                                                STDERR_LINE.decode()]
+
+        assert output_zero_split(STDOUT_LINE + b"\0" + STDERR_LINE + b" = multi\nval\rlines\r\n" + STDERR_LINE) == [
+            STDOUT_LINE.decode(), STDERR_LINE.decode() + " = multi\nval\rlines\r\n" + STDERR_LINE.decode()]
+
+        assert output_zero_split(STDOUT_LINE + b"\n" + STDERR_LINE + b"\r\n\r\0\r") == [
+            STDOUT_LINE.decode() + "\n" + STDERR_LINE.decode() + "\r\n\r"]
+
+    def test_run_cmd_catch_any_exception(self, capsys):
         with patch("subprocess.run", side_effect=Exception('broad tst exception')):
-            assert sh_exec('any_cmd') == (126, )[0]
+            assert run_cmd('any_cmd') == (126, )[0]
 
         output = capsys.readouterr().out
         assert 'any_cmd' in output
         assert " raised exception " in output
         assert 'broad tst exception' in output
 
-    def test_sh_exec_catch_exit_code_exception(self, capsys):
-        assert sh_exec("exit 69", shell=True) == 69
+    def test_run_cmd_catch_exit_code_exception(self, capsys):
+        assert run_cmd("exit 69", shell=True) == 69
 
         output = capsys.readouterr().out
         assert " returned non-zero exit code " in output
 
-        assert sh_exec(sys.executable, extra_args=["-c", "import sys; sys.exit(96)"]) == 96
+        assert run_cmd(sys.executable, "-c", "import sys; sys.exit(96)") == 96
 
         output = capsys.readouterr().out
         assert " returned non-zero exit code 96" in output
 
-    def test_sh_exec_console_output(self, capsys, cons_app):
-        sh_exec('any command')
+    def test_run_cmd_console_output(self, capsys, cons_app):
+        run_cmd('any command')
 
         out, err = capsys.readouterr()
-        assert "    . executing ['any', 'command']" in out
+        assert "    . executing ['any command']" in out
         assert os.getcwd() in out
-        assert active_venv() in out
-        assert "****  subprocess.run(['any', 'command']) raised exception" in out
+        assert os_env_venv() in out
+        assert "****  subprocess.run(['any command']) raised exception" in out
         assert err == ""
 
         with patch('ae.console.ConsoleApp.debug_level', new_callable=PropertyMock, return_value=DEBUG_LEVEL_DISABLED):
-            sh_exec('any command')
+            run_cmd('any command')
 
         out, err = capsys.readouterr()
         assert "    . executing" not in out
-        assert "****  subprocess.run(['any', 'command']) raised exception" in out
+        assert "****  subprocess.run(['any command']) raised exception" in out
         assert err == ""
 
         with patch('ae.console.ConsoleApp.debug_level', new_callable=PropertyMock, return_value=DEBUG_LEVEL_DISABLED):
-            sh_exec('any command', app_obj=cons_app)    # strange: patch('ae.core.AppBase.debug_level') does not work
+            run_cmd("any", "command", app_obj=cons_app)    # strange: patch('ae.core.AppBase.debug_level') does not work
 
         out, err = capsys.readouterr()
         assert "    . executing" not in out
         assert "****  subprocess.run(['any', 'command']) raised exception" in out
         assert err == ""
 
-        sh_exec('any command', app_obj=UNSET)
+        run_cmd('any command', app_obj=UNSET)
 
         out, err = capsys.readouterr()
         assert out == ""
         assert err == ""
 
-    def test_sh_exec_console_output_shell(self, capsys):
-        ret = sh_exec("echo hello world", shell=True)
+    def test_run_cmd_console_output_shell(self, capsys):
+        ret = run_cmd("echo hello world", shell=True)
 
         out, err = capsys.readouterr()
         assert ret == 0
-        assert "    . executing echo hello world at " in out
+        assert "    . executing ['echo hello world'] at " in out
         assert out.count("hello world") == 1   # strange: capsys does not get the echo command output w/ shell=True arg
         assert err == ""
 
-        ret = sh_exec("echo hello world", app_obj=UNSET, shell=True)
+        ret = run_cmd("echo hello world", app_obj=UNSET, shell=True)
 
         out, err = capsys.readouterr()
         assert ret == 0
@@ -326,53 +351,43 @@ class TestShellExecutions:
         assert err == ""
 
     @patch.object(subprocess, 'run', autospec=True)
-    def test_sh_exec_run_args(self, mock_method):
+    def test_run_cmd_run_args(self, mock_method):
         cmd_line = "cmd arg1 arg2"
         extra_args = ['extra_arg1', 'extra_arg2']
+        env = os.environ.copy()
 
-        sh_exec(cmd_line, tuple(extra_args))
+        run_cmd(*(shlex.split(cmd_line) + extra_args))
 
-        mock_method.assert_called_with(
-            shlex.split(cmd_line) + extra_args, stdout=None, stderr=None, input=b'', check=True, shell=False, env=None)
+        mock_method.assert_called_with(tuple(shlex.split(cmd_line) + extra_args), check=True, env=env)
 
-        sh_exec(cmd_line, shell=True)
+        run_cmd(cmd_line, shell=True)
 
-        mock_method.assert_called_with(
-            cmd_line, stdout=None, stderr=None, input=b'', check=True, shell=True, env=None)
+        mock_method.assert_called_with((cmd_line, ), check=True, shell=True, env=env)
 
-        sh_exec(cmd_line, {_: "any" for _ in extra_args}, shell=True, err_redirect=subprocess.DEVNULL)
+        run_cmd(cmd_line, *extra_args, input='con_inp')
 
-        mock_method.assert_called_with(
-            cmd_line + " " + " ".join(extra_args), stdout=None, stderr=subprocess.DEVNULL, input=b'', check=True,
-            shell=True, env=None)
+        mock_method.assert_called_with((cmd_line, ) + tuple(extra_args), input='con_inp', check=True, env=env)
 
-        sh_exec(cmd_line, extra_args, console_input='con_inp')
+        run_cmd(cmd_line, *extra_args, output_lines=[])
 
-        mock_method.assert_called_with(
-            shlex.split(cmd_line) + extra_args, stdout=None, stderr=None, input=b'con_inp',
-            check=True, shell=False, env=None)
+        mock_method.assert_called_with((cmd_line, ) + tuple(extra_args), check=True, stdout=subprocess.PIPE, env=env)
 
-        sh_exec(cmd_line, extra_args, output_lines=[])
+        run_cmd(cmd_line, *extra_args, input='con_inp', output_lines=[], stderr=subprocess.STDOUT)
 
-        mock_method.assert_called_with(
-            shlex.split(cmd_line) + extra_args, stdout=subprocess.PIPE, stderr=None, input=b'',
-            check=True, shell=False, env=None)
+        mock_method.assert_called_with((cmd_line, ) + tuple(extra_args), input='con_inp',
+                                       stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=True, env=env)
 
-        sh_exec(cmd_line, extra_args, console_input='con_inp', output_lines=[], err_redirect=subprocess.STDOUT)
+        env_vars = {'A': "1", 'C': "tst_string value"}
 
-        mock_method.assert_called_with(
-            shlex.split(cmd_line) + extra_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=b'con_inp',
-            check=True, shell=False, env=None)
+        run_cmd(cmd_line, *extra_args, env=env_vars)
 
-        env_vars = {'A': "1", 'C': "tst_string value"}
+        mock_method.assert_called_with((cmd_line, ) + tuple(extra_args), env=env_vars, check=True)
 
-        sh_exec(cmd_line, extra_args, env_vars=env_vars)
+        run_cmd(cmd_line, *extra_args, env=None)
 
-        mock_method.assert_called_with(
-            shlex.split(cmd_line) + extra_args, stdout=None, stderr=None, input=b'',
-            check=True, shell=False, env=env_vars)
+        mock_method.assert_called_with((cmd_line, ) + tuple(extra_args), check=True, env=None)
 
-    def test_sh_exec_run_returned_values(self):
+    def test_run_cmd_run_returned_values(self):
         def _run_return(*_args, **_kwargs):
             """ mock to simulate subprocess.run return object. """
             class _Return:
@@ -384,7 +399,7 @@ class TestShellExecutions:
         with patch('ae.shell.subprocess.run', new=_run_return):  # @patch.object(subprocess, 'run', new=_run_return)
             output_lines = []
 
-            assert sh_exec("cmd_line", output_lines=output_lines, err_redirect=subprocess.PIPE) == RETURN_CODE
+            assert run_cmd("cmd_line", output_lines=output_lines, stderr=subprocess.PIPE) == RETURN_CODE
 
             assert output_lines[0] == STDOUT_LINE.decode()
             assert output_lines[1] == STDERR_BEG_MARKER
@@ -393,7 +408,7 @@ class TestShellExecutions:
 
             output_lines = []
 
-            assert sh_exec("cmd_line", output_lines=output_lines) == RETURN_CODE
+            assert run_cmd("cmd_line", output_lines=output_lines) == RETURN_CODE
 
             assert len(output_lines) == 1
             assert output_lines[0] == STDOUT_LINE.decode()
@@ -403,7 +418,7 @@ class TestShellExecutions:
 
             output_lines = ['first line', 'second line']
 
-            assert sh_exec("cmd_line", output_lines=output_lines, err_redirect=subprocess.STDOUT) == RETURN_CODE
+            assert run_cmd("cmd_line", output_lines=output_lines, err_redirect=subprocess.STDOUT) == RETURN_CODE
 
             assert len(output_lines) == 3
             assert output_lines[0] == 'first line'
@@ -413,11 +428,10 @@ class TestShellExecutions:
             assert STDERR_BEG_MARKER not in output_lines
             assert STDERR_END_MARKER not in output_lines
 
-    def test_sh_exec_stderr_redirection(self, capfd, cons_app):  # pytest/capsys replaces sys.stdout/.stderr
-        args = {'command_line': sys.executable,
-                'extra_args': ["-c", "import sys; print('tst_std_err', file=sys.stderr); print('tst_std_out')"]}
+    def test_run_cmd_stderr_redirection(self, capfd, cons_app):  # pytest/capsys replaces sys.stdout/.stderr
+        args = [sys.executable, "-c", "import sys; print('tst_std_err', file=sys.stderr); print('tst_std_out')"]
 
-        assert sh_exec(**args) == 0
+        assert run_cmd(*args) == 0
 
         out, err = capfd.readouterr()
         assert out.count('tst_std_out') == 2
@@ -426,7 +440,7 @@ class TestShellExecutions:
 
         redirected = []
 
-        assert sh_exec(**args, output_lines=redirected) == 0
+        assert run_cmd(*args, output_lines=redirected) == 0
 
         out, err = capfd.readouterr()
         assert out.count('tst_std_out') == 1
@@ -436,7 +450,7 @@ class TestShellExecutions:
 
         redirected = []
 
-        assert sh_exec(**args, output_lines=redirected, err_redirect=subprocess.DEVNULL) == 0
+        assert run_cmd(*args, output_lines=redirected, stderr=subprocess.DEVNULL) == 0
 
         out, err = capfd.readouterr()
         assert err == ''
@@ -444,7 +458,7 @@ class TestShellExecutions:
 
         redirected = []
 
-        assert sh_exec(**args, output_lines=redirected, err_redirect=subprocess.PIPE) == 0
+        assert run_cmd(*args, output_lines=redirected, stderr=subprocess.PIPE) == 0
 
         out, err = capfd.readouterr()
         assert err == ''
@@ -452,18 +466,17 @@ class TestShellExecutions:
 
         redirected = []
 
-        assert sh_exec(**args, output_lines=redirected, err_redirect=subprocess.STDOUT) == 0
+        assert run_cmd(*args, output_lines=redirected, stderr=subprocess.STDOUT) == 0
 
         out, err = capfd.readouterr()
         assert err == ''
         assert redirected == ['tst_std_err', 'tst_std_out']
 
-    def test_sh_exit_if_exec_err_any_command(self, capsys, cons_app):
+    def test_run_logged_cmd_any_command(self, capsys, cons_app):
         output = ['old output']
 
         with patch('ae.console.ConsoleApp.debug', new_callable=PropertyMock, return_value=True):  # with app_obj kwarg
-            ret = sh_exit_if_exec_err(0, 'git --version', output_lines=output,
-                                      exit_on_err=False, exit_msg='tst exit message', app_obj=cons_app)
+            ret = run_logged_cmd(0, "git", "--version", output_lines=output, exit_on_err=False, app_obj=cons_app)
 
         assert ret == 0
         assert len(output) >= 2
@@ -471,49 +484,46 @@ class TestShellExecutions:
         assert isinstance(output[1], str)   # e.g. == 'git version 2.55.0'
         out, err = capsys.readouterr()
         assert 'old output' not in out
-        assert 'git --version' in out
-        assert 'tst exit message' not in out
+        assert "\n    . executing ['git', '--version'] at os.getcwd()='" in out
         assert err == ""
 
-    def test_sh_exit_if_exec_err_any_invalid_command(self, capsys, cons_app, patched_shutdown_wrapper):
-        ret = sh_exit_if_exec_err(693, 'tst_command_line', exit_on_err=False, exit_msg='tst exit message')
+    def test_run_logged_cmd_any_invalid_command(self, capsys, cons_app, patched_shutdown_wrapper):
+        ret = run_logged_cmd(693, 'tst_command_line', exit_on_err=False)
 
         assert ret == (126, )[0]
         out, err = capsys.readouterr()
         assert out.count('tst_command_line') == 3
-        assert 'tst exit message' in out
         assert err == ""
 
-    def test_sh_exit_if_exec_err_caught_shutdown_exception(self, capsys, cons_app, patched_shutdown_wrapper):
-        ret = patched_shutdown_wrapper(sh_exit_if_exec_err, 693, 'tst_command_line', exit_msg='tst exit message')
+    def test_run_logged_cmd_caught_shutdown_exception(self, capsys, cons_app, patched_shutdown_wrapper):
+        ret = patched_shutdown_wrapper(run_logged_cmd, 693, 'tst_command_line', exit_on_err='tst exit message')
 
         assert len(ret) == 1
         assert ret[0]['exit_code'] == 693    # 1st arg == error code
         assert 'tst_command_line' in ret[0]['error_message']
-        assert 'sh_exit_if_exec_err(' in ret[0]['error_message']
+        assert 'run_logged_cmd(' in ret[0]['error_message']
         assert "(693, " in ret[0]['error_message']
         out, err = capsys.readouterr()
         assert out.count('tst_command_line') == 3
         assert 'tst exit message' in out
         assert err == ""
 
-    def test_sh_exit_if_exec_err_exception(self, capsys, cons_app):
+    def test_run_logged_cmd_exception(self, capsys, cons_app):
         output = ['old output']
 
-        ret = sh_exit_if_exec_err(693, "", output_lines=output, exit_on_err=False, exit_msg='tst exit message')
+        ret = run_logged_cmd(693, "", output_lines=output, exit_on_err=False)
 
         assert ret == (126, )[0]
         assert output == ['old output']
         out, err = capsys.readouterr()
-        assert f". executing [] at os.getcwd()='{os.getcwd()}' in active_venv()='{active_venv()}'" in out
-        assert 'tst exit message' in out
+        assert f". executing [''] at os.getcwd()='{os.getcwd()}' in os_env_venv()='{os_env_venv()}'" in out
         assert err == ""
 
-    def test_sh_exit_if_exec_err_with_app(self, capsys, cons_app):
+    def test_run_logged_cmd_with_app(self, capsys, cons_app):
         output = []
 
         with patch('ae.console.ConsoleApp.debug', new_callable=PropertyMock, return_value=False):
-            ret = sh_exit_if_exec_err(0, "_err", output_lines=output, exit_on_err=False, err_redirect=subprocess.STDOUT)
+            ret = run_logged_cmd(0, "_err", output_lines=output, exit_on_err=False, stderr=subprocess.STDOUT)
 
         assert ret == (126, )[0]
         assert output == []
@@ -521,26 +531,24 @@ class TestShellExecutions:
         assert out.count('_err') == 3
         assert err == ""
 
-    def test_sh_exit_if_exec_err_with_app_kwarg_and_exit_msg(self, capsys, cons_app):
+    def test_run_logged_cmd_with_app_kwarg_and_empty_exit_on_err_msg(self, capsys, cons_app):
         output = []
 
         with patch('ae.console.ConsoleApp.debug', new_callable=PropertyMock, return_value=True):  # with app_obj kwarg
-            ret = sh_exit_if_exec_err(0, 'error-command', output_lines=output,
-                                      exit_on_err=False, exit_msg='tst exit message', app_obj=cons_app)
+            ret = run_logged_cmd(0, 'error-command', output_lines=output, exit_on_err="", app_obj=cons_app)
 
         assert ret == (126, )[0]
         assert output == []
         out, err = capsys.readouterr()
-        assert out.count('error-command') == 3
-        assert 'tst exit message' in out
+        assert out.count('error-command') == 3  # extra empty line
         assert err == ""
 
-    def test_sh_exit_if_exec_err_with_app_extending_output(self, capsys, cons_app):
+    def test_run_logged_cmd_with_app_extending_output(self, capsys, cons_app):
         output = ['any old line output']
 
-        with (patch('ae.shell.sh_exec', new=lambda *_, **kwargs: kwargs['output_lines'].append('new out line') or 0),
+        with (patch('ae.shell.run_cmd', new=lambda *_, **kwargs: kwargs['output_lines'].append('new out line') or 0),
               patch('ae.console.ConsoleApp.debug', new_callable=PropertyMock, return_value=True)):
-            ret = sh_exit_if_exec_err(369, "any_cmd", output_lines=output)  # extended output_lines and ret==0
+            ret = run_logged_cmd(369, "any_cmd", output_lines=output)  # extended output_lines and ret==0
 
         assert ret == 0
         assert output == ['any old line output', 'new out line']
@@ -551,14 +559,13 @@ class TestShellExecutions:
         assert out.count('any_cmd') == 1
         assert err == ""
 
-    def test_sh_exit_if_exec_err_without_app(self, capsys):
+    def test_run_logged_cmd_without_app(self, capsys):
         output = []
 
-        ret = sh_exit_if_exec_err(0, "_err_cmd", output_lines=output, exit_on_err=False, exit_msg='tst exit message')
+        ret = run_logged_cmd(0, "_err_cmd", output_lines=output, exit_on_err=False)
 
         assert ret == (126, )[0]
         assert output == []
         out, err = capsys.readouterr()
         assert out.count('_err_cmd') == 3
-        assert 'tst exit message' not in out
         assert err == ""

ae_shell-0.3.16/ae/shell.py

@@ -1,276 +0,0 @@
-"""
-shell execution and environment helpers
-=======================================
-
-this module is designed to provide a comprehensive set of constants and helper functions
-to manage shell printouts, OS environment variables and to execute shell commands.
-
-- :func:`debug_or_verbose`: checks if the application is running in debug or verbose mode.
-- :func:`get_domain_user_var`: retrieves an OS environment variable value for a specific domain and/or user.
-- :func:`hint`: provides a hint message for console printouts based on the provided arguments.
-- :func:`in_os_env`: context manager to temporarily add environment variables from the ``.env`` files onto `os.environ`.
-- :func:`mask_token`: hide/mask tokens in a text block, to prevent to show them in logs and printouts.
-- :func:`sh_exec`: execute command in the current working directory of the OS console/shell.
-- :func:`sh_exit_if_exec_err`: extended version of :func:`sh_exec` with automatically checks for errors
-  after a command is executed and handles application termination gracefully.
-
-- :data:`STDERR_BEG_MARKER`: marker used in the console output for the beginning of merged-in stderr output.
-- :data:`STDERR_END_MARKER`: marker used in the console output for the end of merged-in stderr output.
-"""
-import os
-import shlex
-import subprocess
-
-from collections.abc import Callable, Iterable, Iterator, MutableMapping
-from contextlib import contextmanager
-from typing import Any, cast, overload
-
-from ae.base import UNSET, dummy_function, env_str, norm_name                               # type: ignore
-from ae.system import active_venv, load_env_var_defaults                                    # type: ignore
-from ae.core import main_app_instance, AppBase                                              # type: ignore
-from ae.console import MAIN_SECTION_NAME, ConsoleApp                                        # type: ignore
-
-
-__version__ = '0.3.16'
-
-
-STDERR_BEG_MARKER = 'vvv   STDERR   vvv'  #: :paramref:`ae.shell.sh_exec.output_lines` begin stderr lines marker
-STDERR_END_MARKER = '^^^   STDERR   ^^^'  #: end stderr lines marker in :paramref:`ae.shell.sh_exec.output_lines`
-
-
-def debug_or_verbose(app_obj: ConsoleApp | None = None) -> bool:
-    """ determine if the current app runs in debug|verbose mode, while preventing early .get_option() call an app init.
-
-    :param app_obj:             optional ConsoleApp instance (def=main_app_instance()).
-    :return:                    a boolean False when the main app debug level is :data:`~ae.core.DEBUG_LEVEL_DISABLED`
-                                and the app option --more_verbose is not specified (in cfg-file or at the command line),
-                                else True.
-
-    .. note:: the return value on app startup/initialization, before the command line parsing, is always True.
-
-    .. hint::
-        the debug mode can be activated via the :class:`~ae.console.ConsoleApp` option `debug_level`, specified either
-        in a config file or via the command line options. the verbose mode get activated via the `more_verbose`  option.
-    """
-    app_obj = app_obj or main_app_instance()
-    return bool(
-        not app_obj                                         # prevent exception in early app startup and in test runs
-        or app_obj.debug                                    # main_app.debug_level > DEBUG_LEVEL_DISABLED
-        or not isinstance(app_obj, ConsoleApp)              # return True for pure ae.core.AppBase instances
-        or not getattr(app_obj, '_parsed_arguments', None)  # ConsoleApp._parsed_arguments args Namespace not created
-        or app_obj.get_option('more_verbose'))              # optional ConsoleApp instance verbose option
-
-
-def get_domain_user_var(variable_name: str, domain: str = "", user: str = "") -> Any:
-    """ determine the value of an OS environment variable for a specific domain and/or username.
-
-    :param variable_name:       name of the config variable.
-    :param domain:              name of the domain.
-    :param user:                name/id of the user to get a user-specific value of.
-    :return:                    domain/user-specific value of the specified config variable.
-    """
-    parts = (MAIN_SECTION_NAME, variable_name.lower(), f'AT_{norm_name(domain)}'.lower(), norm_name(user).lower())
-    value = None
-    if domain:
-        if user:
-            value = env_str('_'.join(parts), convert_name=True)
-        if value is None:
-            value = env_str('_'.join(parts[:-1]), convert_name=True)
-    elif user:
-        value = env_str('_'.join(parts[:2] + parts[-1:]), convert_name=True)
-
-    if value is None:
-        value = env_str('_'.join(parts[:2]), convert_name=True)
-
-    return value
-
-
-def hint(command: str, action: Callable | str, message_suffix: str = "") -> str:
-    """ return hint string in debug/verbose mode, to be appended onto a shell/console output.
-
-    :param command:             shell command.
-    :param action:              shell command action function/method.
-    :param message_suffix:      extra message text, added to the end of the returned console output string.
-    :return:                    in debug/verbose mode return a string with leading line feed to be sent
-                                to console output, else return an empty string.
-    """
-    if not isinstance(action, str):
-        action = action.__name__
-    return f"{os.linesep}      (run: {command} {action}{message_suffix})" if debug_or_verbose() else ""
-
-
-@contextmanager
-def in_os_env(start_dir: str = "") -> Iterator[MutableMapping[str, str]]:
-    """ temporarily add environment variables from the dotenv files that not exist in os.environ to it.
-
-    :param start_dir:           path to the folder where the first dotenv file (with the highest priority) is stored.
-    :return:                    yielding the os env variables that got added to os.environ in this temporary context.
-    """
-    loaded_env_vars = load_env_var_defaults(start_dir, os.environ)
-    try:
-        yield loaded_env_vars
-    finally:
-        for var_name in loaded_env_vars:
-            os.environ.pop(var_name)
-
-
-@overload
-def mask_token(text: str) -> str: ...
-
-
-@overload
-def mask_token(text: list[str]) -> list[str]: ...
-
-
-def mask_token(text: str | list[str]) -> str | list[str]:
-    """ hide most parts of any Codeberg/GitHub/GitHub URL tokens found in the specified text/-lines.
-
-    :param text:                text, specified either as str object or as a list of str objects (lines),
-                                each str/line get searched for URL tokens, to hide/mask the most part of them.
-    :return:                    text with masked URL tokens (only leaving the first/last 3 token characters unmasked).
-
-    .. note:: see also :func:`ae.base.mask_url` of a more generic way to hide passwords and tokens in URLs.
-    """
-    if is_str_arg := isinstance(text, str):
-        lines = [text]
-    else:
-        lines = list(text)  # copy to not change text list content
-
-    url_beg = 'https://'  # PDV_REPO_HOST_PROTOCOL
-    url_beg_len = len(url_beg)
-    for tok_beg, tok_end in ((':', '@codeberg.org'), ('glpat-', '@gitlab.com'), ('ghp_', '@github.com')):
-        for idx, line in enumerate(lines):
-            beg = -1
-            while ((beg := line.find(url_beg, beg + 1)) != -1 and
-                   (beg := line.find(tok_beg, beg + url_beg_len)) != -1 and
-                   (end := line.find(tok_end, beg)) != -1):
-                line = line[:beg + 3] + "***-masked-token-***" + line[end - 3:]
-            lines[idx] = line
-
-    return lines[0] if is_str_arg else lines
-
-
-# pylint: disable-next=too-many-arguments,too-many-positional-arguments
-def sh_exec(command_line: str, extra_args: Iterable[str] = (), console_input: str = "",
-            output_lines: list[str] | None = None, app_obj: AppBase | None = None, shell: bool = False,
-            env_vars: dict[str, str] | None = None, err_redirect: int | None = None) -> int:
-    """ execute command in the current working directory of the OS console/shell.
-
-    :param command_line:        command line string to execute on the console/shell. could contain command line args
-                                separated by whitespace characters (alternatively use :paramref:`.extra_args`).
-    :param extra_args:          optional iterable with extra command line arguments.
-    :param console_input:       optional string to be sent to the stdin stream of the console/shell.
-    :param output_lines:        specify a list to be extended with the lines printed on the console/shell stdout stream,
-                                and to also hide this output on the console. if and how the stderr stream get also
-                                hidden/redirected to this list can be controlled via the value passed in the argument
-                                :paramref:`.err_redirect`.
-    :param app_obj:             optional :class:`~ae.core.AppBase`/:class:`~ae.console.ConsoleApp` instance, used for
-                                logging. if not specified or None and if :func:`~ae.core.main_app_instance()` returns
-                                None then the Python :func:`print` function is used.
-                                specify :data:`~ae.base.UNSET` to suppress any printing/logging output.
-    :param shell:               pass True to execute command in the default OS shell (for more info check the
-                                documentation of the parameter :paramref:`~subprocess.run.shell` of the
-                                :meth:`subprocess.run` function).
-    :param env_vars:            OS shell environment variables to be used instead of the console/bash defaults.
-    :param err_redirect:        this argument controls if and how the output of the executed command onto the console
-                                stderr stream gets captured/redirected. it gets passed directly onto the
-                                :paramref:`~subprocess.run.stderr` argument of :func:`subprocess.run` function.
-                                if the argument of :paramref:`.output_lines` is a list, and you specified the
-                                argument value :data:`subprocess.PIPE`, then the stderr output will get added at the
-                                end of this list (enclosed between the list items :data:`STDERR_BEG_MARKER` and
-                                :data:`STDERR_END_MARKER`). if the argument of :paramref:`.output_lines` is a
-                                list, and you specified the argument value :data:`subprocess.STDOUT` then the stderr
-                                output will get merged without any markers into this list in the order they get printed.
-                                specify :data:`subprocess.DEVNULL` to hide any stderr output onto the console/shell
-                                as well as in the :paramref:`.output_lines` list. if you specify `None`
-                                (the default argument) then the stderr output will be printed only on the console.
-    :return:                    return code of the executed command or 126 if execution raised any other exception.
-    """
-    all_args = command_line + (" " + " ".join(extra_args) if extra_args else "") if shell else (
-            shlex.split(command_line) + list(extra_args))
-    # noinspection PyTypeChecker
-    masked_args = mask_token(all_args)
-    if app_obj is None:
-        app_obj = main_app_instance()
-    print_out = app_obj.po if app_obj else dummy_function if app_obj is UNSET else print
-    debug_out = app_obj.dpo if app_obj else dummy_function if app_obj is UNSET else print
-
-    debug_out(f"    . executing {masked_args} at {os.getcwd()=} in {active_venv()=}")
-    result: subprocess.CompletedProcess | subprocess.CalledProcessError     # having: stdout/stderr/returncode
-    try:
-        result = subprocess.run(all_args,
-                                stdout=subprocess.PIPE if isinstance(output_lines, list) else None,
-                                stderr=err_redirect,
-                                input=console_input.encode(),
-                                check=True,
-                                shell=shell,
-                                env=env_vars)
-    except subprocess.CalledProcessError as exc:
-        debug_out(f"****  subprocess.run({masked_args}) returned non-zero exit code {exc.returncode}; {exc=}")
-        result = exc
-    except Exception as exc:                                         # pylint: disable=broad-except
-        print_out(f"****  subprocess.run({masked_args}) raised exception {exc}")
-        return (126, )[0]       # put return/exit code into tuple for global code search
-
-    if isinstance(output_lines, list):
-        if result.stdout:
-            output_lines.extend([line for line in result.stdout.decode().splitlines() if line.strip()])
-        if err_redirect == subprocess.PIPE and result.stderr:
-            output_lines.append(STDERR_BEG_MARKER)
-            output_lines.extend([line for line in result.stderr.decode().splitlines() if line.strip()])
-            output_lines.append(STDERR_END_MARKER)
-
-    return result.returncode
-
-
-# pylint: disable-next=too-many-arguments,too-many-positional-arguments
-def sh_exit_if_exec_err(err_code: int, command_line: str,
-                        extra_args: Iterable[str] = (), output_lines: list[str] | None = None, exit_on_err: bool = True,
-                        exit_msg: str = "", app_obj: ConsoleApp | None = None, shell: bool = False,
-                        env_vars: dict[str, str] | None = None, err_redirect: int | None = subprocess.DEVNULL) -> int:
-    """ execute command in the current working directory, optionally capturing console output and exit app on error.
-
-    :param err_code:            error code to pass to the console as exit code if the command set an error code and
-                                value of the :paramref:`.exit_on_err` argument is `True`.
-    :param command_line:        command line string to execute. this argument could contain additional command line
-                                arguments, separated by whitespace characters. alternatively use the argument
-                                :paramref:`.extra_args` which allows to pass command line argument values,
-                                with containing space characters.
-    :param extra_args:          optional iterable of extra command line arguments.
-    :param output_lines:        optional list extended with the lines printed to stdout/stderr on execution.
-    :param exit_on_err:         pass False to not exit the app on error.
-    :param exit_msg:            additional text to print on stdout/console if the app debug level is greater or equal
-                                to 1 (:data:`~ae.core.DEBUG_LEVEL_ENABLED`) or if an error occurred.
-    :param app_obj:             :class:`~ae.console.ConsoleApp` instance, used for logging/force-ignorable error.
-    :param shell:               pass True to execute command in the default OS shell (see :meth:`subprocess.run`).
-    :param env_vars:            OS shell environment variables to be used instead of the console/bash defaults.
-    :param err_redirect:        control how the output on stderr gets captured, redirected and returned. see also
-                                :paramref:`sh_exec.err_redirect` for more details to the supported argument values.
-                                if this argument is not specified or has the value :data:`subprocess.DEVNULL` then the
-                                stderr output will get suppressed on the console and will also not get added to the
-                                list argument in :paramref:`.output_lines`.
-    :return:                    0 on success, or if an error occurred the error number set by the executed command.
-    """
-    if output_lines is None:
-        output_lines = []
-        output_len = 0
-    else:
-        output_len = len(output_lines)
-    app_obj = app_obj or cast(ConsoleApp, main_app_instance())  # calls app_obj./ConsoleApp.chk() method
-
-    sh_err = sh_exec(command_line, extra_args=extra_args, output_lines=output_lines, app_obj=app_obj, shell=shell,
-                     env_vars=env_vars, err_redirect=err_redirect)
-
-    if app_obj and (app_obj.debug or sh_err and exit_on_err):
-        for line in output_lines[output_len:]:
-            if app_obj.verbose or not line.startswith("LOG:  "):  # if verbose show mypy's endless (stderr) log entries
-                app_obj.po(" " * 6 + line)
-        command = mask_token(f"{command_line} " + " ".join('"' + _a + '"' if " " in _a else _a for _a in extra_args))
-        if sh_err == 0:
-            app_obj.dpo(f"    = successfully executed {command=}")
-        else:
-            if exit_msg:
-                app_obj.po(f"      {exit_msg}")
-            app_obj.chk(err_code, not exit_on_err, f"sh_exit_if_exec_err({err_code}, {command!r}) error {sh_err}")
-
-    return sh_err