ansible-core 2.19.0b6__py3-none-any.whl → 2.19.0b7__py3-none-any.whl

ansible_test/_internal/__init__.py

@@ -59,6 +59,10 @@ from .config import (
     TestConfig,
 )
 
+from .debugging import (
+    initialize_debugger,
+)
+
 
 def main(cli_args: t.Optional[list[str]] = None) -> None:
     """Wrapper around the main program function to invoke cleanup functions at exit."""
@@ -77,6 +81,7 @@ def main_internal(cli_args: t.Optional[list[str]] = None) -> None:
         display.redact = config.redact
         display.color = config.color
         display.fd = sys.stderr if config.display_stderr else sys.stdout
+        initialize_debugger(config)
         configure_timeout(config)
         report_locale(isinstance(config, TestConfig) and not config.delegate)
 

ansible_test/_internal/ansible_util.py

@@ -330,6 +330,6 @@ def run_playbook(
     if args.verbosity:
         cmd.append('-%s' % ('v' * args.verbosity))
 
-    install_requirements(args, args.controller_python, ansible=True)  # run_playbook()
+    install_requirements(args, None, args.controller_python, ansible=True)  # run_playbook()
     env = ansible_environment(args)
     intercept_python(args, args.controller_python, cmd, env, capture=capture)

ansible_test/_internal/classification/python.py

@@ -221,6 +221,12 @@ def relative_to_absolute(name: str, level: int, module: str, path: str, lineno:
     else:
         parts = module.split('.')
 
+        if path.endswith('/__init__.py'):
+            # Ensure the correct relative module is calculated for both not_init.py and __init__.py:
+            # a/b/not_init.py -> a.b.not_init  # used as-is
+            # a/b/__init__.py -> a.b           # needs "__init__" part appended to ensure relative imports work
+            parts.append('__init__')
+
         if level >= len(parts):
             display.warning('Cannot resolve relative import "%s%s" above module "%s" at %s:%d' % ('.' * level, name, module, path, lineno))
             absolute_name = 'relative.abovelevel'

ansible_test/_internal/cli/commands/__init__.py

@@ -162,11 +162,6 @@ def do_commands(
         help='only verify code coverage can be enabled',
     )
 
-    testing.add_argument(
-        '--metadata',
-        help=argparse.SUPPRESS,
-    )
-
     testing.add_argument(
         '--base-branch',
         metavar='BRANCH',

ansible_test/_internal/cli/environments.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import argparse
 import enum
 import functools
+import os
 import typing as t
 
 from ..constants import (
@@ -147,8 +148,20 @@ def add_global_options(
         help='install command requirements',
     )
 
+    global_parser.add_argument(
+        '--host-path',
+        help=argparse.SUPPRESS,  # for internal use only by ansible-test
+    )
+
+    global_parser.add_argument(
+        '--metadata',
+        default=os.environ.get('ANSIBLE_TEST_METADATA_PATH'),
+        help=argparse.SUPPRESS,  # for internal use only by ansible-test
+    )
+
     add_global_remote(global_parser, controller_mode)
     add_global_docker(global_parser, controller_mode)
+    add_global_debug(global_parser)
 
 
 def add_composite_environment_options(
@@ -161,11 +174,6 @@ def add_composite_environment_options(
     composite_parser = t.cast(argparse.ArgumentParser, parser.add_argument_group(
         title='composite environment arguments (mutually exclusive with "environment arguments" above)'))
 
-    composite_parser.add_argument(
-        '--host-path',
-        help=argparse.SUPPRESS,
-    )
-
     action_types: list[t.Type[CompositeAction]] = []
 
     def register_action_type(action_type: t.Type[CompositeAction]) -> t.Type[CompositeAction]:
@@ -440,6 +448,44 @@ def add_global_docker(
     )
 
 
+def add_global_debug(
+    parser: argparse.ArgumentParser,
+) -> None:
+    """Add global debug options."""
+    # These `--dev-*` options are experimental features that may change or be removed without regard for backward compatibility.
+    # Additionally, they're features that are not likely to be used by most users.
+    # To avoid confusion, they're hidden from `--help` and tab completion by default, except for ansible-core-ci users.
+    suppress = None if get_ci_provider().supports_core_ci_auth() else argparse.SUPPRESS
+
+    parser.add_argument(
+        '--dev-debug-on-demand',
+        action='store_true',
+        default=False,
+        help=suppress or 'enable remote debugging only under a debugger',
+    )
+
+    parser.add_argument(
+        '--dev-debug-cli',
+        action='store_true',
+        default=False,
+        help=suppress or 'enable remote debugging for the Ansible CLI',
+    )
+
+    parser.add_argument(
+        '--dev-debug-ansiballz',
+        action='store_true',
+        default=False,
+        help=suppress or 'enable remote debugging for AnsiballZ modules',
+    )
+
+    parser.add_argument(
+        '--dev-debug-self',
+        action='store_true',
+        default=False,
+        help=suppress or 'enable remote debugging for ansible-test',
+    )
+
+
 def add_environment_docker(
     exclusive_parser: argparse.ArgumentParser,
     environments_parser: argparse.ArgumentParser,

ansible_test/_internal/commands/coverage/__init__.py

@@ -77,7 +77,7 @@ class CoverageConfig(EnvironmentConfig):
 def initialize_coverage(args: CoverageConfig, host_state: HostState) -> coverage_module:
     """Delegate execution if requested, install requirements, then import and return the coverage module. Raises an exception if coverage is not available."""
     configure_pypi_proxy(args, host_state.controller_profile)  # coverage
-    install_requirements(args, host_state.controller_profile.python, coverage=True)  # coverage
+    install_requirements(args, host_state.controller_profile, host_state.controller_profile.python, coverage=True)  # coverage
 
     try:
         import coverage

ansible_test/_internal/commands/integration/__init__.py

@@ -105,6 +105,7 @@ from ...host_profiles import (
     HostProfile,
     PosixProfile,
     SshTargetHostProfile,
+    DebuggableProfile,
 )
 
 from ...provisioning import (
@@ -459,10 +460,10 @@ def command_integration_filtered(
 
         if isinstance(target_profile, ControllerProfile):
             if host_state.controller_profile.python.path != target_profile.python.path:
-                install_requirements(args, target_python, command=True, controller=False)  # integration
+                install_requirements(args, target_profile, target_python, command=True, controller=False)  # integration
         elif isinstance(target_profile, SshTargetHostProfile):
             connection = target_profile.get_controller_target_connections()[0]
-            install_requirements(args, target_python, command=True, controller=False, connection=connection)  # integration
+            install_requirements(args, target_profile, target_python, command=True, controller=False, connection=connection)  # integration
 
     coverage_manager = CoverageManager(args, host_state, inventory_path)
     coverage_manager.setup()
@@ -616,7 +617,7 @@ def command_integration_script(
         if args.verbosity:
             cmd.append('-' + ('v' * args.verbosity))
 
-        env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env)
+        env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env, host_state)
         cwd = os.path.join(test_env.targets_dir, target.relative_path)
 
         env.update(
@@ -737,7 +738,7 @@ def command_integration_role(
             if args.verbosity:
                 cmd.append('-' + ('v' * args.verbosity))
 
-            env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env)
+            env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env, host_state)
             cwd = test_env.integration_dir
 
             env.update(
@@ -793,6 +794,7 @@ def integration_environment(
     ansible_config: t.Optional[str],
     env_config: t.Optional[CloudEnvironmentConfig],
     test_env: IntegrationEnvironment,
+    host_state: HostState,
 ) -> dict[str, str]:
     """Return a dictionary of environment variables to use when running the given integration test target."""
     env = ansible_environment(args, ansible_config=ansible_config)
@@ -813,6 +815,9 @@ def integration_environment(
     if args.debug_strategy:
         env.update(ANSIBLE_STRATEGY='debug')
 
+    if isinstance(host_state.controller_profile, DebuggableProfile):
+        env.update(host_state.controller_profile.get_ansible_cli_environment_variables())
+
     if 'non_local/' in target.aliases:
         if args.coverage:
             display.warning('Skipping coverage reporting on Ansible modules for non-local test: %s' % target.name)
@@ -820,6 +825,7 @@ def integration_environment(
         env.update(ANSIBLE_TEST_REMOTE_INTERPRETER='')
 
     env.update(integration)
+    env.update(target.env_set)
 
     return env
 
@@ -974,6 +980,13 @@ def requirements(host_profile: HostProfile) -> None:
     """Install requirements after bootstrapping and delegation."""
     if isinstance(host_profile, ControllerHostProfile) and host_profile.controller:
         configure_pypi_proxy(host_profile.args, host_profile)  # integration, windows-integration, network-integration
-        install_requirements(host_profile.args, host_profile.python, ansible=True, command=True)  # integration, windows-integration, network-integration
+
+        install_requirements(  # integration, windows-integration, network-integration
+            args=host_profile.args,
+            host_profile=host_profile,
+            python=host_profile.python,
+            ansible=True,
+            command=True,
+        )
     elif isinstance(host_profile, PosixProfile) and not isinstance(host_profile, ControllerProfile):
         configure_pypi_proxy(host_profile.args, host_profile)  # integration

ansible_test/_internal/commands/integration/cloud/httptester.py

@@ -71,7 +71,7 @@ class HttptesterProvider(CloudProvider):
             return
 
         # Read the password from the container environment.
-        # This allows the tests to work when re-using an existing container.
+        # This allows the tests to work when reusing an existing container.
         # The password is marked as sensitive, since it may differ from the one we generated.
         krb5_password = descriptor.details.container.env_dict()[KRB5_PASSWORD_ENV]
         display.sensitive.add(krb5_password)

ansible_test/_internal/commands/sanity/__init__.py

@@ -76,6 +76,7 @@ from ...python_requirements import (
     PipInstall,
     collect_requirements,
     run_pip,
+    install_requirements,
 )
 
 from ...config import (
@@ -178,6 +179,7 @@ def command_sanity(args: SanityConfig) -> None:
     if args.delegate:
         raise Delegate(host_state=host_state, require=changes, exclude=args.exclude)
 
+    install_requirements(args, host_state.controller_profile, host_state.controller_profile.python)  # sanity
     configure_pypi_proxy(args, host_state.controller_profile)  # sanity
 
     if disabled:
@@ -1085,7 +1087,7 @@ class SanityScript(SanityTest, metaclass=abc.ABCMeta):
 
 
 class SanityVersionNeutral(SanityTest, metaclass=abc.ABCMeta):
-    """Base class for sanity test plugins which are idependent of the python version being used."""
+    """Base class for sanity test plugins which are independent of the python version being used."""
 
     @abc.abstractmethod
     def test(self, args: SanityConfig, targets: SanityTargets) -> TestResult:

ansible_test/_internal/commands/sanity/integration_aliases.py

@@ -181,6 +181,8 @@ class IntegrationAliasesTest(SanitySingleVersion):
         group_numbers = self.ci_test_groups.get(name, None)
 
         if group_numbers:
+            group_numbers = [num for num in group_numbers if num not in (6, 7)]  # HACK: ignore special groups 6 and 7
+
             if min(group_numbers) != 1:
                 display.warning('Min test group "%s" in %s is %d instead of 1.' % (name, self.CI_YML, min(group_numbers)), unique=True)
 
@@ -291,6 +293,9 @@ class IntegrationAliasesTest(SanitySingleVersion):
             if target.name == 'ansible-test-container':
                 continue  # special test target which uses group 6 -- nothing else should be in that group
 
+            if target.name in ('dnf-oldest', 'dnf-latest'):
+                continue  # special test targets which use group 7 -- nothing else should be in that group
+
             if f'{self.TEST_ALIAS_PREFIX}/posix/' not in target.aliases:
                 continue
 
@@ -351,6 +356,12 @@ class IntegrationAliasesTest(SanitySingleVersion):
             if path == 'test/integration/targets/ansible-test-container':
                 continue  # special test target which uses group 6 -- nothing else should be in that group
 
+            if path in (
+                'test/integration/targets/dnf-oldest',
+                'test/integration/targets/dnf-latest',
+            ):
+                continue  # special test targets which use group 7 -- nothing else should be in that group
+
             messages.append(SanityMessage(unassigned_message, '%s/aliases' % path))
 
         for path in conflicting_paths:

ansible_test/_internal/commands/shell/__init__.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import dataclasses
 import os
 import sys
 import typing as t
@@ -14,6 +15,10 @@ from ...util import (
     HostConnectionError,
 )
 
+from ...ansible_util import (
+    ansible_environment,
+)
+
 from ...config import (
     ShellConfig,
 )
@@ -32,6 +37,7 @@ from ...host_profiles import (
     ControllerProfile,
     PosixProfile,
     SshTargetHostProfile,
+    DebuggableProfile,
 )
 
 from ...provisioning import (
@@ -39,7 +45,6 @@ from ...provisioning import (
 )
 
 from ...host_configs import (
-    ControllerConfig,
     OriginConfig,
 )
 
@@ -48,12 +53,21 @@ from ...inventory import (
     create_posix_inventory,
 )
 
+from ...python_requirements import (
+    install_requirements,
+)
+
+from ...util_common import (
+    get_injector_env,
+)
+
+from ...delegation import (
+    metadata_context,
+)
+
 
 def command_shell(args: ShellConfig) -> None:
     """Entry point for the `shell` command."""
-    if args.raw and isinstance(args.targets[0], ControllerConfig):
-        raise ApplicationError('The --raw option has no effect on the controller.')
-
     if not args.export and not args.cmd and not sys.stdin.isatty():
         raise ApplicationError('Standard input must be a TTY to launch a shell.')
 
@@ -62,6 +76,8 @@ def command_shell(args: ShellConfig) -> None:
     if args.delegate:
         raise Delegate(host_state=host_state)
 
+    install_requirements(args, host_state.controller_profile, host_state.controller_profile.python)  # shell
+
     if args.raw and not isinstance(args.controller, OriginConfig):
         display.warning('The --raw option will only be applied to the target.')
 
@@ -92,6 +108,20 @@ def command_shell(args: ShellConfig) -> None:
         con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL)
         return
 
+    if isinstance(con, LocalConnection) and isinstance(target_profile, DebuggableProfile) and target_profile.debugging_enabled:
+        # HACK: ensure the pydevd port visible in the shell is the forwarded port, not the original
+        args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.pydevd_port)
+
+    with metadata_context(args):
+        interactive_shell(args, target_profile, con)
+
+
+def interactive_shell(
+    args: ShellConfig,
+    target_profile: SshTargetHostProfile,
+    con: Connection,
+) -> None:
+    """Run an interactive shell."""
     if isinstance(con, SshConnection) and args.raw:
         cmd: list[str] = []
     elif isinstance(target_profile, PosixProfile):
@@ -111,6 +141,15 @@ def command_shell(args: ShellConfig) -> None:
 
             env = {name: os.environ[name] for name in optional_vars if name in os.environ}
 
+            if isinstance(con, LocalConnection):  # configure the controller environment
+                env.update(ansible_environment(args))
+                env.update(get_injector_env(target_profile.python, env))
+                env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path))
+
+                if isinstance(target_profile, DebuggableProfile):
+                    env.update(target_profile.get_ansiballz_environment_variables())
+                    env.update(target_profile.get_ansible_cli_environment_variables())
+
             if env:
                 cmd = ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()]
 

ansible_test/_internal/commands/units/__init__.py

@@ -64,6 +64,7 @@ from ...executor import (
 
 from ...python_requirements import (
     install_requirements,
+    post_install,
 )
 
 from ...content_config import (
@@ -230,7 +231,9 @@ def command_units(args: UnitsConfig) -> None:
         controller = any(test_context == TestContext.controller for test_context, python, paths, env in final_candidates)
 
         if args.requirements_mode != 'skip':
-            install_requirements(args, target_profile.python, ansible=controller, command=True, controller=False)  # units
+            install_requirements(args, target_profile, target_profile.python, ansible=controller, command=True, controller=False)  # units
+        else:
+            post_install(target_profile)
 
         test_sets.extend(final_candidates)
 

ansible_test/_internal/config.py

@@ -20,6 +20,7 @@ from .util_common import (
 
 from .metadata import (
     Metadata,
+    DebuggerFlags,
 )
 
 from .data import (
@@ -118,6 +119,26 @@ class EnvironmentConfig(CommonConfig):
         self.dev_systemd_debug: bool = args.dev_systemd_debug
         self.dev_probe_cgroups: t.Optional[str] = args.dev_probe_cgroups
 
+        debugger_flags = DebuggerFlags(
+            on_demand=args.dev_debug_on_demand,
+            cli=args.dev_debug_cli,
+            ansiballz=args.dev_debug_ansiballz,
+            self=args.dev_debug_self,
+        )
+
+        self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata(debugger_flags=debugger_flags)
+        self.metadata_path: t.Optional[str] = None
+
+        def metadata_callback(payload_config: PayloadConfig) -> None:
+            """Add the metadata file to the payload file list."""
+            config = self
+            files = payload_config.files
+
+            if config.metadata_path:
+                files.append((os.path.abspath(config.metadata_path), config.metadata_path))
+
+        data_context().register_payload_callback(metadata_callback)
+
         def host_callback(payload_config: PayloadConfig) -> None:
             """Add the host files to the payload file list."""
             config = self
@@ -220,22 +241,9 @@ class TestConfig(EnvironmentConfig):
         self.junit: bool = getattr(args, 'junit', False)
         self.failure_ok: bool = getattr(args, 'failure_ok', False)
 
-        self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata()
-        self.metadata_path: t.Optional[str] = None
-
         if self.coverage_check:
             self.coverage = True
 
-        def metadata_callback(payload_config: PayloadConfig) -> None:
-            """Add the metadata file to the payload file list."""
-            config = self
-            files = payload_config.files
-
-            if config.metadata_path:
-                files.append((os.path.abspath(config.metadata_path), config.metadata_path))
-
-        data_context().register_payload_callback(metadata_callback)
-
 
 class ShellConfig(EnvironmentConfig):
     """Configuration for the shell command."""

ansible_test/_internal/debugging.py

@@ -0,0 +1,166 @@
+"""Setup and configure remote debugging."""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import re
+
+from .util import (
+    cache,
+    display,
+    raw_command,
+    ApplicationError,
+)
+
+from .processes import (
+    Process,
+    get_current_process,
+)
+
+from .config import (
+    EnvironmentConfig,
+)
+
+from .metadata import (
+    DebuggerSettings,
+    DebuggerFlags,
+)
+
+from . import (
+    data_context,
+    CommonConfig,
+)
+
+
+def initialize_debugger(args: CommonConfig) -> None:
+    """Initialize the debugger settings before delegation."""
+    if not isinstance(args, EnvironmentConfig):
+        return
+
+    if args.metadata.loaded:
+        return  # after delegation
+
+    if collection := data_context().content.collection:
+        args.metadata.collection_root = collection.root
+
+    load_debugger_settings(args)
+
+
+def parse_debugger_settings(value: str) -> DebuggerSettings:
+    """Parse remote debugger settings and apply defaults."""
+    try:
+        settings = DebuggerSettings(**json.loads(value))
+    except Exception as ex:
+        raise ApplicationError(f"Invalid debugger settings: {ex}") from ex
+
+    if not settings.module:
+        if not settings.package or 'pydevd-pycharm' in settings.package:
+            module = 'pydevd_pycharm'
+        else:
+            module = 'pydevd'
+
+        settings = dataclasses.replace(settings, module=module)
+
+    if settings.package is None:
+        if settings.module == 'pydevd_pycharm':
+            if pycharm_version := detect_pycharm_version():
+                package = f'pydevd-pycharm~={pycharm_version}'
+            else:
+                package = None
+        else:
+            package = 'pydevd'
+
+        settings = dataclasses.replace(settings, package=package)
+
+    settings.settrace.setdefault('suspend', False)
+
+    if port := detect_pydevd_port():
+        settings = dataclasses.replace(settings, port=port)
+
+        if detect_pycharm_process():
+            # This only works with the default PyCharm debugger.
+            # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers.
+            # Further investigation is required to understand the cause.
+            settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess'])
+
+    return settings
+
+
+def load_debugger_settings(args: EnvironmentConfig) -> None:
+    """Load the remote debugger settings."""
+    if args.metadata.debugger_flags.on_demand:
+        # On-demand debugging only enables debugging if we're running under a debugger, otherwise it's a no-op.
+
+        if not detect_pydevd_port():
+            display.info('Debugging disabled because no debugger was detected.', verbosity=1)
+            args.metadata.debugger_flags = DebuggerFlags.all(False)
+            return
+
+        display.info('Enabling on-demand debugging.', verbosity=1)
+
+        if not args.metadata.debugger_flags.enable:
+            # Assume the user wants all debugging features enabled, since on-demand debugging with no features is pointless.
+            args.metadata.debugger_flags = DebuggerFlags.all(True)
+
+    if not args.metadata.debugger_flags.enable:
+        return
+
+    value = os.environ.get('ANSIBLE_TEST_REMOTE_DEBUGGER') or '{}'
+    settings = parse_debugger_settings(value)
+
+    display.info(f'>>> Debugger Settings\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3)
+
+    args.metadata.debugger_settings = settings
+
+
+@cache
+def detect_pydevd_port() -> int | None:
+    """Return the port for the pydevd instance hosting this process, or `None` if not detected."""
+    current_process = get_current_process_cached()
+    args = current_process.args
+
+    if any('/pydevd.py' in arg for arg in args) and (port_idx := args.index('--port')):
+        port = int(args[port_idx + 1])
+        display.info(f'Detected pydevd debugger port {port}.', verbosity=1)
+        return port
+
+    return None
+
+
+@cache
+def detect_pycharm_version() -> str | None:
+    """Return the version of PyCharm running ansible-test, or `None` if PyCharm was not detected. The result is cached."""
+    if pycharm := detect_pycharm_process():
+        output = raw_command([pycharm.args[0], '--version'], capture=True)[0]
+
+        if match := re.search('^Build #PY-(?P<version>[0-9.]+)$', output, flags=re.MULTILINE):
+            version = match.group('version')
+            display.info(f'Detected PyCharm version {version}.', verbosity=1)
+            return version
+
+    display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version could not be detected.')
+
+    return None
+
+
+@cache
+def detect_pycharm_process() -> Process | None:
+    """Return the PyCharm process running ansible-test, or `None` if PyCharm was not detected. The result is cached."""
+    current_process = get_current_process_cached()
+    parent = current_process.parent
+
+    while parent:
+        if parent.path.name == 'pycharm':
+            return parent
+
+        parent = parent.parent
+
+    return None
+
+
+@cache
+def get_current_process_cached() -> Process:
+    """Return the current process. The result is cached."""
+    return get_current_process()