ansible-core 2.13.8rc1__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ansible_test/_internal/classification/__init__.py

@@ -661,21 +661,58 @@ class PathMapper:
 
     def _classify_ansible(self, path):  # type: (str) -> t.Optional[t.Dict[str, str]]
         """Return the classification for the given path using rules specific to Ansible."""
+        dirname = os.path.dirname(path)
+        filename = os.path.basename(path)
+        name, ext = os.path.splitext(filename)
+
+        minimal: dict[str, str] = {}
+
+        packaging = {
+            'integration': 'packaging/',
+        }
+
+        # Early classification that needs to occur before common classification belongs here.
+
         if path.startswith('test/units/compat/'):
             return {
                 'units': 'test/units/',
             }
 
+        if dirname == '.azure-pipelines/commands':
+            test_map = {
+                'cloud.sh': 'integration:cloud/',
+                'linux.sh': 'integration:all',
+                'network.sh': 'network-integration:all',
+                'remote.sh': 'integration:all',
+                'sanity.sh': 'sanity:all',
+                'units.sh': 'units:all',
+                'windows.sh': 'windows-integration:all',
+            }
+
+            test_match = test_map.get(filename)
+
+            if test_match:
+                test_command, test_target = test_match.split(':')
+
+                return {
+                    test_command: test_target,
+                }
+
+            cloud_target = f'cloud/{name}/'
+
+            if cloud_target in self.integration_targets_by_alias:
+                return {
+                    'integration': cloud_target,
+                }
+
+        # Classification common to both ansible and collections.
+
         result = self._classify_common(path)
 
         if result is not None:
             return result
 
-        dirname = os.path.dirname(path)
-        filename = os.path.basename(path)
-        name, ext = os.path.splitext(filename)
-
-        minimal = {}  # type: t.Dict[str, str]
+        # Classification here is specific to ansible, and runs after common classification.
 
         if path.startswith('bin/'):
             return all_tests(self.args)  # broad impact, run all tests
@@ -715,6 +752,9 @@ class PathMapper:
             return minimal
 
         if path.startswith('packaging/'):
+            if path.startswith('packaging/pep517_backend/'):
+                return packaging
+
             return minimal
 
         if path.startswith('test/ansible_test/'):
@@ -791,39 +831,6 @@ class PathMapper:
         if path.startswith('test/support/'):
             return all_tests(self.args)  # test infrastructure, run all tests
 
-        if path.startswith('test/utils/shippable/'):
-            if dirname == 'test/utils/shippable':
-                test_map = {
-                    'cloud.sh': 'integration:cloud/',
-                    'linux.sh': 'integration:all',
-                    'network.sh': 'network-integration:all',
-                    'remote.sh': 'integration:all',
-                    'sanity.sh': 'sanity:all',
-                    'units.sh': 'units:all',
-                    'windows.sh': 'windows-integration:all',
-                }
-
-                test_match = test_map.get(filename)
-
-                if test_match:
-                    test_command, test_target = test_match.split(':')
-
-                    return {
-                        test_command: test_target,
-                    }
-
-                cloud_target = 'cloud/%s/' % name
-
-                if cloud_target in self.integration_targets_by_alias:
-                    return {
-                        'integration': cloud_target,
-                    }
-
-            return all_tests(self.args)  # test infrastructure, run all tests
-
-        if path.startswith('test/utils/'):
-            return minimal
-
         if '/' not in path:
             if path in (
                     '.gitattributes',
@@ -835,16 +842,17 @@ class PathMapper:
                 return minimal
 
             if path in (
-                    'setup.py',
+                'MANIFEST.in',
+                'pyproject.toml',
+                'requirements.txt',
+                'setup.cfg',
+                'setup.py',
             ):
-                return all_tests(self.args)  # broad impact, run all tests
+                return packaging
 
             if ext in (
-                    '.in',
                     '.md',
                     '.rst',
-                    '.toml',
-                    '.txt',
             ):
                 return minimal
 

ansible_test/_internal/cli/argparsing/argcompletion.py

@@ -16,10 +16,19 @@ class Substitute:
 try:
     import argcomplete
 
-    from argcomplete import (
-        CompletionFinder,
-        default_validator,
-    )
+    try:
+        # argcomplete 3+
+        # see: https://github.com/kislyuk/argcomplete/commit/bd781cb08512b94966312377186ebc5550f46ae0
+        from argcomplete.finders import (
+            CompletionFinder,
+            default_validator,
+        )
+    except ImportError:
+        # argcomplete <3
+        from argcomplete import (
+            CompletionFinder,
+            default_validator,
+        )
 
     warn = argcomplete.warn  # pylint: disable=invalid-name
 except ImportError:
@@ -70,7 +79,13 @@ class CompType(enum.Enum):
 def register_safe_action(action_type):  # type: (t.Type[argparse.Action]) -> None
     """Register the given action as a safe action for argcomplete to use during completion if it is not already registered."""
     if argcomplete and action_type not in argcomplete.safe_actions:
-        argcomplete.safe_actions += (action_type,)
+        if isinstance(argcomplete.safe_actions, set):
+            # argcomplete 3+
+            # see: https://github.com/kislyuk/argcomplete/commit/bd781cb08512b94966312377186ebc5550f46ae0
+            argcomplete.safe_actions.add(action_type)
+        else:
+            # argcomplete <3
+            argcomplete.safe_actions += (action_type,)
 
 
 def get_comp_type():  # type: () -> t.Optional[CompType]

ansible_test/_internal/cli/commands/sanity.py

@@ -16,10 +16,6 @@ from ...target import (
     walk_sanity_targets,
 )
 
-from ...data import (
-    data_context,
-)
-
 from ..environments import (
     CompositeActionCompletionFinder,
     ControllerMode,
@@ -81,17 +77,6 @@ def do_sanity(
         help='enable optional errors',
     )
 
-    if data_context().content.is_ansible:
-        sanity.add_argument(
-            '--keep-git',
-            action='store_true',
-            help='transfer git related files to the remote host/container',
-        )
-    else:
-        sanity.set_defaults(
-            keep_git=False,
-        )
-
     sanity.add_argument(
         '--lint',
         action='store_true',

ansible_test/_internal/commands/coverage/combine.py

@@ -33,6 +33,7 @@ from ...executor import (
 
 from ...data import (
     data_context,
+    PayloadConfig,
 )
 
 from ...host_configs import (
@@ -81,9 +82,10 @@ def combine_coverage_files(args, host_state):  # type: (CoverageCombineConfig, H
 
             pairs = [(path, os.path.relpath(path, data_context().content.root)) for path in exported_paths]
 
-            def coverage_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+            def coverage_callback(payload_config: PayloadConfig) -> None:
                 """Add the coverage files to the payload file list."""
                 display.info('Including %d exported coverage file(s) in payload.' % len(pairs), verbosity=1)
+                files = payload_config.files
                 files.extend(pairs)
 
             data_context().register_payload_callback(coverage_callback)

ansible_test/_internal/commands/integration/__init__.py

@@ -89,6 +89,7 @@ from .cloud import (
 
 from ...data import (
     data_context,
+    PayloadConfig,
 )
 
 from ...host_configs import (
@@ -213,11 +214,13 @@ def delegate_inventory(args, inventory_path_src):  # type: (IntegrationConfig, s
     if isinstance(args, PosixIntegrationConfig):
         return
 
-    def inventory_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+    def inventory_callback(payload_config: PayloadConfig) -> None:
         """
         Add the inventory file to the payload file list.
         This will preserve the file during delegation even if it is ignored or is outside the content and install roots.
         """
+        files = payload_config.files
+
         inventory_path = get_inventory_relative_path(args)
         inventory_tuple = inventory_path_src, inventory_path
 
@@ -940,11 +943,12 @@ def command_integration_filter(args,  # type: TIntegrationConfig
     vars_file_src = os.path.join(data_context().content.root, data_context().content.integration_vars_path)
 
     if os.path.exists(vars_file_src):
-        def integration_config_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+        def integration_config_callback(payload_config: PayloadConfig) -> None:
             """
             Add the integration config vars file to the payload file list.
             This will preserve the file during delegation even if the file is ignored by source control.
             """
+            files = payload_config.files
             files.append((vars_file_src, data_context().content.integration_vars_path))
 
         data_context().register_payload_callback(integration_config_callback)

ansible_test/_internal/commands/integration/cloud/__init__.py

@@ -47,6 +47,7 @@ from ....ci import (
 
 from ....data import (
     data_context,
+    PayloadConfig,
 )
 
 from ....docker_util import (
@@ -189,13 +190,14 @@ class CloudBase(metaclass=abc.ABCMeta):
         self.args = args
         self.platform = self.__module__.rsplit('.', 1)[-1]
 
-        def config_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+        def config_callback(payload_config: PayloadConfig) -> None:
             """Add the config file to the payload file list."""
             if self.platform not in self.args.metadata.cloud_config:
                 return  # platform was initialized, but not used -- such as being skipped due to all tests being disabled
 
             if self._get_cloud_config(self._CONFIG_PATH, ''):
                 pair = (self.config_path, os.path.relpath(self.config_path, data_context().content.root))
+                files = payload_config.files
 
                 if pair not in files:
                     display.info('Including %s config: %s -> %s' % (self.platform, pair[0], pair[1]), verbosity=3)

ansible_test/_internal/commands/sanity/__init__.py

@@ -159,6 +159,10 @@ def command_sanity(args):  # type: (SanityConfig) -> None
     if args.skip_test:
         tests = [target for target in tests if target.name not in args.skip_test]
 
+    if not args.host_path:
+        for test in tests:
+            test.origin_hook(args)
+
     targets_use_pypi = any(isinstance(test, SanityMultipleVersion) and test.needs_pypi for test in tests) and not args.list_tests
     host_state = prepare_profiles(args, targets_use_pypi=targets_use_pypi)  # sanity
 
@@ -756,6 +760,9 @@ class SanityTest(metaclass=abc.ABCMeta):
         """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
         return CONTROLLER_PYTHON_VERSIONS
 
+    def origin_hook(self, args: SanityConfig) -> None:
+        """This method is called on the origin, before the test runs or delegation occurs."""
+
     def filter_targets(self, targets):  # type: (t.List[TestTarget]) -> t.List[TestTarget]  # pylint: disable=unused-argument
         """Return the given list of test targets, filtered to include only those relevant for the test."""
         if self.no_targets:

ansible_test/_internal/commands/sanity/pylint.py

@@ -17,6 +17,10 @@ from . import (
     SANITY_ROOT,
 )
 
+from ...io import (
+    make_dirs,
+)
+
 from ...test import (
     TestResult,
 )
@@ -40,6 +44,7 @@ from ...ansible_util import (
     get_collection_detail,
     CollectionDetail,
     CollectionDetailError,
+    ResultType,
 )
 
 from ...config import (
@@ -245,6 +250,12 @@ class PylintTest(SanitySingleVersion):
         # expose plugin paths for use in custom plugins
         env.update(dict(('ANSIBLE_TEST_%s_PATH' % k.upper(), os.path.abspath(v) + os.path.sep) for k, v in data_context().content.plugin_paths.items()))
 
+        # Set PYLINTHOME to prevent pylint from checking for an obsolete directory, which can result in a test failure due to stderr output.
+        # See: https://github.com/PyCQA/pylint/blob/e6c6bf5dfd61511d64779f54264b27a368c43100/pylint/constants.py#L148
+        pylint_home = os.path.join(ResultType.TMP.path, 'pylint')
+        make_dirs(pylint_home)
+        env.update(PYLINTHOME=pylint_home)
+
         if paths:
             display.info('Checking %d file(s) in context "%s" with config: %s' % (len(paths), context, rcfile), verbosity=1)
 

ansible_test/_internal/commands/sanity/validate_modules.py

@@ -1,9 +1,12 @@
 """Sanity test using validate-modules."""
 from __future__ import annotations
 
+import atexit
 import collections
+import contextlib
 import json
 import os
+import tarfile
 import typing as t
 
 from . import (
@@ -16,6 +19,10 @@ from . import (
     SANITY_ROOT,
 )
 
+from ...io import (
+    make_dirs,
+)
+
 from ...test import (
     TestResult,
 )
@@ -30,7 +37,9 @@ from ...util import (
 )
 
 from ...util_common import (
+    process_scoped_temporary_directory,
     run_command,
+    ResultType,
 )
 
 from ...ansible_util import (
@@ -49,12 +58,21 @@ from ...ci import (
 
 from ...data import (
     data_context,
+    PayloadConfig,
 )
 
 from ...host_configs import (
     PythonConfig,
 )
 
+from ...git import (
+    Git,
+)
+
+from ...provider.source import (
+    SourceProvider as GitSourceProvider,
+)
+
 
 class ValidateModulesTest(SanitySingleVersion):
     """Sanity test using validate-modules."""
@@ -130,14 +148,17 @@ class ValidateModulesTest(SanitySingleVersion):
             except CollectionDetailError as ex:
                 display.warning('Skipping validate-modules collection version checks since collection detail loading failed: %s' % ex.reason)
         else:
-            base_branch = args.base_branch or get_ci_provider().get_base_branch()
+            path = self.get_archive_path(args)
+
+            if os.path.exists(path):
+                temp_dir = process_scoped_temporary_directory(args)
+
+                with tarfile.open(path) as file:
+                    file.extractall(temp_dir)
 
-            if base_branch:
                 cmd.extend([
-                    '--base-branch', base_branch,
+                    '--original-plugins', temp_dir,
                 ])
-            else:
-                display.warning('Cannot perform module comparison against the base branch because the base branch was not detected.')
 
         errors = []
 
@@ -188,3 +209,43 @@ class ValidateModulesTest(SanitySingleVersion):
             return SanityFailure(self.name, messages=all_errors)
 
         return SanitySuccess(self.name)
+
+    def origin_hook(self, args: SanityConfig) -> None:
+        """This method is called on the origin, before the test runs or delegation occurs."""
+        if not data_context().content.is_ansible:
+            return
+
+        if not isinstance(data_context().source_provider, GitSourceProvider):
+            display.warning('The validate-modules sanity test cannot compare against the base commit because git is not being used.')
+            return
+
+        base_commit = args.base_branch or get_ci_provider().get_base_commit(args)
+
+        if not base_commit:
+            display.warning('The validate-modules sanity test cannot compare against the base commit because it was not detected.')
+            return
+
+        path = self.get_archive_path(args)
+
+        def cleanup() -> None:
+            """Cleanup callback called when the process exits."""
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(path)
+
+        def git_callback(payload_config: PayloadConfig) -> None:
+            """Include the previous plugin content archive in the payload."""
+            files = payload_config.files
+            files.append((path, os.path.relpath(path, data_context().content.root)))
+
+        atexit.register(cleanup)
+        data_context().register_payload_callback(git_callback)
+
+        make_dirs(os.path.dirname(path))
+
+        git = Git()
+        git.run_git(['archive', '--output', path, base_commit, 'lib/ansible/modules/', 'lib/ansible/plugins/'])
+
+    @staticmethod
+    def get_archive_path(args: SanityConfig) -> str:
+        """Return the path to the original plugin content archive."""
+        return os.path.join(ResultType.TMP.path, f'validate-modules-{args.metadata.session_id}.tar')

ansible_test/_internal/config.py

@@ -24,6 +24,7 @@ from .metadata import (
 
 from .data import (
     data_context,
+    PayloadConfig,
 )
 
 from .host_configs import (
@@ -114,7 +115,7 @@ class EnvironmentConfig(CommonConfig):
         self.dev_systemd_debug: bool = args.dev_systemd_debug
         self.dev_probe_cgroups: t.Optional[str] = args.dev_probe_cgroups
 
-        def host_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+        def host_callback(payload_config: PayloadConfig) -> None:
             """Add the host files to the payload file list."""
             config = self
 
@@ -123,6 +124,8 @@ class EnvironmentConfig(CommonConfig):
                 state_path = os.path.join(config.host_path, 'state.dat')
                 config_path = os.path.join(config.host_path, 'config.dat')
 
+                files = payload_config.files
+
                 files.append((os.path.abspath(settings_path), settings_path))
                 files.append((os.path.abspath(state_path), state_path))
                 files.append((os.path.abspath(config_path), config_path))
@@ -225,9 +228,10 @@ class TestConfig(EnvironmentConfig):
         if self.coverage_check:
             self.coverage = True
 
-        def metadata_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+        def metadata_callback(payload_config: PayloadConfig) -> None:
             """Add the metadata file to the payload file list."""
             config = self
+            files = payload_config.files
 
             if config.metadata_path:
                 files.append((os.path.abspath(config.metadata_path), config.metadata_path))
@@ -258,20 +262,10 @@ class SanityConfig(TestConfig):
         self.list_tests = args.list_tests  # type: bool
         self.allow_disabled = args.allow_disabled  # type: bool
         self.enable_optional_errors = args.enable_optional_errors  # type: bool
-        self.keep_git = args.keep_git  # type: bool
         self.prime_venvs = args.prime_venvs  # type: bool
 
         self.display_stderr = self.lint or self.list_tests
 
-        if self.keep_git:
-            def git_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
-                """Add files from the content root .git directory to the payload file list."""
-                for dirpath, _dirnames, filenames in os.walk(os.path.join(data_context().content.root, '.git')):
-                    paths = [os.path.join(dirpath, filename) for filename in filenames]
-                    files.extend((path, os.path.relpath(path, data_context().content.root)) for path in paths)
-
-            data_context().register_payload_callback(git_callback)
-
 
 class IntegrationConfig(TestConfig):
     """Configuration for the integration command."""

ansible_test/_internal/core_ci.py

@@ -6,6 +6,7 @@ import dataclasses
 import json
 import os
 import re
+import stat
 import traceback
 import uuid
 import errno
@@ -47,6 +48,7 @@ from .ci import (
 
 from .data import (
     data_context,
+    PayloadConfig,
 )
 
 
@@ -447,14 +449,19 @@ class SshKey:
         key, pub = key_pair
         key_dst, pub_dst = self.get_in_tree_key_pair_paths()
 
-        def ssh_key_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
+        def ssh_key_callback(payload_config: PayloadConfig) -> None:
             """
             Add the SSH keys to the payload file list.
             They are either outside the source tree or in the cache dir which is ignored by default.
             """
+            files = payload_config.files
+            permissions = payload_config.permissions
+
             files.append((key, os.path.relpath(key_dst, data_context().content.root)))
             files.append((pub, os.path.relpath(pub_dst, data_context().content.root)))
 
+            permissions[os.path.relpath(key_dst, data_context().content.root)] = stat.S_IRUSR | stat.S_IWUSR
+
         data_context().register_payload_callback(ssh_key_callback)
 
         self.key, self.pub = key, pub

ansible_test/_internal/data.py

@@ -1,6 +1,7 @@
 """Context information for the current invocation of ansible-test."""
 from __future__ import annotations
 
+import collections.abc as c
 import dataclasses
 import os
 import typing as t
@@ -49,6 +50,13 @@ from .provider.layout.unsupported import (
 )
 
 
+@dataclasses.dataclass(frozen=True)
+class PayloadConfig:
+    """Configuration required to build a source tree payload for delegation."""
+    files: list[tuple[str, str]]
+    permissions: dict[str, int]
+
+
 class DataContext:
     """Data context providing details about the current execution environment for ansible-test."""
     def __init__(self):
@@ -62,16 +70,17 @@ class DataContext:
         self.__source_providers = source_providers
         self.__ansible_source = None  # type: t.Optional[t.Tuple[t.Tuple[str, str], ...]]
 
-        self.payload_callbacks = []  # type: t.List[t.Callable[[t.List[t.Tuple[str, str]]], None]]
+        self.payload_callbacks: list[c.Callable[[PayloadConfig], None]] = []
 
         if content_path:
-            content = self.__create_content_layout(layout_providers, source_providers, content_path, False)
+            content, source_provider = self.__create_content_layout(layout_providers, source_providers, content_path, False)
         elif ANSIBLE_SOURCE_ROOT and is_subdir(current_path, ANSIBLE_SOURCE_ROOT):
-            content = self.__create_content_layout(layout_providers, source_providers, ANSIBLE_SOURCE_ROOT, False)
+            content, source_provider = self.__create_content_layout(layout_providers, source_providers, ANSIBLE_SOURCE_ROOT, False)
         else:
-            content = self.__create_content_layout(layout_providers, source_providers, current_path, True)
+            content, source_provider = self.__create_content_layout(layout_providers, source_providers, current_path, True)
 
         self.content = content  # type: ContentLayout
+        self.source_provider = source_provider
 
     def create_collection_layouts(self):  # type: () -> t.List[ContentLayout]
         """
@@ -99,7 +108,7 @@ class DataContext:
                 if collection_path == os.path.join(collection.root, collection.directory):
                     collection_layout = layout
                 else:
-                    collection_layout = self.__create_content_layout(self.__layout_providers, self.__source_providers, collection_path, False)
+                    collection_layout = self.__create_content_layout(self.__layout_providers, self.__source_providers, collection_path, False)[0]
 
                 file_count = len(collection_layout.all_files())
 
@@ -116,7 +125,7 @@ class DataContext:
                                 source_providers,  # type: t.List[t.Type[SourceProvider]]
                                 root,  # type: str
                                 walk,  # type: bool
-                                ):  # type: (...) -> ContentLayout
+                                ) -> t.Tuple[ContentLayout, SourceProvider]:
         """Create a content layout using the given providers and root path."""
         try:
             layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk)
@@ -137,7 +146,7 @@ class DataContext:
 
         layout = layout_provider.create(layout_provider.root, source_provider.get_paths(layout_provider.root))
 
-        return layout
+        return layout, source_provider
 
     def __create_ansible_source(self):
         """Return a tuple of Ansible source files with both absolute and relative paths."""
@@ -172,7 +181,7 @@ class DataContext:
 
         return self.__ansible_source
 
-    def register_payload_callback(self, callback):  # type: (t.Callable[[t.List[t.Tuple[str, str]]], None]) -> None
+    def register_payload_callback(self, callback: c.Callable[[PayloadConfig], None]) -> None:
         """Register the given payload callback."""
         self.payload_callbacks.append(callback)
 

ansible_test/_internal/delegation.py

@@ -172,7 +172,6 @@ def delegate_command(args, host_state, exclude, require):  # type: (EnvironmentC
             con.run(['mkdir', '-p'] + writable_dirs, capture=True)
             con.run(['chmod', '777'] + writable_dirs, capture=True)
             con.run(['chmod', '755', working_directory], capture=True)
-            con.run(['chmod', '644', os.path.join(content_root, args.metadata_path)], capture=True)
             con.run(['useradd', pytest_user, '--create-home'], capture=True)
 
             con.run(insert_options(command, options + ['--requirements-mode', 'only']), capture=False)
@@ -339,7 +338,7 @@ def filter_options(
             ('--metadata', 1, args.metadata_path),
             ('--exclude', 1, exclude),
             ('--require', 1, require),
-            ('--base-branch', 1, args.base_branch or get_ci_provider().get_base_branch()),
+            ('--base-branch', 1, False),
         ])
 
     pass_through_args: list[str] = []