ansible-core 2.13.11__py3-none-any.whl → 2.13.12rc1__py3-none-any.whl

ansible_test/_internal/commands/sanity/integration_aliases.py

@@ -0,0 +1,401 @@
+"""Sanity test to check integration test aliases."""
+from __future__ import annotations
+
+import dataclasses
+import json
+import textwrap
+import os
+import typing as t
+
+from . import (
+    SanitySingleVersion,
+    SanityMessage,
+    SanityFailure,
+    SanitySuccess,
+    SanityTargets,
+    SANITY_ROOT,
+)
+
+from ...test import (
+    TestResult,
+)
+
+from ...config import (
+    SanityConfig,
+)
+
+from ...target import (
+    filter_targets,
+    walk_posix_integration_targets,
+    walk_windows_integration_targets,
+    walk_integration_targets,
+    walk_module_targets,
+    CompletionTarget,
+)
+
+from ..integration.cloud import (
+    get_cloud_platforms,
+)
+
+from ...io import (
+    read_text_file,
+)
+
+from ...util import (
+    display,
+    raw_command,
+)
+
+from ...util_common import (
+    get_docs_url,
+    write_json_test_results,
+    ResultType,
+)
+
+from ...host_configs import (
+    PythonConfig,
+)
+
+
+class IntegrationAliasesTest(SanitySingleVersion):
+    """Sanity test to evaluate integration test aliases."""
+    CI_YML = '.azure-pipelines/azure-pipelines.yml'
+    TEST_ALIAS_PREFIX = 'shippable'  # this will be changed at some point in the future
+
+    DISABLED = 'disabled/'
+    UNSTABLE = 'unstable/'
+    UNSUPPORTED = 'unsupported/'
+
+    EXPLAIN_URL = get_docs_url('https://docs.ansible.com/ansible-core/devel/dev_guide/testing/sanity/integration-aliases.html')
+
+    TEMPLATE_DISABLED = """
+    The following integration tests are **disabled** [[explain]({explain_url}#disabled)]:
+
+    {tests}
+
+    Consider fixing the integration tests before or alongside changes.
+    """
+
+    TEMPLATE_UNSTABLE = """
+    The following integration tests are **unstable** [[explain]({explain_url}#unstable)]:
+
+    {tests}
+
+    Tests may need to be restarted due to failures unrelated to changes.
+    """
+
+    TEMPLATE_UNSUPPORTED = """
+    The following integration tests are **unsupported** [[explain]({explain_url}#unsupported)]:
+
+    {tests}
+
+    Consider running the tests manually or extending test infrastructure to add support.
+    """
+
+    TEMPLATE_UNTESTED = """
+    The following modules have **no integration tests** [[explain]({explain_url}#untested)]:
+
+    {tests}
+
+    Consider adding integration tests before or alongside changes.
+    """
+
+    ansible_only = True
+
+    def __init__(self):
+        super().__init__()
+
+        self._ci_config = {}  # type: t.Dict[str, t.Any]
+        self._ci_test_groups = {}  # type: t.Dict[str, t.List[int]]
+
+    @property
+    def can_ignore(self):  # type: () -> bool
+        """True if the test supports ignore entries."""
+        return False
+
+    @property
+    def no_targets(self):  # type: () -> bool
+        """True if the test does not use test targets. Mutually exclusive with all_targets."""
+        return True
+
+    def load_ci_config(self, python):  # type: (PythonConfig) -> t.Dict[str, t.Any]
+        """Load and return the CI YAML configuration."""
+        if not self._ci_config:
+            self._ci_config = self.load_yaml(python, self.CI_YML)
+
+        return self._ci_config
+
+    @property
+    def ci_test_groups(self):  # type: () -> t.Dict[str, t.List[int]]
+        """Return a dictionary of CI test names and their group(s)."""
+        if not self._ci_test_groups:
+            test_groups = {}  # type: t.Dict[str, t.Set[int]]
+
+            for stage in self._ci_config['stages']:
+                for job in stage['jobs']:
+                    if job.get('template') != 'templates/matrix.yml':
+                        continue
+
+                    parameters = job['parameters']
+
+                    groups = parameters.get('groups', [])
+                    test_format = parameters.get('testFormat', '{0}')
+                    test_group_format = parameters.get('groupFormat', '{0}/{{1}}')
+
+                    for target in parameters['targets']:
+                        test = target.get('test') or target.get('name')
+
+                        if groups:
+                            tests_formatted = [test_group_format.format(test_format).format(test, group) for group in groups]
+                        else:
+                            tests_formatted = [test_format.format(test)]
+
+                        for test_formatted in tests_formatted:
+                            parts = test_formatted.split('/')
+                            key = parts[0]
+
+                            if key in ('sanity', 'units'):
+                                continue
+
+                            try:
+                                group = int(parts[-1])
+                            except ValueError:
+                                continue
+
+                            if group < 1 or group > 99:
+                                continue
+
+                            group_set = test_groups.setdefault(key, set())
+                            group_set.add(group)
+
+            self._ci_test_groups = dict((key, sorted(value)) for key, value in test_groups.items())
+
+        return self._ci_test_groups
+
+    def format_test_group_alias(self, name, fallback=''):  # type: (str, str) -> str
+        """Return a test group alias using the given name and fallback."""
+        group_numbers = self.ci_test_groups.get(name, None)
+
+        if group_numbers:
+            if min(group_numbers) != 1:
+                display.warning('Min test group "%s" in %s is %d instead of 1.' % (name, self.CI_YML, min(group_numbers)), unique=True)
+
+            if max(group_numbers) != len(group_numbers):
+                display.warning('Max test group "%s" in %s is %d instead of %d.' % (name, self.CI_YML, max(group_numbers), len(group_numbers)), unique=True)
+
+            if max(group_numbers) > 9:
+                alias = '%s/%s/group(%s)/' % (self.TEST_ALIAS_PREFIX, name, '|'.join(str(i) for i in range(min(group_numbers), max(group_numbers) + 1)))
+            elif len(group_numbers) > 1:
+                alias = '%s/%s/group[%d-%d]/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers), max(group_numbers))
+            else:
+                alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers))
+        elif fallback:
+            alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, fallback, 1)
+        else:
+            raise Exception('cannot find test group "%s" in %s' % (name, self.CI_YML))
+
+        return alias
+
+    def load_yaml(self, python, path):  # type: (PythonConfig, str) -> t.Dict[str, t.Any]
+        """Load the specified YAML file and return the contents."""
+        yaml_to_json_path = os.path.join(SANITY_ROOT, self.name, 'yaml_to_json.py')
+        return json.loads(raw_command([python.path, yaml_to_json_path], data=read_text_file(path), capture=True)[0])
+
+    def test(self, args, targets, python):  # type: (SanityConfig, SanityTargets, PythonConfig) -> TestResult
+        if args.explain:
+            return SanitySuccess(self.name)
+
+        if not os.path.isfile(self.CI_YML):
+            return SanityFailure(self.name, messages=[SanityMessage(
+                message='file missing',
+                path=self.CI_YML,
+            )])
+
+        results = Results(
+            comments=[],
+            labels={},
+        )
+
+        self.load_ci_config(python)
+        self.check_changes(args, results)
+
+        write_json_test_results(ResultType.BOT, 'data-sanity-ci.json', results.__dict__)
+
+        messages = []
+
+        messages += self.check_posix_targets(args)
+        messages += self.check_windows_targets()
+
+        if messages:
+            return SanityFailure(self.name, messages=messages)
+
+        return SanitySuccess(self.name)
+
+    def check_posix_targets(self, args):  # type: (SanityConfig) -> t.List[SanityMessage]
+        """Check POSIX integration test targets and return messages with any issues found."""
+        posix_targets = tuple(walk_posix_integration_targets())
+
+        clouds = get_cloud_platforms(args, posix_targets)
+        cloud_targets = ['cloud/%s/' % cloud for cloud in clouds]
+
+        all_cloud_targets = tuple(filter_targets(posix_targets, ['cloud/'], directories=False, errors=False))
+        invalid_cloud_targets = tuple(filter_targets(all_cloud_targets, cloud_targets, include=False, directories=False, errors=False))
+
+        messages = []
+
+        for target in invalid_cloud_targets:
+            for alias in target.aliases:
+                if alias.startswith('cloud/') and alias != 'cloud/':
+                    if any(alias.startswith(cloud_target) for cloud_target in cloud_targets):
+                        continue
+
+                    messages.append(SanityMessage('invalid alias `%s`' % alias, '%s/aliases' % target.path))
+
+        messages += self.check_ci_group(
+            targets=tuple(filter_targets(posix_targets, ['cloud/', '%s/generic/' % self.TEST_ALIAS_PREFIX], include=False, directories=False, errors=False)),
+            find=self.format_test_group_alias('linux').replace('linux', 'posix'),
+            find_incidental=['%s/posix/incidental/' % self.TEST_ALIAS_PREFIX],
+        )
+
+        messages += self.check_ci_group(
+            targets=tuple(filter_targets(posix_targets, ['%s/generic/' % self.TEST_ALIAS_PREFIX], directories=False, errors=False)),
+            find=self.format_test_group_alias('generic'),
+        )
+
+        for cloud in clouds:
+            if cloud == 'httptester':
+                find = self.format_test_group_alias('linux').replace('linux', 'posix')
+                find_incidental = ['%s/posix/incidental/' % self.TEST_ALIAS_PREFIX]
+            else:
+                find = self.format_test_group_alias(cloud, 'generic')
+                find_incidental = ['%s/%s/incidental/' % (self.TEST_ALIAS_PREFIX, cloud), '%s/cloud/incidental/' % self.TEST_ALIAS_PREFIX]
+
+            messages += self.check_ci_group(
+                targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], directories=False, errors=False)),
+                find=find,
+                find_incidental=find_incidental,
+            )
+
+        return messages
+
+    def check_windows_targets(self):
+        """
+        :rtype: list[SanityMessage]
+        """
+        windows_targets = tuple(walk_windows_integration_targets())
+
+        messages = []
+
+        messages += self.check_ci_group(
+            targets=windows_targets,
+            find=self.format_test_group_alias('windows'),
+            find_incidental=['%s/windows/incidental/' % self.TEST_ALIAS_PREFIX],
+        )
+
+        return messages
+
+    def check_ci_group(
+            self,
+            targets,  # type: t.Tuple[CompletionTarget, ...]
+            find,  # type: str
+            find_incidental=None,  # type: t.Optional[t.List[str]]
+    ):  # type: (...) -> t.List[SanityMessage]
+        """Check the CI groups set in the provided targets and return a list of messages with any issues found."""
+        all_paths = set(target.path for target in targets)
+        supported_paths = set(target.path for target in filter_targets(targets, [find], directories=False, errors=False))
+        unsupported_paths = set(target.path for target in filter_targets(targets, [self.UNSUPPORTED], directories=False, errors=False))
+
+        if find_incidental:
+            incidental_paths = set(target.path for target in filter_targets(targets, find_incidental, directories=False, errors=False))
+        else:
+            incidental_paths = set()
+
+        unassigned_paths = all_paths - supported_paths - unsupported_paths - incidental_paths
+        conflicting_paths = supported_paths & unsupported_paths
+
+        unassigned_message = 'missing alias `%s` or `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
+        conflicting_message = 'conflicting alias `%s` and `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
+
+        messages = []
+
+        for path in unassigned_paths:
+            if path == 'test/integration/targets/ansible-test-container':
+                continue  # special test target which uses group 6 -- nothing else should be in that group
+
+            messages.append(SanityMessage(unassigned_message, '%s/aliases' % path))
+
+        for path in conflicting_paths:
+            messages.append(SanityMessage(conflicting_message, '%s/aliases' % path))
+
+        return messages
+
+    def check_changes(self, args, results):  # type: (SanityConfig, Results) -> None
+        """Check changes and store results in the provided result dictionary."""
+        integration_targets = list(walk_integration_targets())
+        module_targets = list(walk_module_targets())
+
+        integration_targets_by_name = dict((target.name, target) for target in integration_targets)
+        module_names_by_path = dict((target.path, target.module) for target in module_targets)
+
+        disabled_targets = []
+        unstable_targets = []
+        unsupported_targets = []
+
+        for command in [command for command in args.metadata.change_description.focused_command_targets if 'integration' in command]:
+            for target in args.metadata.change_description.focused_command_targets[command]:
+                if self.DISABLED in integration_targets_by_name[target].aliases:
+                    disabled_targets.append(target)
+                elif self.UNSTABLE in integration_targets_by_name[target].aliases:
+                    unstable_targets.append(target)
+                elif self.UNSUPPORTED in integration_targets_by_name[target].aliases:
+                    unsupported_targets.append(target)
+
+        untested_modules = []
+
+        for path in args.metadata.change_description.no_integration_paths:
+            module = module_names_by_path.get(path)
+
+            if module:
+                untested_modules.append(module)
+
+        comments = [
+            self.format_comment(self.TEMPLATE_DISABLED, disabled_targets),
+            self.format_comment(self.TEMPLATE_UNSTABLE, unstable_targets),
+            self.format_comment(self.TEMPLATE_UNSUPPORTED, unsupported_targets),
+            self.format_comment(self.TEMPLATE_UNTESTED, untested_modules),
+        ]
+
+        comments = [comment for comment in comments if comment]
+
+        labels = dict(
+            needs_tests=bool(untested_modules),
+            disabled_tests=bool(disabled_targets),
+            unstable_tests=bool(unstable_targets),
+            unsupported_tests=bool(unsupported_targets),
+        )
+
+        results.comments += comments
+        results.labels.update(labels)
+
+    def format_comment(self, template, targets):  # type: (str, t.List[str]) -> t.Optional[str]
+        """Format and return a comment based on the given template and targets, or None if there are no targets."""
+        if not targets:
+            return None
+
+        tests = '\n'.join('- %s' % target for target in targets)
+
+        data = dict(
+            explain_url=self.EXPLAIN_URL,
+            tests=tests,
+        )
+
+        message = textwrap.dedent(template).strip().format(**data)
+
+        return message
+
+
+@dataclasses.dataclass
+class Results:
+    """Check results."""
+    comments: t.List[str]
+    labels: t.Dict[str, bool]

ansible_test/_internal/commands/sanity/validate_modules.py

@@ -154,7 +154,11 @@ class ValidateModulesTest(SanitySingleVersion):
                 temp_dir = process_scoped_temporary_directory(args)
 
                 with tarfile.open(path) as file:
-                    file.extractall(temp_dir)
+                    # deprecated: description='extractall fallback without filter' python_version='3.11'
+                    if hasattr(tarfile, 'data_filter'):
+                        file.extractall(temp_dir, filter='data')  # type: ignore[call-arg]
+                    else:
+                        file.extractall(temp_dir)
 
                 cmd.extend([
                     '--original-plugins', temp_dir,

ansible_test/_internal/constants.py

@@ -36,6 +36,7 @@ SECCOMP_CHOICES = [
 # This bin symlink map must exactly match the contents of the bin directory.
 # It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible.
 # It is also used to construct the injector directory at runtime.
+# It is also used to construct entry points when not running ansible-test from source.
 ANSIBLE_BIN_SYMLINK_MAP = {
     'ansible': '../lib/ansible/cli/adhoc.py',
     'ansible-config': '../lib/ansible/cli/config.py',

ansible_test/_internal/delegation.py

@@ -28,7 +28,6 @@ from .util import (
     SubprocessError,
     display,
     filter_args,
-    ANSIBLE_BIN_PATH,
     ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_ROOT,
     OutputStream,
@@ -39,6 +38,10 @@ from .util_common import (
     process_scoped_temporary_directory,
 )
 
+from .ansible_util import (
+    get_ansible_bin_path,
+)
+
 from .containers import (
     support_container_context,
     ContainerDatabase,
@@ -140,7 +143,7 @@ def delegate_command(args, host_state, exclude, require):  # type: (EnvironmentC
             con.extract_archive(chdir=working_directory, src=payload_file)
     else:
         content_root = working_directory
-        ansible_bin_path = ANSIBLE_BIN_PATH
+        ansible_bin_path = get_ansible_bin_path(args)
 
     command = generate_command(args, host_state.controller_profile.python, ansible_bin_path, content_root, exclude, require)
 

ansible_test/_internal/provider/layout/ansible.py

@@ -20,7 +20,7 @@ class AnsibleLayout(LayoutProvider):
     @staticmethod
     def is_content_root(path):  # type: (str) -> bool
         """Return True if the given path is a content root for this provider."""
-        return os.path.exists(os.path.join(path, 'setup.py')) and os.path.exists(os.path.join(path, 'bin/ansible-test'))
+        return os.path.isfile(os.path.join(path, 'pyproject.toml')) and os.path.isdir(os.path.join(path, 'test/lib/ansible_test'))
 
     def create(self, root, paths):  # type: (str, t.List[str]) -> ContentLayout
         """Create a Layout using the given root and paths."""

ansible_test/_internal/provider/source/unversioned.py

@@ -48,9 +48,6 @@ class UnversionedSource(SourceProvider):
             'tests': (
                 'output',
             ),
-            'docs/docsite': (
-                '_build',
-            ),
         }
 
         kill_sub_file = {

ansible_test/_internal/python_requirements.py

@@ -245,6 +245,13 @@ def collect_requirements(
             # installed packages may have run-time dependencies on setuptools
             uninstall_packages.remove('setuptools')
 
+        # hack to allow the package-data sanity test to keep wheel in the venv
+        install_commands = [command for command in commands if isinstance(command, PipInstall)]
+        install_wheel = any(install.has_package('wheel') for install in install_commands)
+
+        if install_wheel:
+            uninstall_packages.remove('wheel')
+
         commands.extend(collect_uninstall(packages=uninstall_packages))
 
     return commands

ansible_test/_internal/util.py

@@ -69,14 +69,12 @@ ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # assume running from install
 ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT)
-ANSIBLE_BIN_PATH = os.path.dirname(os.path.abspath(sys.argv[0]))
 ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible')
 ANSIBLE_SOURCE_ROOT = None
 
 if not os.path.exists(ANSIBLE_LIB_ROOT):
     # running from source
     ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT)))
-    ANSIBLE_BIN_PATH = os.path.join(ANSIBLE_ROOT, 'bin')
     ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible')
     ANSIBLE_SOURCE_ROOT = ANSIBLE_ROOT
 

ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1

@@ -15,7 +15,7 @@
 # To run this script in Powershell:
 #
 # [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
-# $url = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1"
+# $url = "https://raw.githubusercontent.com/ansible/ansible/devel/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1"
 # $file = "$env:temp\ConfigureRemotingForAnsible.ps1"
 #
 # (New-Object -TypeName System.Net.WebClient).DownloadFile($url, $file)

ansible_test/_util/target/setup/bootstrap.sh

@@ -171,6 +171,8 @@ bootstrap_remote_freebsd()
         # Declare platform/python version combinations which do not have supporting OS packages available.
         # For these combinations ansible-test will use pip to install the requirements instead.
         case "${platform_version}/${python_version}" in
+            "12.4/3.9")
+                ;;
             "13.0/3.8")
                 jinja2_pkg=""  # not available
                 cryptography_pkg=""  # not available

ansible_test/_util/target/setup/requirements.py

@@ -134,6 +134,14 @@ def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
         options.extend(packages)
 
         for path, content in requirements:
+            if path.split(os.sep)[0] in ('test', 'requirements'):
+                # Support for pre-build is currently limited to requirements embedded in ansible-test and those used by ansible-core.
+                # Requirements from ansible-core can be found in the 'test' and 'requirements' directories.
+                # This feature will probably be extended to support collections after further testing.
+                # Requirements from collections can be found in the 'tests' directory.
+                for pre_build in parse_pre_build_instructions(content):
+                    pre_build.execute(pip)
+
             write_text_file(os.path.join(tempdir, path), content, True)
             options.extend(['-r', path])
 
@@ -150,6 +158,61 @@ def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
         remove_tree(tempdir)
 
 
+class PreBuild:
+    """Parsed pre-build instructions."""
+
+    def __init__(self, requirement):  # type: (str) -> None
+        self.requirement = requirement
+        self.constraints = []  # type: list[str]
+
+    def execute(self, pip):  # type: (str) -> None
+        """Execute these pre-build instructions."""
+        tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-pre-build')
+
+        try:
+            options = common_pip_options()
+            options.append(self.requirement)
+
+            constraints = '\n'.join(self.constraints) + '\n'
+            constraints_path = os.path.join(tempdir, 'constraints.txt')
+
+            write_text_file(constraints_path, constraints, True)
+
+            env = common_pip_environment()
+            env.update(PIP_CONSTRAINT=constraints_path)
+
+            command = [sys.executable, pip, 'wheel'] + options
+
+            execute_command(command, env=env, cwd=tempdir)
+        finally:
+            remove_tree(tempdir)
+
+
+def parse_pre_build_instructions(requirements):  # type: (str) -> list[PreBuild]
+    """Parse the given pip requirements and return a list of extracted pre-build instructions."""
+    # CAUTION: This code must be kept in sync with the sanity test hashing code in:
+    #          test/lib/ansible_test/_internal/commands/sanity/__init__.py
+
+    pre_build_prefix = '# pre-build '
+    pre_build_requirement_prefix = pre_build_prefix + 'requirement: '
+    pre_build_constraint_prefix = pre_build_prefix + 'constraint: '
+
+    lines = requirements.splitlines()
+    pre_build_lines = [line for line in lines if line.startswith(pre_build_prefix)]
+
+    instructions = []  # type: list[PreBuild]
+
+    for line in pre_build_lines:
+        if line.startswith(pre_build_requirement_prefix):
+            instructions.append(PreBuild(line[len(pre_build_requirement_prefix):]))
+        elif line.startswith(pre_build_constraint_prefix):
+            instructions[-1].constraints.append(line[len(pre_build_constraint_prefix):])
+        else:
+            raise RuntimeError('Unsupported pre-build comment: ' + line)
+
+    return instructions
+
+
 def uninstall(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
     """Perform a pip uninstall."""
     packages = options['packages']