ansible-core 2.13.8rc1__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ansible_test/_internal/metadata.py

@@ -4,6 +4,7 @@ import typing as t
 
 from .util import (
     display,
+    generate_name,
 )
 
 from .io import (
@@ -25,6 +26,7 @@ class Metadata:
         self.cloud_config = None  # type: t.Optional[t.Dict[str, t.Dict[str, t.Union[int, str, bool]]]]
         self.change_description = None  # type: t.Optional[ChangeDescription]
         self.ci_provider = None  # type: t.Optional[str]
+        self.session_id = generate_name()
 
     def populate_changes(self, diff):  # type: (t.Optional[t.List[str]]) -> None
         """Populate the changeset using the given diff."""
@@ -52,6 +54,7 @@ class Metadata:
             cloud_config=self.cloud_config,
             ci_provider=self.ci_provider,
             change_description=self.change_description.to_dict(),
+            session_id=self.session_id,
         )
 
     def to_file(self, path):  # type: (str) -> None
@@ -76,6 +79,7 @@ class Metadata:
         metadata.cloud_config = data['cloud_config']
         metadata.ci_provider = data['ci_provider']
         metadata.change_description = ChangeDescription.from_dict(data['change_description'])
+        metadata.session_id = data['session_id']
 
         return metadata
 

ansible_test/_internal/payload.py

@@ -27,6 +27,7 @@ from .util import (
 
 from .data import (
     data_context,
+    PayloadConfig,
 )
 
 from .util_common import (
@@ -44,11 +45,74 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
         return
 
     files = list(data_context().ansible_source)
-    filters = {}
+    permissions: dict[str, int] = {}
+    filters: dict[str, t.Callable[[tarfile.TarInfo], t.Optional[tarfile.TarInfo]]] = {}
+
+    # Exclude vendored files from the payload.
+    # They may not be compatible with the delegated environment.
+    files = [
+        (abs_path, rel_path) for abs_path, rel_path in files
+        if not rel_path.startswith('lib/ansible/_vendor/')
+        or rel_path == 'lib/ansible/_vendor/__init__.py'
+    ]
+
+    def apply_permissions(tar_info: tarfile.TarInfo, mode: int) -> t.Optional[tarfile.TarInfo]:
+        """
+        Apply the specified permissions to the given file.
+        Existing file type bits are preserved.
+        """
+        tar_info.mode &= ~(stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+        tar_info.mode |= mode
+
+        return tar_info
+
+    def make_executable(tar_info: tarfile.TarInfo) -> t.Optional[tarfile.TarInfo]:
+        """
+        Make the given file executable and readable by all, and writeable by the owner.
+        Existing file type bits are preserved.
+        This ensures consistency of test results when using unprivileged users.
+        """
+        return apply_permissions(
+            tar_info,
+            stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
+            stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH |
+            stat.S_IWUSR
+        )
+
+    def make_non_executable(tar_info: tarfile.TarInfo) -> t.Optional[tarfile.TarInfo]:
+        """
+        Make the given file readable by all, and writeable by the owner.
+        Existing file type bits are preserved.
+        This ensures consistency of test results when using unprivileged users.
+        """
+        return apply_permissions(
+            tar_info,
+            stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
+            stat.S_IWUSR
+        )
+
+    def detect_permissions(tar_info: tarfile.TarInfo) -> t.Optional[tarfile.TarInfo]:
+        """
+        Detect and apply the appropriate permissions for a file.
+        Existing file type bits are preserved.
+        This ensures consistency of test results when using unprivileged users.
+        """
+        if tar_info.path.startswith('ansible/'):
+            mode = permissions.get(os.path.relpath(tar_info.path, 'ansible'))
+        elif data_context().content.collection and is_subdir(tar_info.path, data_context().content.collection.directory):
+            mode = permissions.get(os.path.relpath(tar_info.path, data_context().content.collection.directory))
+        else:
+            mode = None
+
+        if mode:
+            tar_info = apply_permissions(tar_info, mode)
+        elif tar_info.mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):
+            # If any execute bit is set, treat the file as executable.
+            # This ensures that sanity tests which check execute bits behave correctly.
+            tar_info = make_executable(tar_info)
+        else:
+            tar_info = make_non_executable(tar_info)
 
-    def make_executable(tar_info):  # type: (tarfile.TarInfo) -> t.Optional[tarfile.TarInfo]
-        """Make the given file executable."""
-        tar_info.mode |= stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP
         return tar_info
 
     if not ANSIBLE_SOURCE_ROOT:
@@ -85,10 +149,15 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
         # there are no extra files when testing ansible itself
         extra_files = []
 
+    payload_config = PayloadConfig(
+        files=content_files,
+        permissions=permissions,
+    )
+
     for callback in data_context().payload_callbacks:
         # execute callbacks only on the content paths
         # this is done before placing them in the appropriate subdirectory (see below)
-        callback(content_files)
+        callback(payload_config)
 
     # place ansible source files under the 'ansible' directory on the delegated host
     files = [(src, os.path.join('ansible', dst)) for src, dst in files]
@@ -109,7 +178,7 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
     with tarfile.open(dst_path, mode='w:gz', compresslevel=4, format=tarfile.GNU_FORMAT) as tar:
         for src, dst in files:
             display.info('%s -> %s' % (src, dst), verbosity=4)
-            tar.add(src, dst, filter=filters.get(dst))
+            tar.add(src, dst, filter=filters.get(dst, detect_permissions))
 
     duration = time.time() - start
     payload_size_bytes = os.path.getsize(dst_path)

ansible_test/_internal/python_requirements.py

@@ -52,6 +52,7 @@ from .data import (
 from .host_configs import (
     PosixConfig,
     PythonConfig,
+    VirtualPythonConfig,
 )
 
 from .connections import (
@@ -259,6 +260,20 @@ def run_pip(
     connection = connection or LocalConnection(args)
     script = prepare_pip_script(commands)
 
+    if isinstance(args, IntegrationConfig):
+        # Integration tests can involve two hosts (controller and target).
+        # The connection type can be used to disambiguate between the two.
+        context = " (controller)" if isinstance(connection, LocalConnection) else " (target)"
+    else:
+        context = ""
+
+    if isinstance(python, VirtualPythonConfig):
+        context += " [venv]"
+
+    # The interpreter path is not included below.
+    # It can be seen by running ansible-test with increased verbosity (showing all commands executed).
+    display.info(f'Installing requirements for Python {python.version}{context}')
+
     if not args.explain:
         try:
             connection.run([python.path], data=script, capture=False)

ansible_test/_internal/target.py

@@ -623,13 +623,9 @@ class IntegrationTarget(CompletionTarget):
         if 'needs/httptester' in groups:
             groups.append('cloud/httptester')  # backwards compatibility for when it was not a cloud plugin
 
-        if '_' in self.name:
-            prefix = self.name[:self.name.find('_')]
-        else:
-            prefix = None
-
-        if prefix in prefixes:
-            group = prefixes[prefix]
+        for prefix, group in prefixes.items():
+            if not self.name.startswith(f'{prefix}_'):
+                continue
 
             if group != prefix:
                 group = '%s/%s' % (group, prefix)

ansible_test/_internal/test.py

@@ -46,7 +46,7 @@ def calculate_confidence(path, line, metadata):  # type: (str, int, Metadata) ->
         return 0
 
     # changes were made to the same file and line
-    if any(r[0] <= line <= r[1] in r for r in ranges):
+    if any(r[0] <= line <= r[1] for r in ranges):
         return 100
 
     # changes were made to the same file and the line number is unknown

ansible_test/_internal/util.py

@@ -23,10 +23,14 @@ import time
 import functools
 import shlex
 import typing as t
+import warnings
 
 from struct import unpack, pack
 from termios import TIOCGWINSZ
 
+# CAUTION: Avoid third-party imports in this module whenever possible.
+#          Any third-party imports occurring here will result in an error if they are vendored by ansible-core.
+
 try:
     from typing_extensions import TypeGuard  # TypeGuard was added in Python 3.9
 except ImportError:
@@ -333,6 +337,17 @@ def get_ansible_version():  # type: () -> str
     return ansible_version
 
 
+def _enable_vendoring() -> None:
+    """Enable support for loading Python packages vendored by ansible-core."""
+    # Load the vendoring code by file path, since ansible may not be in our sys.path.
+    # Convert warnings into errors, to avoid problems from surfacing later.
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings('error')
+
+        load_module(os.path.join(ANSIBLE_LIB_ROOT, '_vendor', '__init__.py'), 'ansible_vendor')
+
+
 @cache
 def get_available_python_versions():  # type: () -> t.Dict[str, str]
     """Return a dictionary indicating which supported Python versions are available."""
@@ -1133,3 +1148,5 @@ def type_guard(sequence: t.Sequence[t.Any], guard_type: t.Type[C]) -> TypeGuard[
 
 
 display = Display()  # pylint: disable=locally-disabled, invalid-name
+
+_enable_vendoring()

ansible_test/_util/controller/sanity/mypy/ansible-test.ini

@@ -14,6 +14,9 @@ disable_error_code = misc
 [mypy-argcomplete]
 ignore_missing_imports = True
 
+[mypy-argcomplete.finders]
+ignore_missing_imports = True
+
 [mypy-coverage]
 ignore_missing_imports = True
 

ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py

@@ -25,9 +25,7 @@ import json
 import errno
 import os
 import re
-import subprocess
 import sys
-import tempfile
 import traceback
 import warnings
 
@@ -307,8 +305,8 @@ class ModuleValidator(Validator):
 
     ACCEPTLIST_FUTURE_IMPORTS = frozenset(('absolute_import', 'division', 'print_function'))
 
-    def __init__(self, path, analyze_arg_spec=False, collection=None, collection_version=None,
-                 base_branch=None, git_cache=None, reporter=None, routing=None, plugin_type='module'):
+    def __init__(self, path, git_cache: GitCache, analyze_arg_spec=False, collection=None, collection_version=None,
+                 reporter=None, routing=None, plugin_type='module'):
         super(ModuleValidator, self).__init__(reporter=reporter or Reporter())
 
         self.path = path
@@ -334,8 +332,8 @@ class ModuleValidator(Validator):
             self.collection_version_str = collection_version
             self.collection_version = SemanticVersion(collection_version)
 
-        self.base_branch = base_branch
-        self.git_cache = git_cache or GitCache()
+        self.git_cache = git_cache
+        self.base_module = self.git_cache.get_original_path(self.path)
 
         self._python_module_override = False
 
@@ -347,11 +345,6 @@ class ModuleValidator(Validator):
         except Exception:
             self.ast = None
 
-        if base_branch:
-            self.base_module = self._get_base_file()
-        else:
-            self.base_module = None
-
     def _create_version(self, v, collection_name=None):
         if not v:
             raise ValueError('Empty string is not a valid version')
@@ -374,13 +367,7 @@ class ModuleValidator(Validator):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        if not self.base_module:
-            return
-
-        try:
-            os.remove(self.base_module)
-        except Exception:
-            pass
+        pass
 
     @property
     def object_name(self):
@@ -423,35 +410,9 @@ class ModuleValidator(Validator):
         except AttributeError:
             return False
 
-    def _get_base_branch_module_path(self):
-        """List all paths within lib/ansible/modules to try and match a moved module"""
-        return self.git_cache.base_module_paths.get(self.object_name)
-
-    def _has_alias(self):
-        """Return true if the module has any aliases."""
-        return self.object_name in self.git_cache.head_aliased_modules
-
-    def _get_base_file(self):
-        # In case of module moves, look for the original location
-        base_path = self._get_base_branch_module_path()
-
-        command = ['git', 'show', '%s:%s' % (self.base_branch, base_path or self.path)]
-        p = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, check=False)
-
-        if int(p.returncode) != 0:
-            return None
-
-        t = tempfile.NamedTemporaryFile(delete=False)
-        t.write(p.stdout)
-        t.close()
-
-        return t.name
-
-    def _is_new_module(self):
-        if self._has_alias():
-            return False
-
-        return not self.object_name.startswith('_') and bool(self.base_branch) and not bool(self.base_module)
+    def _is_new_module(self) -> bool | None:
+        """Return True if the content is new, False if it is not and None if the information is not available."""
+        return self.git_cache.is_new(self.path)
 
     def _check_interpreter(self, powershell=False):
         if powershell:
@@ -2006,7 +1967,7 @@ class ModuleValidator(Validator):
                 )
 
     def _check_for_new_args(self, doc):
-        if not self.base_branch or self._is_new_module():
+        if not self.base_module:
             return
 
         with CaptureStd():
@@ -2238,7 +2199,7 @@ class ModuleValidator(Validator):
             # We can only validate PowerShell arg spec if it is using the new Ansible.Basic.AnsibleModule util
             pattern = r'(?im)^#\s*ansiblerequires\s+\-csharputil\s*Ansible\.Basic'
             if re.search(pattern, self.text) and self.object_name not in self.PS_ARG_VALIDATE_REJECTLIST:
-                with ModuleValidator(docs_path, base_branch=self.base_branch, git_cache=self.git_cache) as docs_mv:
+                with ModuleValidator(docs_path, git_cache=self.git_cache) as docs_mv:
                     docs = docs_mv._validate_docs()[1]
                     self._validate_ansible_module_call(docs)
 
@@ -2283,6 +2244,84 @@ class PythonPackageValidator(Validator):
             )
 
 
+class GitCache(metaclass=abc.ABCMeta):
+    """Base class for access to original files."""
+    @abc.abstractmethod
+    def get_original_path(self, path: str) -> str | None:
+        """Return the path to the original version of the specified file, or None if there isn't one."""
+
+    @abc.abstractmethod
+    def is_new(self, path: str) -> bool | None:
+        """Return True if the content is new, False if it is not and None if the information is not available."""
+
+    @staticmethod
+    def create(original_plugins: str | None, plugin_type: str) -> GitCache:
+        return CoreGitCache(original_plugins, plugin_type) if original_plugins else NoOpGitCache()
+
+
+class CoreGitCache(GitCache):
+    """Provides access to original files when testing core."""
+    def __init__(self, original_plugins: str | None, plugin_type: str) -> None:
+        super().__init__()
+
+        self.original_plugins = original_plugins
+
+        rel_path = 'lib/ansible/modules/' if plugin_type == 'module' else f'lib/ansible/plugins/{plugin_type}/'
+        head_tree = self._find_files(rel_path)
+
+        head_aliased_modules = set()
+
+        for path in head_tree:
+            filename = os.path.basename(path)
+
+            if filename.startswith('_') and filename != '__init__.py':
+                if os.path.islink(path):
+                    head_aliased_modules.add(os.path.basename(os.path.realpath(path)))
+
+        self._head_aliased_modules = head_aliased_modules
+
+    def get_original_path(self, path: str) -> str | None:
+        """Return the path to the original version of the specified file, or None if there isn't one."""
+        path = os.path.join(self.original_plugins, path)
+
+        if not os.path.exists(path):
+            path = None
+
+        return path
+
+    def is_new(self, path: str) -> bool | None:
+        """Return True if the content is new, False if it is not and None if the information is not available."""
+        if os.path.basename(path).startswith('_'):
+            return False
+
+        if os.path.basename(path) in self._head_aliased_modules:
+            return False
+
+        return not self.get_original_path(path)
+
+    @staticmethod
+    def _find_files(path: str) -> list[str]:
+        """Return a list of files found in the specified directory."""
+        paths = []
+
+        for (dir_path, dir_names, file_names) in os.walk(path):
+            for file_name in file_names:
+                paths.append(os.path.join(dir_path, file_name))
+
+        return sorted(paths)
+
+
+class NoOpGitCache(GitCache):
+    """Provides a no-op interface for access to original files."""
+    def get_original_path(self, path: str) -> str | None:
+        """Return the path to the original version of the specified file, or None if there isn't one."""
+        return None
+
+    def is_new(self, path: str) -> bool | None:
+        """Return True if the content is new, False if it is not and None if the information is not available."""
+        return None
+
+
 def re_compile(value):
     """
     Argparse expects things to raise TypeError, re.compile raises an re.error
@@ -2308,8 +2347,6 @@ def run():
                         type=re_compile)
     parser.add_argument('--arg-spec', help='Analyze module argument spec',
                         action='store_true', default=False)
-    parser.add_argument('--base-branch', default=None,
-                        help='Used in determining if new options were added')
     parser.add_argument('--format', choices=['json', 'plain'], default='plain',
                         help='Output format. Default: "%(default)s"')
     parser.add_argument('--output', default='-',
@@ -2326,13 +2363,14 @@ def run():
     parser.add_argument('--plugin-type',
                         default='module',
                         help='The plugin type to validate. Defaults to %(default)s')
+    parser.add_argument('--original-plugins')
 
     args = parser.parse_args()
 
     args.plugins = [m.rstrip('/') for m in args.plugins]
 
     reporter = Reporter()
-    git_cache = GitCache(args.base_branch, args.plugin_type)
+    git_cache = GitCache.create(args.original_plugins, args.plugin_type)
 
     check_dirs = set()
 
@@ -2357,7 +2395,7 @@ def run():
             if ModuleValidator.is_on_rejectlist(path):
                 continue
             with ModuleValidator(path, collection=args.collection, collection_version=args.collection_version,
-                                 analyze_arg_spec=args.arg_spec, base_branch=args.base_branch,
+                                 analyze_arg_spec=args.arg_spec,
                                  git_cache=git_cache, reporter=reporter, routing=routing,
                                  plugin_type=args.plugin_type) as mv1:
                 mv1.validate()
@@ -2382,7 +2420,7 @@ def run():
                 if ModuleValidator.is_on_rejectlist(path):
                     continue
                 with ModuleValidator(path, collection=args.collection, collection_version=args.collection_version,
-                                     analyze_arg_spec=args.arg_spec, base_branch=args.base_branch,
+                                     analyze_arg_spec=args.arg_spec,
                                      git_cache=git_cache, reporter=reporter, routing=routing,
                                      plugin_type=args.plugin_type) as mv2:
                     mv2.validate()
@@ -2398,78 +2436,6 @@ def run():
         sys.exit(reporter.json(warnings=args.warnings, output=args.output))
 
 
-class GitCache:
-    def __init__(self, base_branch, plugin_type):
-        self.base_branch = base_branch
-        self.plugin_type = plugin_type
-
-        self.rel_path = 'lib/ansible/modules/'
-        if plugin_type != 'module':
-            self.rel_path = 'lib/ansible/plugins/%s/' % plugin_type
-
-        if self.base_branch:
-            self.base_tree = self._git(['ls-tree', '-r', '--name-only', self.base_branch, self.rel_path])
-        else:
-            self.base_tree = []
-
-        try:
-            self.head_tree = self._git(['ls-tree', '-r', '--name-only', 'HEAD', self.rel_path])
-        except GitError as ex:
-            if ex.status == 128:
-                # fallback when there is no .git directory
-                self.head_tree = self._get_module_files()
-            else:
-                raise
-        except OSError as ex:
-            if ex.errno == errno.ENOENT:
-                # fallback when git is not installed
-                self.head_tree = self._get_module_files()
-            else:
-                raise
-
-        allowed_exts = ('.py', '.ps1')
-        if plugin_type != 'module':
-            allowed_exts = ('.py', )
-        self.base_module_paths = dict((os.path.basename(p), p) for p in self.base_tree if os.path.splitext(p)[1] in allowed_exts)
-
-        self.base_module_paths.pop('__init__.py', None)
-
-        self.head_aliased_modules = set()
-
-        for path in self.head_tree:
-            filename = os.path.basename(path)
-
-            if filename.startswith('_') and filename != '__init__.py':
-                if os.path.islink(path):
-                    self.head_aliased_modules.add(os.path.basename(os.path.realpath(path)))
-
-    def _get_module_files(self):
-        module_files = []
-
-        for (dir_path, dir_names, file_names) in os.walk(self.rel_path):
-            for file_name in file_names:
-                module_files.append(os.path.join(dir_path, file_name))
-
-        return module_files
-
-    @staticmethod
-    def _git(args):
-        cmd = ['git'] + args
-        p = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False)
-
-        if p.returncode != 0:
-            raise GitError(p.stderr, p.returncode)
-
-        return p.stdout.splitlines()
-
-
-class GitError(Exception):
-    def __init__(self, message, status):
-        super(GitError, self).__init__(message)
-
-        self.status = status
-
-
 def main():
     try:
         run()