ansible-core 2.19.0b4__py3-none-any.whl → 2.19.0b6__py3-none-any.whl

ansible/executor/module_common.py

@@ -37,9 +37,12 @@ from ast import AST, Import, ImportFrom
 from io import BytesIO
 
 from ansible._internal import _locking
+from ansible._internal._ansiballz import _builder
+from ansible._internal import _ansiballz
 from ansible._internal._datatag import _utils
 from ansible.module_utils._internal import _dataclass_validation
 from ansible.module_utils.common.yaml import yaml_load
+from ansible.module_utils.datatag import deprecator_from_collection_name
 from ansible._internal._datatag._tags import Origin
 from ansible.module_utils.common.json import Direction, get_module_encoder
 from ansible.release import __version__, __author__
@@ -53,9 +56,9 @@ from ansible.plugins.loader import module_utils_loader
 from ansible._internal._templating._engine import TemplateOptions, TemplateEngine
 from ansible.template import Templar
 from ansible.utils.collection_loader._collection_finder import _get_collection_metadata, _nested_dict_get
-from ansible.module_utils._internal import _json, _ansiballz
+from ansible.module_utils._internal import _json
+from ansible.module_utils._internal._ansiballz import _loader
 from ansible.module_utils import basic as _basic
-from ansible.module_utils.common import messages as _messages
 
 if t.TYPE_CHECKING:
     from ansible import template as _template
@@ -117,7 +120,7 @@ def _strip_comments(source: str) -> str:
 
 
 def _read_ansiballz_code() -> str:
-    code = (pathlib.Path(__file__).parent.parent / '_internal/_ansiballz.py').read_text()
+    code = (pathlib.Path(_ansiballz.__file__).parent / '_wrapper.py').read_text()
 
     if not C.DEFAULT_KEEP_REMOTE_FILES:
         # Keep comments when KEEP_REMOTE_FILES is set.  That way users will see
@@ -166,7 +169,7 @@ NEW_STYLE_PYTHON_MODULE_RE = re.compile(
 
 
 class ModuleDepFinder(ast.NodeVisitor):
-    # DTFIX-RELEASE: add support for ignoring imports with a "controller only" comment, this will allow replacing import_controller_module with standard imports
+    # DTFIX-FUTURE: add support for ignoring imports with a "controller only" comment, this will allow replacing import_controller_module with standard imports
     def __init__(self, module_fqn, tree, is_pkg_init=False, *args, **kwargs):
         """
         Walk the ast tree for the python module.
@@ -439,7 +442,7 @@ class ModuleUtilLocatorBase:
                 version=removal_version,
                 removed=removed,
                 date=removal_date,
-                deprecator=_messages.PluginInfo._from_collection_name(self._collection_name),
+                deprecator=deprecator_from_collection_name(self._collection_name),
             )
         if 'redirect' in routing_entry:
             self.redirected = True
@@ -618,7 +621,7 @@ class CollectionModuleUtilLocator(ModuleUtilLocatorBase):
         if pkg_path:
             origin = Origin(path=os.path.join(pkg_path, src_path))
         else:
-            # DTFIX-RELEASE: not sure if this case is even reachable
+            # DTFIX-FUTURE: not sure if this case is even reachable
             origin = Origin(description=f'<synthetic collection package for {collection_pkg_name}!r>')
 
         self.source_code = origin.tag(src)
@@ -658,7 +661,7 @@ metadata_versions: dict[t.Any, type[ModuleMetadata]] = {
 
 
 def _get_module_metadata(module: ast.Module) -> ModuleMetadata:
-    # DTFIX-RELEASE: while module metadata works, this feature isn't fully baked and should be turned off before release
+    # DTFIX2: while module metadata works, this feature isn't fully baked and should be turned off before release
     metadata_nodes: list[ast.Assign] = []
 
     for node in module.body:
@@ -709,7 +712,14 @@ def _get_module_metadata(module: ast.Module) -> ModuleMetadata:
     return metadata
 
 
-def recursive_finder(name: str, module_fqn: str, module_data: str | bytes, zf: zipfile.ZipFile, date_time: datetime.datetime) -> ModuleMetadata:
+def recursive_finder(
+    name: str,
+    module_fqn: str,
+    module_data: str | bytes,
+    zf: zipfile.ZipFile,
+    date_time: datetime.datetime,
+    extension_manager: _builder.ExtensionManager,
+) -> ModuleMetadata:
     """
     Using ModuleDepFinder, make sure we have all of the module_utils files that
     the module and its module_utils files needs. (no longer actually recursive)
@@ -755,12 +765,14 @@ def recursive_finder(name: str, module_fqn: str, module_data: str | bytes, zf: z
 
     # include module_utils that are always required
     modules_to_process.extend((
-        _ModuleUtilsProcessEntry.from_module(_ansiballz),
+        _ModuleUtilsProcessEntry.from_module(_loader),
         _ModuleUtilsProcessEntry.from_module(_basic),
         _ModuleUtilsProcessEntry.from_module_name(_json.get_module_serialization_profile_module_name(profile, True)),
         _ModuleUtilsProcessEntry.from_module_name(_json.get_module_serialization_profile_module_name(profile, False)),
     ))
 
+    modules_to_process.extend(_ModuleUtilsProcessEntry.from_module_name(name) for name in extension_manager.module_names)
+
     module_info: ModuleUtilLocatorBase
 
     # we'll be adding new modules inline as we discover them, so just keep going til we've processed them all
@@ -815,12 +827,13 @@ def recursive_finder(name: str, module_fqn: str, module_data: str | bytes, zf: z
                 modules_to_process.append(_ModuleUtilsProcessEntry(normalized_name, False, module_info.redirected, is_optional=entry.is_optional))
 
     for py_module_name in py_module_cache:
-        py_module_file_name = py_module_cache[py_module_name][1]
+        source_code, py_module_file_name = py_module_cache[py_module_name]
+
+        zf.writestr(_make_zinfo(py_module_file_name, date_time, zf=zf), source_code)
+
+        if extension_manager.debugger_enabled and (origin := Origin.get_tag(source_code)) and origin.path:
+            extension_manager.source_mapping[origin.path] = py_module_file_name
 
-        zf.writestr(
-            _make_zinfo(py_module_file_name, date_time, zf=zf),
-            py_module_cache[py_module_name][0]
-        )
         mu_file = to_text(py_module_file_name, errors='surrogate_or_strict')
         display.vvvvv("Including module_utils file %s" % mu_file)
 
@@ -879,17 +892,27 @@ def _get_ansible_module_fqn(module_path):
     return remote_module_fqn
 
 
-def _add_module_to_zip(zf: zipfile.ZipFile, date_time: datetime.datetime, remote_module_fqn: str, b_module_data: bytes) -> None:
+def _add_module_to_zip(
+    zf: zipfile.ZipFile,
+    date_time: datetime.datetime,
+    remote_module_fqn: str,
+    b_module_data: bytes,
+    module_path: str,
+    extension_manager: _builder.ExtensionManager,
+) -> None:
     """Add a module from ansible or from an ansible collection into the module zip"""
     module_path_parts = remote_module_fqn.split('.')
 
     # Write the module
-    module_path = '/'.join(module_path_parts) + '.py'
+    zip_module_path = '/'.join(module_path_parts) + '.py'
     zf.writestr(
-        _make_zinfo(module_path, date_time, zf=zf),
+        _make_zinfo(zip_module_path, date_time, zf=zf),
         b_module_data
     )
 
+    if extension_manager.debugger_enabled:
+        extension_manager.source_mapping[module_path] = zip_module_path
+
     existing_paths: frozenset[str]
 
     # Write the __init__.py's necessary to get there
@@ -928,10 +951,12 @@ class _BuiltModule:
 class _CachedModule:
     """Cached Python module created by AnsiballZ."""
 
-    # DTFIX-RELEASE: secure this (locked down pickle, don't use pickle, etc.)
+    # DTFIX5: secure this (locked down pickle, don't use pickle, etc.)
 
     zip_data: bytes
     metadata: ModuleMetadata
+    source_mapping: dict[str, str]
+    """A mapping of controller absolute source locations to target relative source locations within the AnsiballZ payload."""
 
     def dump(self, path: str) -> None:
         temp_path = pathlib.Path(path + '-part')
@@ -991,10 +1016,8 @@ def _find_module_utils(
         module_substyle = 'powershell'
         b_module_data = b_module_data.replace(REPLACER_WINDOWS, b'#AnsibleRequires -PowerShell Ansible.ModuleUtils.Legacy')
     elif re.search(b'#Requires -Module', b_module_data, re.IGNORECASE) \
-            or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE)\
-            or re.search(b'#AnsibleRequires -OSVersion', b_module_data, re.IGNORECASE) \
-            or re.search(b'#AnsibleRequires -Powershell', b_module_data, re.IGNORECASE) \
-            or re.search(b'#AnsibleRequires -CSharpUtil', b_module_data, re.IGNORECASE):
+            or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE) \
+            or re.search(b'#AnsibleRequires -(OSVersion|PowerShell|CSharpUtil|Wrapper)', b_module_data, re.IGNORECASE):
         module_style = 'new'
         module_substyle = 'powershell'
     elif REPLACER_JSONARGS in b_module_data:
@@ -1031,6 +1054,7 @@ def _find_module_utils(
 
     if module_substyle == 'python':
         date_time = datetime.datetime.now(datetime.timezone.utc)
+
         if date_time.year < 1980:
             raise AnsibleError(f'Cannot create zipfile due to pre-1980 configured date: {date_time}')
 
@@ -1040,19 +1064,19 @@ def _find_module_utils(
             display.warning(u'Bad module compression string specified: %s.  Using ZIP_STORED (no compression)' % module_compression)
             compression_method = zipfile.ZIP_STORED
 
+        extension_manager = _builder.ExtensionManager.create(task_vars=task_vars)
+        extension_key = '~'.join(extension_manager.extension_names) if extension_manager.extension_names else 'none'
         lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache')  # type: ignore[attr-defined]
-        cached_module_filename = os.path.join(lookup_path, "%s-%s" % (remote_module_fqn, module_compression))
+        cached_module_filename = os.path.join(lookup_path, '-'.join((remote_module_fqn, module_compression, extension_key)))
 
         os.makedirs(os.path.dirname(cached_module_filename), exist_ok=True)
 
-        zipdata: bytes | None = None
-        module_metadata: ModuleMetadata | None = None
+        cached_module: _CachedModule | None = None
 
         # Optimization -- don't lock if the module has already been cached
         if os.path.exists(cached_module_filename):
             display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename)
             cached_module = _CachedModule.load(cached_module_filename)
-            zipdata, module_metadata = cached_module.zip_data, cached_module.metadata
         else:
             display.debug('ANSIBALLZ: Acquiring lock')
             lock_path = f'{cached_module_filename}.lock'
@@ -1067,35 +1091,40 @@ def _find_module_utils(
                     zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
 
                     # walk the module imports, looking for module_utils to send- they'll be added to the zipfile
-                    module_metadata = recursive_finder(module_name, remote_module_fqn, Origin(path=module_path).tag(b_module_data), zf, date_time)
+                    module_metadata = recursive_finder(
+                        module_name,
+                        remote_module_fqn,
+                        Origin(path=module_path).tag(b_module_data),
+                        zf,
+                        date_time,
+                        extension_manager,
+                    )
 
                     display.debug('ANSIBALLZ: Writing module into payload')
-                    _add_module_to_zip(zf, date_time, remote_module_fqn, b_module_data)
+                    _add_module_to_zip(zf, date_time, remote_module_fqn, b_module_data, module_path, extension_manager)
 
                     zf.close()
-                    zipdata = base64.b64encode(zipoutput.getvalue())
+                    zip_data = base64.b64encode(zipoutput.getvalue())
 
                     # Write the assembled module to a temp file (write to temp
                     # so that no one looking for the file reads a partially
                     # written file)
                     os.makedirs(lookup_path, exist_ok=True)
                     display.debug('ANSIBALLZ: Writing module')
-                    cached_module = _CachedModule(zip_data=zipdata, metadata=module_metadata)
+                    cached_module = _CachedModule(zip_data=zip_data, metadata=module_metadata, source_mapping=extension_manager.source_mapping)
                     cached_module.dump(cached_module_filename)
                     display.debug('ANSIBALLZ: Done creating module')
 
-            if not zipdata:
+            if not cached_module:
                 display.debug('ANSIBALLZ: Reading module after lock')
                 # Another process wrote the file while we were waiting for
                 # the write lock.  Go ahead and read the data from disk
                 # instead of re-creating it.
                 try:
                     cached_module = _CachedModule.load(cached_module_filename)
-                except IOError:
+                except OSError as ex:
                     raise AnsibleError('A different worker process failed to create module file. '
-                                       'Look at traceback for that process for debugging information.')
-
-                zipdata, module_metadata = cached_module.zip_data, cached_module.metadata
+                                       'Look at traceback for that process for debugging information.') from ex
 
         o_interpreter, o_args = _extract_interpreter(b_module_data)
         if o_interpreter is None:
@@ -1109,40 +1138,36 @@ def _find_module_utils(
         if not isinstance(rlimit_nofile, int):
             rlimit_nofile = int(templar._engine.template(rlimit_nofile, options=TemplateOptions(value_for_omit=0)))
 
-        coverage_config = os.environ.get('_ANSIBLE_COVERAGE_CONFIG')
-
-        if coverage_config:
-            coverage_output = os.environ['_ANSIBLE_COVERAGE_OUTPUT']
-        else:
-            coverage_output = None
-
-        if not isinstance(module_metadata, ModuleMetadataV1):
+        if not isinstance(cached_module.metadata, ModuleMetadataV1):
             raise NotImplementedError()
 
         params = dict(ANSIBLE_MODULE_ARGS=module_args,)
-        encoder = get_module_encoder(module_metadata.serialization_profile, Direction.CONTROLLER_TO_MODULE)
+        encoder = get_module_encoder(cached_module.metadata.serialization_profile, Direction.CONTROLLER_TO_MODULE)
+
         try:
             encoded_params = json.dumps(params, cls=encoder)
         except TypeError as ex:
             raise AnsibleError(f'Failed to serialize arguments for the {module_name!r} module.') from ex
 
+        extension_manager.source_mapping = cached_module.source_mapping
+
         code = _get_ansiballz_code(shebang)
         args = dict(
-            zipdata=to_text(zipdata),
             ansible_module=module_name,
             module_fqn=remote_module_fqn,
-            params=encoded_params,
-            profile=module_metadata.serialization_profile,
+            profile=cached_module.metadata.serialization_profile,
             date_time=date_time,
-            coverage_config=coverage_config,
-            coverage_output=coverage_output,
             rlimit_nofile=rlimit_nofile,
+            params=encoded_params,
+            extensions=extension_manager.get_extensions(),
+            zip_data=to_text(cached_module.zip_data),
         )
 
         args_string = '\n'.join(f'{key}={value!r},' for key, value in args.items())
 
         wrapper = f"""{code}
 
+
 if __name__ == "__main__":
     _ansiballz_main(
 {args_string}
@@ -1151,6 +1176,7 @@ if __name__ == "__main__":
 
         output.write(to_bytes(wrapper))
 
+        module_metadata = cached_module.metadata
         b_module_data = output.getvalue()
 
     elif module_substyle == 'powershell':

ansible/executor/powershell/async_watchdog.ps1

@@ -40,7 +40,7 @@ param([ScriptBlock]$ScriptBlock, $Param)
 & $ScriptBlock.Ast.GetScriptBlock() @Param
 '@).AddParameters(
     @{
-        ScriptBlock = $execInfo.ScriptBlock
+        ScriptBlock = $execInfo.ScriptInfo.ScriptBlock
         Param = $execInfo.Parameters
     })
 
@@ -64,7 +64,7 @@ $jobError = $null
 try {
     $jobAsyncResult = $ps.BeginInvoke($pipelineInput, $invocationSettings, $null, $null)
     $jobAsyncResult.AsyncWaitHandle.WaitOne($Timeout * 1000) > $null
-    $result.finished = 1
+    $result.finished = $true
 
     if ($jobAsyncResult.IsCompleted) {
         $jobOutput = $ps.EndInvoke($jobAsyncResult)

ansible/executor/powershell/async_wrapper.ps1

@@ -113,7 +113,7 @@ try {
     }
     $execWrapper = @{
         name = 'exec_wrapper-async.ps1'
-        script = $execAction.Script
+        script = $execAction.ScriptInfo.Script
         params = $execAction.Parameters
     } | ConvertTo-Json -Compress -Depth 99
     $asyncInput = "$execWrapper`n`0`0`0`0`n$($execAction.InputData)"
@@ -135,8 +135,8 @@ try {
     # We need to write the result file before the process is started to ensure
     # it can read the file.
     $result = @{
-        started = 1
-        finished = 0
+        started = $true
+        finished = $false
         results_file = $resultsPath
         ansible_job_id = $localJid
         _ansible_suppress_tmpdir_delete = $true

ansible/executor/powershell/become_wrapper.ps1

@@ -7,6 +7,7 @@ using namespace System.Collections
 using namespace System.Diagnostics
 using namespace System.IO
 using namespace System.Management.Automation
+using namespace System.Management.Automation.Security
 using namespace System.Net
 using namespace System.Text
 
@@ -53,7 +54,7 @@ $executablePath = Join-Path -Path $PSHome -ChildPath $executable
 $actionInfo = Get-AnsibleExecWrapper -EncodeInputOutput
 $bootstrapManifest = ConvertTo-Json -InputObject @{
     n = "exec_wrapper-become-$([Guid]::NewGuid()).ps1"
-    s = $actionInfo.Script
+    s = $actionInfo.ScriptInfo.Script
     p = $actionInfo.Parameters
 } -Depth 99 -Compress
 
@@ -68,9 +69,26 @@ $m=foreach($i in $input){
 $m=$m|ConvertFrom-Json
 $p=@{}
 foreach($o in $m.p.PSObject.Properties){$p[$o.Name]=$o.Value}
+'@
+
+if ([SystemPolicy]::GetSystemLockdownPolicy() -eq 'Enforce') {
+    # If we started in CLM we need to execute the script from a file so that
+    # PowerShell validates our exec_wrapper is trusted and will run in FLM.
+    $command += @'
+$n=Join-Path $env:TEMP $m.n
+$null=New-Item $n -Value $m.s -Type File -Force
+try{$input|&$n @p}
+finally{if(Test-Path -LiteralPath $n){Remove-Item -LiteralPath $n -Force}}
+'@
+}
+else {
+    # If we started in FLM we pass the script through stdin and execute in
+    # memory.
+    $command += @'
 $c=[System.Management.Automation.Language.Parser]::ParseInput($m.s,$m.n,[ref]$null,[ref]$null).GetScriptBlock()
-$input | & $c @p
+$input|&$c @p
 '@
+}
 
 # Strip out any leading or trailing whitespace and remove empty lines.
 $command = @(

ansible/executor/powershell/bootstrap_wrapper.ps1

@@ -18,10 +18,32 @@ foreach ($obj in $code.params.PSObject.Properties) {
     $splat[$obj.Name] = $obj.Value
 }
 
-$cmd = [System.Management.Automation.Language.Parser]::ParseInput(
-    $code.script,
-    "$($code.name).ps1", # Name is used in stack traces.
-    [ref]$null,
-    [ref]$null).GetScriptBlock()
+$filePath = $null
+try {
+    $cmd = if ($ExecutionContext.SessionState.LanguageMode -eq 'FullLanguage') {
+        # In FLM we can just invoke the code as a scriptblock without touching the
+        # disk.
+        [System.Management.Automation.Language.Parser]::ParseInput(
+            $code.script,
+            "$($code.name).ps1", # Name is used in stack traces.
+            [ref]$null,
+            [ref]$null).GetScriptBlock()
+    }
+    else {
+        # CLM needs to execute code from a file for it to run in FLM when trusted.
+        # Set-Item on 5.1 doesn't have a way to use UTF-8 without a BOM but luckily
+        # New-Item does that by default for both 5.1 and 7. We need to ensure we
+        # use UTF-8 without BOM so the signature is correct.
+        $filePath = Join-Path -Path $env:TEMP -ChildPath "$($code.name)-$(New-Guid).ps1"
+        $null = New-Item -Path $filePath -Value $code.script -ItemType File -Force
+
+        $filePath
+    }
 
-$input | & $cmd @splat
+    $input | & $cmd @splat
+}
+finally {
+    if ($filePath -and (Test-Path -LiteralPath $filePath)) {
+        Remove-Item -LiteralPath $filePath -Force
+    }
+}

ansible/executor/powershell/coverage_wrapper.ps1

@@ -28,8 +28,12 @@ $Host.Runspace.Debugger.SetDebugMode([DebugModes]::RemoteScript)
 Function New-CoverageBreakpointsForScriptBlock {
     Param (
         [Parameter(Mandatory)]
-        [ScriptBlock]
-        $ScriptBlock,
+        [string]
+        $ScriptName,
+
+        [Parameter(Mandatory)]
+        [ScriptBlockAst]
+        $ScriptBlockAst,
 
         [Parameter(Mandatory)]
         [String]
@@ -39,7 +43,7 @@ Function New-CoverageBreakpointsForScriptBlock {
     $predicate = {
         $args[0] -is [CommandBaseAst]
     }
-    $scriptCmds = $ScriptBlock.Ast.FindAll($predicate, $true)
+    $scriptCmds = $ScriptBlockAst.FindAll($predicate, $true)
 
     # Create an object that tracks the Ansible path of the file and the breakpoints that have been set in it
     $info = [PSCustomObject]@{
@@ -68,7 +72,7 @@ Function New-CoverageBreakpointsForScriptBlock {
         }
 
         # Action is explicitly $null as it will slow down the runtime quite dramatically.
-        $b = $lineCtor.Invoke(@($ScriptBlock.File, $cmd.Extent.StartLineNumber, $cmd.Extent.StartColumnNumber, $null))
+        $b = $lineCtor.Invoke(@($ScriptName, $cmd.Extent.StartLineNumber, $cmd.Extent.StartColumnNumber, $null))
         $info.Breakpoints.Add($b)
     }
 
@@ -102,11 +106,16 @@ try {
     $coveragePathFilter = $PathFilter.Split(":", [StringSplitOptions]::RemoveEmptyEntries)
     $breakpointInfo = @(
         foreach ($scriptName in @($ModuleName; $actionParams.PowerShellModules)) {
-            $scriptInfo = Get-AnsibleScript -Name $scriptName -IncludeScriptBlock
+            # We don't use -IncludeScriptBlock as the script might be untrusted
+            # and will run under CLM. While we recreate the ScriptBlock here it
+            # is only to get the AST that contains the statements and their
+            # line numbers to create the breakpoint info for.
+            $scriptInfo = Get-AnsibleScript -Name $scriptName
 
             if (Compare-PathFilterPattern -Patterns $coveragePathFilter -Path $scriptInfo.Path) {
                 $covParams = @{
-                    ScriptBlock = $scriptInfo.ScriptBlock
+                    ScriptName = $scriptInfo.Name
+                    ScriptBlockAst = [ScriptBlock]::Create($scriptInfo.Script).Ast
                     AnsiblePath = $scriptInfo.Path
                 }
                 New-CoverageBreakpointsForScriptBlock @covParams