pybuilder-integration 105__tar.gz → 107__tar.gz

{pybuilder_integration-105 → pybuilder_integration-107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 105
+Version: 107
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 
@@ -10,9 +10,10 @@ Maintainer-email:
 License: Apache 2.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
-Requires-Dist: pytest
+Requires-Dist: pytest==7.4.4
 Requires-Dist: tavern==1.25.2
-Requires-Dist: boto3
+Requires-Dist: boto3==1.43.72
+Requires-Dist: defusedxml==0.7.1
 Dynamic: classifier
 Dynamic: description
 Dynamic: home-page

{pybuilder_integration-105 → pybuilder_integration-107}/pybuilder_integration/tasks.py

@@ -3,9 +3,10 @@ import os
 import re
 import shutil
 import subprocess
-import sys
 import tempfile
-import xml.etree.ElementTree as ET
+
+from defusedxml import ElementTree as ET
+from defusedxml.common import DefusedXmlException
 
 import pytest
 from pybuilder.core import Project, Logger, RequirementsFile
@@ -27,6 +28,7 @@ from pybuilder_integration.tool_utility import install_cypress
 _VERIFY_ENVIRONMENT_SUBPROCESS_PROPERTIES = [
     INTEGRATION_ARTIFACT_BUCKET,
     ENVIRONMENT,
+    ENVIRONMENT_VARIABLES,
     ARTIFACT_MANAGER,
     CYPRESS_TEST_DIR,
     CYPRESS_CONFIG_FILE,
@@ -134,16 +136,217 @@ def _run_verify_environment_passes_in_subprocesses(project: Project, logger: Log
         if failures:
             raise BuildFailedException("Parallel verify_environment failed: " + "; ".join(failures))
     finally:
+        for _, process, _ in processes:
+            poll = getattr(process, "poll", None)
+            if callable(poll) and poll() is None:
+                process.terminate()
+                process.wait()
+            log_handle = getattr(process, "_pybuilder_integration_log_handle", None)
+            if log_handle and not log_handle.closed:
+                log_handle.close()
+        _merge_parallel_verify_environment_outputs(project, passes)
         if os.path.exists(properties_file):
             os.remove(properties_file)
 
 
-def _start_verify_environment_subprocess(project: Project, logger: Logger, pass_config, properties_file):
-    log_file = os.path.join(prepare_logs_directory(project), f"verify_environment_{pass_config['name']}.log")
-    command = _build_verify_environment_subprocess_command(project, pass_config)
+_CHILD_OUTPUT_SOURCES = (
+    ("logs", "logs"),
+    ("reports", "reports"),
+    ("reports", "test-reports"),
+)
+_SUBPROCESS_WORKDIR_SKIP = {".pybuilder", "target", ".git", ".nudge-ci-venv"}
+
+
+def _merge_parallel_verify_environment_outputs(project: Project, passes):
+    parent_target = project.expand_path("$dir_target")
+    parent_logs_directory = prepare_logs_directory(project)
+    parent_destinations = {
+        "logs": parent_logs_directory,
+        "reports": prepare_reports_directory(project),
+    }
+    parent_cypress_log = os.path.join(parent_logs_directory, "cypress_run.log")
+    has_parent_cypress_log = False
+    for pass_config in passes:
+        stage_target = os.path.join(parent_target, "verify_environment", pass_config["name"])
+        for destination_kind, source_dirname in _CHILD_OUTPUT_SOURCES:
+            source = os.path.join(stage_target, source_dirname, "integration")
+            if not os.path.exists(source):
+                continue
+            destination = os.path.join(
+                parent_destinations[destination_kind],
+                "parallel",
+                pass_config["name"],
+            )
+            shutil.copytree(source, destination, dirs_exist_ok=True)
+            if destination_kind == "logs":
+                cypress_log = os.path.join(source, "cypress_run.log")
+                if os.path.exists(cypress_log):
+                    _copy_cypress_log(cypress_log, parent_cypress_log, has_parent_cypress_log)
+                    has_parent_cypress_log = True
+
+
+def _copy_cypress_log(source, destination, append):
+    flags = os.O_WRONLY | os.O_CREAT | (os.O_APPEND if append else os.O_TRUNC)
+    mode = "ab" if append else "wb"
+    with open(source, "rb") as source_file:
+        with os.fdopen(os.open(destination, flags, 0o600), mode) as destination_file:
+            shutil.copyfileobj(source_file, destination_file)
+
+
+def _verify_environment_subprocess_stage_dir(project: Project, pass_config):
+    return os.path.join(
+        project.expand_path("$dir_target"),
+        "verify_environment",
+        pass_config["name"],
+    )
+
+
+def _verify_environment_subprocess_workdir_path(project: Project, pass_config):
+    return os.path.join(_verify_environment_subprocess_stage_dir(project, pass_config), "project")
+
+
+def _prepare_verify_environment_subprocess_workdir(project: Project, pass_config):
+    stage_dir = _verify_environment_subprocess_stage_dir(project, pass_config)
+    workdir = _verify_environment_subprocess_workdir_path(project, pass_config)
+    if os.path.exists(stage_dir):
+        shutil.rmtree(stage_dir)
+    os.makedirs(workdir)
+    for name in os.listdir(project.basedir):
+        if name in _SUBPROCESS_WORKDIR_SKIP:
+            continue
+        os.symlink(os.path.join(project.basedir, name), os.path.join(workdir, name))
+    _link_parent_target_tree(project, workdir)
+    source_plugins = os.path.join(project.basedir, ".pybuilder")
+    if os.path.isdir(source_plugins):
+        destination_plugins = os.path.join(workdir, ".pybuilder")
+        shutil.copytree(source_plugins, destination_plugins, symlinks=True)
+        _materialize_external_venv_pythons(os.path.join(destination_plugins, "plugins"))
+    return workdir
+
+
+def _link_parent_target_tree(project: Project, workdir):
+    parent_target = project.expand_path("$dir_target")
+    child_target = os.path.join(workdir, "target")
+    os.makedirs(child_target, exist_ok=True)
+    if not os.path.isdir(parent_target):
+        return
+    for name in os.listdir(parent_target):
+        if name == "verify_environment":
+            continue
+        os.symlink(os.path.join(parent_target, name), os.path.join(child_target, name))
+
+
+def _is_venv_python_launcher(name):
+    if name in {"python", "python3"}:
+        return True
+    if name.startswith("python3."):
+        return name[len("python"):].replace(".", "").isdigit()
+    return False
+
+
+def _is_real_directory(path):
+    return os.path.isdir(path) and not os.path.islink(path)
+
+
+def _is_path_within(path, root):
+    root_real = os.path.realpath(root)
+    try:
+        return os.path.commonpath([root_real, os.path.realpath(path)]) == root_real
+    except ValueError:
+        return False
+
+
+def _materialize_external_venv_pythons(plugins_root):
+    if not _is_real_directory(plugins_root):
+        return
+    plugins_root_real = os.path.realpath(plugins_root)
+    for version_dir in os.listdir(plugins_root):
+        version_root = os.path.join(plugins_root, version_dir)
+        bin_dir = os.path.join(version_root, "bin")
+        if not _is_real_directory(version_root) or not _is_real_directory(bin_dir):
+            continue
+        bin_dir_real = os.path.realpath(bin_dir)
+        if not _is_path_within(bin_dir_real, plugins_root_real):
+            continue
+        version_root_real = os.path.realpath(version_root)
+        for name in os.listdir(bin_dir_real):
+            if not _is_venv_python_launcher(name):
+                continue
+            path = os.path.join(bin_dir_real, name)
+            if not os.path.islink(path):
+                continue
+            target = os.path.realpath(path)
+            if not os.path.isfile(target):
+                continue
+            if _is_path_within(target, version_root_real):
+                continue
+            # mkstemp uses O_EXCL so a planted *.materialize symlink cannot
+            # redirect the write outside the staged workdir.
+            fd, temporary_copy = tempfile.mkstemp(
+                prefix=f".{name}.",
+                suffix=".materialize",
+                dir=bin_dir_real,
+            )
+            try:
+                with os.fdopen(fd, "wb") as destination, open(target, "rb") as source:
+                    shutil.copyfileobj(source, destination)
+                    destination.flush()
+                    os.fchmod(destination.fileno(), 0o700)
+                os.replace(temporary_copy, path)
+            except Exception:
+                try:
+                    os.unlink(temporary_copy)
+                except OSError:
+                    pass
+                raise
+
+
+def _plugin_python_executable(plugins_root):
+    if not os.path.isdir(plugins_root):
+        return None
+    for version_dir in sorted(os.listdir(plugins_root)):
+        candidate = os.path.join(plugins_root, version_dir, "bin", "python")
+        if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+            return candidate
+    return None
+
+
+_PYBUILDER_CLI_MAIN = "import sys; from pybuilder.cli import main; raise SystemExit(main(*sys.argv[1:]))"
+
+
+def _isolated_plugin_python_command(plugin_python, args):
+    pyb_script = os.path.join(os.path.dirname(plugin_python), "pyb")
+    if os.path.isfile(pyb_script):
+        return [plugin_python, pyb_script] + args
+    return [plugin_python, "-c", _PYBUILDER_CLI_MAIN] + args
+
+
+def _uses_isolated_plugin_python(command):
+    if len(command) < 2:
+        return False
+    if command[1] == "-c":
+        return len(command) >= 3 and command[2] == _PYBUILDER_CLI_MAIN
+    return os.path.basename(command[1]) == "pyb"
+
+
+def _verify_environment_subprocess_env(workdir, command, properties_file):
     env = os.environ.copy()
     env[SUBPROCESS_PROPERTIES_FILE_ENV] = properties_file
-    logger.info(f"Starting {pass_config['name']} verify_environment subprocess: {' '.join(command)}")
+    env.pop("GITHUB_STEP_SUMMARY", None)
+    env.pop("PYTHONHOME", None)
+    env.pop("VIRTUAL_ENV", None)
+    env.pop("PYTHONPATH", None)
+    return env
+
+
+def _start_verify_environment_subprocess(project: Project, logger: Logger, pass_config, properties_file):
+    log_file = os.path.join(prepare_logs_directory(project), f"verify_environment_{pass_config['name']}.log")
+    workdir = _prepare_verify_environment_subprocess_workdir(project, pass_config)
+    command = _build_verify_environment_subprocess_command(project, pass_config, workdir)
+    env = _verify_environment_subprocess_env(workdir, command, properties_file)
+    logger.info(
+        f"Starting {pass_config['name']} verify_environment subprocess in {workdir}: {' '.join(command)}"
+    )
     log_handle = os.fdopen(
         os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
         "w",
@@ -153,7 +356,7 @@ def _start_verify_environment_subprocess(project: Project, logger: Logger, pass_
         process = subprocess.Popen(command,
                                    stdout=log_handle,
                                    stderr=subprocess.STDOUT,
-                                   cwd=project.basedir,
+                                   cwd=workdir,
                                    env=env,
                                    shell=False)
     except Exception:
@@ -163,20 +366,29 @@ def _start_verify_environment_subprocess(project: Project, logger: Logger, pass_
     return process, log_file
 
 
-def _build_verify_environment_subprocess_command(project: Project, pass_config):
-    stage_target = os.path.join(project.expand_path("$dir_target"), "verify_environment", pass_config["name"])
-    return [
-        sys.executable,
-        "-c",
-        "from pybuilder.cli import main; main()",
+def _build_verify_environment_subprocess_command(project: Project, pass_config, workdir=None):
+    stage_target = _verify_environment_subprocess_stage_dir(project, pass_config)
+    args = [
+        "--no-venvs",
         pass_config["task"],
         "-P", f"{WORKING_TEST_DIR}={pass_config['test_dir']}",
         "-P", f"{PARALLEL_VERIFY_ENVIRONMENT}=False",
         "-P", f"{PROMOTE_ARTIFACT}=False",
         "-P", f"dir_target={stage_target}",
         "-P", f"dir_logs={os.path.join(stage_target, 'logs')}",
-        "-P", f"dir_reports={os.path.join(stage_target, 'reports')}",
+        "-P", f"dir_reports={os.path.join(stage_target, 'test-reports')}",
     ]
+    if not workdir:
+        raise BuildFailedException(
+            f"Cannot start isolated verify_environment subprocess for {pass_config['name']}: workdir is required"
+        )
+    plugin_python = _plugin_python_executable(os.path.join(workdir, ".pybuilder", "plugins"))
+    if not plugin_python:
+        raise BuildFailedException(
+            f"Cannot start isolated verify_environment subprocess for {pass_config['name']}: "
+            "copied plugin python is missing or not executable"
+        )
+    return _isolated_plugin_python_command(plugin_python, args)
 
 
 def _write_subprocess_properties_file(project: Project):
@@ -191,13 +403,12 @@ def _collect_json_safe_project_properties(project: Project):
     safe_properties = {}
     for name in _VERIFY_ENVIRONMENT_SUBPROCESS_PROPERTIES:
         value = project.get_property(name, None)
-        if value is None:
-            continue
-        try:
-            json.dumps(value)
-        except (TypeError, ValueError):
-            continue
-        safe_properties[name] = value
+        if value is not None:
+            try:
+                json.dumps(value)
+            except (TypeError, ValueError):
+                continue
+            safe_properties[name] = value
     return safe_properties
 
 
@@ -376,7 +587,7 @@ def _build_cypress_summary(log_path):
                     name = tc.get("name", "")
                     msg = (failure.get("message") or failure.text or "").split("\n")[0][:120]
                     failures.append((f"{suite} > {name}", msg))
-        except ET.ParseError:
+        except (ET.ParseError, DefusedXmlException):
             pass
 
     passed = total - failed - skipped

{pybuilder_integration-105 → pybuilder_integration-107}/pybuilder_integration.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 105
+Version: 107
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 
@@ -10,9 +10,10 @@ Maintainer-email:
 License: Apache 2.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
-Requires-Dist: pytest
+Requires-Dist: pytest==7.4.4
 Requires-Dist: tavern==1.25.2
-Requires-Dist: boto3
+Requires-Dist: boto3==1.43.72
+Requires-Dist: defusedxml==0.7.1
 Dynamic: classifier
 Dynamic: description
 Dynamic: home-page

pybuilder_integration-107/pybuilder_integration.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pytest==7.4.4
+tavern==1.25.2
+boto3==1.43.72
+defusedxml==0.7.1

{pybuilder_integration-105 → pybuilder_integration-107}/setup.py

@@ -21,7 +21,7 @@ class install(_install):
 if __name__ == '__main__':
     setup(
         name = 'pybuilder-integration',
-        version = '105',
+        version = '107',
         description = 'A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.',
         long_description = 'A pybuilder plugin that runs integration tests against a target.  This is intended to be a broader scope than unit-tests encompassing dependant functionality.',
         long_description_content_type = None,
@@ -49,9 +49,10 @@ if __name__ == '__main__':
         data_files = [],
         package_data = {},
         install_requires = [
-            'pytest',
+            'pytest==7.4.4',
             'tavern==1.25.2',
-            'boto3'
+            'boto3==1.43.72',
+            'defusedxml==0.7.1'
         ],
         extras_require = {},
         dependency_links = [],

pybuilder_integration-105/pybuilder_integration.egg-info/requires.txt

@@ -1,3 +0,0 @@
-pytest
-tavern==1.25.2
-boto3