pybuilder-integration 108__tar.gz → 110__tar.gz

{pybuilder_integration-108 → pybuilder_integration-110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 108
+Version: 110
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 

{pybuilder_integration-108 → pybuilder_integration-110}/pybuilder_integration/directory_utility.py

@@ -29,8 +29,7 @@ def prepare_directory(dir_variable, project):
 
 
 def _ensure_directory_exists(path):
-    if not os.path.exists(path):
-        os.makedirs(path)
+    os.makedirs(path, exist_ok=True)
     return path
 
 

{pybuilder_integration-108 → pybuilder_integration-110}/pybuilder_integration/properties.py

@@ -20,5 +20,7 @@ CONSOLIDATE_TESTS = "consolidate_tavern"
 SKIP_NON_MATCHING_TAVERN_ARTIFACTS = "skip_non_matching_tavern_artifacts"
 TESTING_SCOPE = "testing_scope"
 PARALLEL_VERIFY_ENVIRONMENT = "parallel_verify_environment"
+PARALLEL_TAVERN_SUITES = "parallel_tavern_suites"
+PARALLEL_TAVERN_SUITES_WORKERS = "parallel_tavern_suites_workers"
 SUBPROCESS_PROPERTIES_FILE_ENV = "PYBUILDER_INTEGRATION_PROPERTIES_FILE"
 CYPRESS_CACHE_FOLDER = "cypress_cache_folder"

{pybuilder_integration-108 → pybuilder_integration-110}/pybuilder_integration/tasks.py

@@ -1,5 +1,6 @@
 import atexit
 import errno
+import hashlib
 import json
 import os
 import re
@@ -7,6 +8,7 @@ import shutil
 import subprocess
 import tempfile
 import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from defusedxml import ElementTree as ET
 from defusedxml.common import DefusedXmlException
@@ -48,11 +50,16 @@ _VERIFY_ENVIRONMENT_SUBPROCESS_PROPERTIES = [
     CONSOLIDATE_TESTS,
     SKIP_NON_MATCHING_TAVERN_ARTIFACTS,
     TESTING_SCOPE,
+    PARALLEL_TAVERN_SUITES,
+    PARALLEL_TAVERN_SUITES_WORKERS,
     "abort_upload",
     "record_cypress",
     "verbose",
 ]
 
+_TAVERN_INSTALL_LOCK = threading.Lock()
+_DEFAULT_PARALLEL_TAVERN_SUITE_WORKERS = 8
+
 def integration_artifact_push(project: Project, logger: Logger, reactor: Reactor):
     logger.info("Starting upload of integration artifacts")
     manager = get_artifact_manager(project)
@@ -886,15 +893,17 @@ def _run_tavern_tests_in_dist_dir(dist_directory, latest, logger, project, react
         logger.debug(f"Run tavern tests in directory: {tavern_test_path} ")
         logger.debug(f"Run tavern tests in directory files: {os.listdir(tavern_test_path)} ")
         if latest:
-            for tavern_dir in os.listdir(tavern_test_path):
-                tavern_test_direcotry = f"{tavern_test_path}/{tavern_dir}"
-                if os.path.isdir(tavern_test_direcotry) and _should_run_latest(tavern_dir, project):
-                    logger.info(f"Running {tavern_test_direcotry}")
-                    _run_tavern_tests_in_dir(test_dir=tavern_test_direcotry,
+            suite_dirs = list(_iter_latest_tavern_suite_dirs(tavern_test_path, project))
+            if _get_bool_property(project, PARALLEL_TAVERN_SUITES, False) and len(suite_dirs) > 1:
+                _run_tavern_suites_parallel(suite_dirs, logger, project, reactor)
+            else:
+                for tavern_test_directory in suite_dirs:
+                    logger.info(f"Running {tavern_test_directory}")
+                    _run_tavern_tests_in_dir(test_dir=tavern_test_directory,
                                              logger=logger,
                                              project=project,
                                              reactor=reactor,
-                                             role=os.path.basename(tavern_dir))
+                                             role=os.path.basename(tavern_test_directory))
         else:
             _run_tavern_tests_in_dir(test_dir=f"{tavern_test_path}",
                                      logger=logger,
@@ -904,6 +913,171 @@ def _run_tavern_tests_in_dist_dir(dist_directory, latest, logger, project, react
     logger.info(f"Ran Tavern tests: {total_time.get_millis()}")
 
 
+def _iter_latest_tavern_suite_dirs(tavern_test_path, project):
+    for tavern_dir in sorted(os.listdir(tavern_test_path)):
+        tavern_test_directory = os.path.join(tavern_test_path, tavern_dir)
+        if os.path.isdir(tavern_test_directory) and _should_run_latest(tavern_dir, project):
+            yield tavern_test_directory
+
+
+def _parallel_tavern_suite_workers(project, suite_count):
+    configured = project.get_property(PARALLEL_TAVERN_SUITES_WORKERS, None)
+    if configured is None or configured == "":
+        return max(1, min(_DEFAULT_PARALLEL_TAVERN_SUITE_WORKERS, suite_count))
+    try:
+        workers = int(configured)
+    except (TypeError, ValueError):
+        workers = _DEFAULT_PARALLEL_TAVERN_SUITE_WORKERS
+    return max(1, min(workers, suite_count))
+
+
+def _run_tavern_suites_parallel(suite_dirs, logger, project, reactor):
+    prepare_reports_directory(project)
+    runnable = []
+    for tavern_test_directory in suite_dirs:
+        _, requested_marker, declared_markers, skip_non_matching = _tavern_marker_context(
+            tavern_test_directory, project
+        )
+        if _should_skip_nonmatching_tavern(requested_marker, declared_markers, skip_non_matching):
+            logger.info(
+                f"Skipping Tavern tests: artifact markers {sorted(declared_markers)} do not match {requested_marker}"
+            )
+            continue
+        runnable.append(tavern_test_directory)
+    if not runnable:
+        return
+    if len(runnable) == 1:
+        tavern_test_directory = runnable[0]
+        logger.info(f"Running {tavern_test_directory}")
+        _run_tavern_tests_in_dir(
+            test_dir=tavern_test_directory,
+            logger=logger,
+            project=project,
+            reactor=reactor,
+            role=os.path.basename(tavern_test_directory),
+        )
+        return
+
+    workers = _parallel_tavern_suite_workers(project, len(runnable))
+    logger.info(f"Running {len(runnable)} tavern suites in parallel (workers={workers})")
+    fingerprint_by_dir = {suite_dir: _tavern_requirements_fingerprint(suite_dir) for suite_dir in runnable}
+    plugin_fingerprint = _plugin_venv_requirements_fingerprint(fingerprint_by_dir)
+    installer = _ParallelTavernRequirementInstaller(plugin_fingerprint)
+    if plugin_fingerprint is not None:
+        sample_dir = next(
+            suite_dir for suite_dir, fingerprint in fingerprint_by_dir.items()
+            if fingerprint == plugin_fingerprint
+        )
+        logger.info(
+            f"Installing tavern requirements from {os.path.basename(sample_dir)} "
+            f"into the plugin venv before parallel pytest"
+        )
+        installer.install(sample_dir, plugin_fingerprint, logger, project, reactor)
+    failures = []
+
+    def run_suite(tavern_test_directory):
+        role = os.path.basename(tavern_test_directory)
+        logger.info(f"Running {tavern_test_directory}")
+        site = installer.install(
+            tavern_test_directory,
+            fingerprint_by_dir[tavern_test_directory],
+            logger,
+            project,
+            reactor,
+        )
+        _run_tavern_tests_in_dir(
+            test_dir=tavern_test_directory,
+            logger=logger,
+            project=project,
+            reactor=reactor,
+            role=role,
+            isolated=True,
+            skip_install=True,
+            extra_pythonpath=site,
+        )
+        return role
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {executor.submit(run_suite, suite_dir): suite_dir for suite_dir in runnable}
+        for future in as_completed(futures):
+            suite_dir = futures[future]
+            role = os.path.basename(suite_dir)
+            try:
+                future.result()
+            except Exception as error:
+                failures.append(f"{role}: {error}")
+
+    if failures:
+        raise BuildFailedException(
+            "Parallel tavern suites failed: " + "; ".join(sorted(failures))
+        )
+
+
+class _ParallelTavernRequirementInstaller:
+    def __init__(self, plugin_venv_fingerprint):
+        self._plugin_venv_fingerprint = plugin_venv_fingerprint
+        self._lock = threading.Lock()
+        self._events = {}
+        self._results = {}
+        self._errors = {}
+
+    def install(self, test_dir, fingerprint, logger, project, reactor):
+        if fingerprint is None:
+            return None
+        with self._lock:
+            if fingerprint in self._results:
+                return self._results[fingerprint]
+            if fingerprint in self._errors:
+                raise self._errors[fingerprint]
+            event = self._events.get(fingerprint)
+            if event is None:
+                event = threading.Event()
+                self._events[fingerprint] = event
+                leader = True
+            else:
+                leader = False
+        if not leader:
+            event.wait()
+            with self._lock:
+                if fingerprint in self._errors:
+                    raise self._errors[fingerprint]
+                if fingerprint in self._results:
+                    return self._results[fingerprint]
+                raise BuildFailedException(
+                    f"Tavern requirements install for {os.path.basename(test_dir)} finished without a result"
+                )
+        try:
+            if fingerprint == self._plugin_venv_fingerprint:
+                _install_tavern_requirements(test_dir, logger, project, reactor)
+                site = None
+            else:
+                site = _install_tavern_suite_site(test_dir, logger, project, reactor, fingerprint)
+            with self._lock:
+                self._results[fingerprint] = site
+            return site
+        except Exception as error:
+            with self._lock:
+                self._errors[fingerprint] = error
+            raise
+        finally:
+            event.set()
+
+
+def _plugin_venv_requirements_fingerprint(fingerprint_by_dir):
+    counts = {}
+    order = []
+    for fingerprint in fingerprint_by_dir.values():
+        if fingerprint is None:
+            continue
+        if fingerprint not in counts:
+            order.append(fingerprint)
+            counts[fingerprint] = 0
+        counts[fingerprint] += 1
+    if not order:
+        return None
+    return max(order, key=lambda fingerprint: (counts[fingerprint], -order.index(fingerprint)))
+
+
 def verify_cypress(project: Project, logger: Logger, reactor: Reactor):
     # Get directories with test and cypress executable
     work_dir = project.expand_path(f"${CYPRESS_TEST_DIR}")
@@ -1087,7 +1261,117 @@ def verify_tavern(project: Project, logger: Logger, reactor: Reactor):
         package_artifacts(project, test_dir, "tavern", project.get_property(ROLE))
 
 
-def _run_tavern_tests_in_dir(test_dir: str, logger: Logger, project: Project, reactor: Reactor, role=None):
+def _tavern_marker_context(test_dir, project):
+    extra_args = [project.expand(prop) for prop in project.get_property(TAVERN_ADDITIONAL_ARGS, [])]
+    requested_marker = _extract_tavern_marker_filter(extra_args)
+    declared_markers = read_tavern_service_markers(test_dir)
+    skip_non_matching = _get_bool_property(project, SKIP_NON_MATCHING_TAVERN_ARTIFACTS, False)
+    return extra_args, requested_marker, declared_markers, skip_non_matching
+
+
+def _should_skip_nonmatching_tavern(requested_marker, declared_markers, skip_non_matching):
+    return bool(skip_non_matching and requested_marker and declared_markers and requested_marker not in declared_markers)
+
+
+def _install_tavern_requirements(test_dir, logger, project, reactor):
+    requirements_file = os.path.join(test_dir, "requirements.txt")
+    if not os.path.exists(requirements_file):
+        return
+    dependency = RequirementsFile(requirements_file)
+    with _TAVERN_INSTALL_LOCK:
+        install_dependencies(logger, project, dependency, reactor.pybuilder_venv,
+                             f"{prepare_logs_directory(project)}/install_tavern_pip_dependencies.log")
+
+
+def _plugin_venv_executable_path(executable):
+    if executable is None:
+        return None
+    if isinstance(executable, (list, tuple)):
+        if not executable:
+            return None
+        return _plugin_venv_executable_path(executable[0])
+    try:
+        path = os.fspath(executable)
+    except TypeError:
+        return None
+    if isinstance(path, bytes):
+        path = os.fsdecode(path)
+    return path
+
+
+def _plugin_venv_python(reactor):
+    venv = getattr(reactor, "pybuilder_venv", None)
+    path = _plugin_venv_executable_path(getattr(venv, "executable", None))
+    if path and os.path.isfile(path) and os.access(path, os.X_OK):
+        return path
+    raise BuildFailedException("plugin venv python is missing or not executable")
+
+
+def _tavern_requirements_fingerprint(test_dir):
+    requirements_file = os.path.join(test_dir, "requirements.txt")
+    if not os.path.isfile(requirements_file):
+        return None
+    with open(requirements_file, "rb") as requirements:
+        return hashlib.sha256(requirements.read()).hexdigest()
+
+
+def _tavern_suite_site_directory(project, fingerprint):
+    root = project.expand_path("$dir_target/integration/tavern-sites")
+    directory_utility._ensure_directory_exists(root)
+    site = os.path.join(root, fingerprint)
+    if os.path.islink(site) or os.path.isfile(site):
+        os.unlink(site)
+    elif os.path.isdir(site):
+        shutil.rmtree(site)
+    os.makedirs(site, mode=0o700)
+    return site
+
+
+def _tavern_pip_cache_directory(project, fingerprint):
+    cache = os.path.join(project.expand_path("$dir_target/integration/tavern-pip-cache"), fingerprint)
+    return directory_utility._ensure_directory_exists(cache)
+
+
+def _isolated_suite_subprocess_env(extra_pythonpath=None):
+    env = os.environ.copy()
+    env.pop("GITHUB_STEP_SUMMARY", None)
+    env.pop("PYTHONHOME", None)
+    env.pop("VIRTUAL_ENV", None)
+    env.pop("PYTHONPATH", None)
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    if extra_pythonpath:
+        env["PYTHONPATH"] = extra_pythonpath
+    return env
+
+
+def _install_tavern_suite_site(test_dir, logger, project, reactor, fingerprint):
+    requirements_file = os.path.join(test_dir, "requirements.txt")
+    if not os.path.isfile(requirements_file):
+        return None
+    role = os.path.basename(test_dir)
+    site = _tavern_suite_site_directory(project, fingerprint)
+    env = _isolated_suite_subprocess_env()
+    env["PIP_CACHE_DIR"] = _tavern_pip_cache_directory(project, fingerprint)
+    command = [_plugin_venv_python(reactor), "-m", "pip", "install", "-r", requirements_file, "-t", site]
+    logger.info(f"Installing tavern requirements from {role} into isolated site {site}")
+    process = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        cwd=test_dir,
+        env=env,
+        shell=False,
+    )
+    returncode = _stream_subprocess_output(process, logger, role)
+    if returncode != 0:
+        raise BuildFailedException(
+            f"Failed to install tavern requirements from {role} into isolated site {site}"
+        )
+    return site
+
+
+def _run_tavern_tests_in_dir(test_dir: str, logger: Logger, project: Project, reactor: Reactor, role=None,
+                             isolated=False, skip_install=False, extra_pythonpath=None):
     logger.info("Running tavern tests: {}".format(test_dir))
     if not os.path.exists(test_dir):
         logger.info("Skipping tavern run: no tests")
@@ -1095,51 +1379,51 @@ def _run_tavern_tests_in_dir(test_dir: str, logger: Logger, project: Project, re
     logger.info(f"Found {len(os.listdir(test_dir))} files in tavern test directory")
     # todo is this unique enough for each run?
     output_file, run_name = get_test_report_file(project, test_dir)
-    extra_args = [project.expand(prop) for prop in project.get_property(TAVERN_ADDITIONAL_ARGS, [])]
-    requested_marker = _extract_tavern_marker_filter(extra_args)
-    declared_markers = read_tavern_service_markers(test_dir)
-    skip_non_matching_artifacts = _get_bool_property(project, SKIP_NON_MATCHING_TAVERN_ARTIFACTS, False)
-    if skip_non_matching_artifacts and requested_marker and declared_markers and requested_marker not in declared_markers:
+    extra_args, requested_marker, declared_markers, skip_non_matching = _tavern_marker_context(test_dir, project)
+    if _should_skip_nonmatching_tavern(requested_marker, declared_markers, skip_non_matching):
         logger.info(
             f"Skipping Tavern tests: artifact markers {sorted(declared_markers)} do not match {requested_marker}"
         )
         return True
-    from sys import path as syspath
-    syspath.insert(0, test_dir)
-    # install any requirements that my exist
-    requirements_file = os.path.join(test_dir, "requirements.txt")
-    if os.path.exists(requirements_file):
-        dependency = RequirementsFile(requirements_file)
-        install_dependencies(logger, project, dependency, reactor.pybuilder_venv,
-                             f"{prepare_logs_directory(project)}/install_tavern_pip_dependencies.log")
+    if not isolated:
+        from sys import path as syspath
+        syspath.insert(0, test_dir)
+    if not skip_install:
+        _install_tavern_requirements(test_dir, logger, project, reactor)
     args = ["--junit-xml", f"{output_file}", test_dir] + extra_args
     if project.get_property("verbose"):
         args.append("-s")
         args.append("-v")
-    if _get_bool_property(project, RUN_PARALLEL, False):
+    if not isolated and _get_bool_property(project, RUN_PARALLEL, False):
         args.extend(['-n', 'auto'])
-    os.environ['TARGET'] = project.get_property(INTEGRATION_TARGET_URL)
-    os.environ['PUBLIC_TARGET'] = project.get_property(INTEGRATION_PUBLIC_TARGET_URL)
-    os.environ[ENVIRONMENT] = project.get_property(ENVIRONMENT)
-    logger.info(f"Running against target: {project.get_property(INTEGRATION_TARGET_URL)} and "
-                f"public target: {project.get_property(INTEGRATION_PUBLIC_TARGET_URL)}")
-    cache_wd = os.getcwd()
-    try:
-        os.chdir(test_dir)
-        logger.debug(f"Running args: {args} ")
-        ret = pytest.main(args)
-        if ret == pytest.ExitCode.NO_TESTS_COLLECTED:
-            fallback_args = _without_tavern_marker_filter(args)
-            if fallback_args != args and (not skip_non_matching_artifacts or not declared_markers):
-                logger.warn(
-                    "No Tavern tests matched the marker filter; rerunning the full suite"
-                )
-                logger.debug(f"Running fallback args: {fallback_args} ")
-                ret = pytest.main(fallback_args)
-    finally:
-        os.chdir(cache_wd)
+    target_url = project.get_property(INTEGRATION_TARGET_URL)
+    public_target_url = project.get_property(INTEGRATION_PUBLIC_TARGET_URL)
+    environment = project.get_property(ENVIRONMENT)
+    pytest_env = _isolated_suite_subprocess_env(extra_pythonpath) if isolated else os.environ.copy()
+    pytest_env['TARGET'] = target_url
+    pytest_env['PUBLIC_TARGET'] = public_target_url
+    pytest_env[ENVIRONMENT] = environment
+    if not isolated:
+        os.environ['TARGET'] = target_url
+        os.environ['PUBLIC_TARGET'] = public_target_url
+        os.environ[ENVIRONMENT] = environment
+    logger.info(f"Running against target: {target_url} and "
+                f"public target: {public_target_url}")
+    logger.debug(f"Running args: {args} ")
+    label = role or os.path.basename(test_dir)
+    ret = _invoke_pytest(args, test_dir, pytest_env, logger, label, isolated, reactor)
+    if _is_pytest_no_tests_collected(ret):
+        fallback_args = _without_tavern_marker_filter(args)
+        if fallback_args != args and (not skip_non_matching or not declared_markers):
+            logger.warn(
+                "No Tavern tests matched the marker filter; rerunning the full suite"
+            )
+            logger.debug(f"Running fallback args: {fallback_args} ")
+            ret = _invoke_pytest(
+                fallback_args, test_dir, pytest_env, logger, label, isolated, reactor
+            )
 
-    if ret != 0:
+    if _is_pytest_failure(ret):
         if role:
             roles = []
             if project.get_property(CONSOLIDATE_TESTS, False):
@@ -1156,6 +1440,54 @@ def _run_tavern_tests_in_dir(test_dir: str, logger: Logger, project: Project, re
     return True
 
 
+def _is_pytest_no_tests_collected(ret):
+    if ret is None:
+        return False
+    return int(ret) == int(pytest.ExitCode.NO_TESTS_COLLECTED)
+
+
+def _is_pytest_failure(ret):
+    if ret is None:
+        return True
+    return int(ret) != 0
+
+
+def _invoke_pytest(args, test_dir, env, logger, label, isolated, reactor=None):
+    if not isolated:
+        cache_wd = os.getcwd()
+        try:
+            os.chdir(test_dir)
+            return pytest.main(args)
+        finally:
+            os.chdir(cache_wd)
+    command = [_plugin_venv_python(reactor), "-u", "-m", "pytest"] + args
+    process = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        cwd=test_dir,
+        env=env,
+        shell=False,
+    )
+    return _stream_subprocess_output(process, logger, label)
+
+
+def _stream_subprocess_output(process, logger, label):
+    stdout = process.stdout
+    if stdout is None:
+        return process.wait()
+    try:
+        while True:
+            chunk = stdout.readline()
+            if not chunk:
+                break
+            redacted = _redact_secret_log_line(_decode_subprocess_stdout_line(chunk))
+            logger.info(f"[{label}] {redacted}")
+    finally:
+        stdout.close()
+    return process.wait()
+
+
 def _without_tavern_marker_filter(args):
     marker_options = {"-m", "--markexpr"}
     fallback_args = []

{pybuilder_integration-108 → pybuilder_integration-110}/pybuilder_integration.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 108
+Version: 110
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 

{pybuilder_integration-108 → pybuilder_integration-110}/setup.py

@@ -21,7 +21,7 @@ class install(_install):
 if __name__ == '__main__':
     setup(
         name = 'pybuilder-integration',
-        version = '108',
+        version = '110',
         description = 'A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.',
         long_description = 'A pybuilder plugin that runs integration tests against a target.  This is intended to be a broader scope than unit-tests encompassing dependant functionality.',
         long_description_content_type = None,