pybuilder-integration 109__tar.gz → 110__tar.gz

{pybuilder_integration-109 → pybuilder_integration-110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 109
+Version: 110
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 

{pybuilder_integration-109 → pybuilder_integration-110}/pybuilder_integration/tasks.py

@@ -1,5 +1,6 @@
 import atexit
 import errno
+import hashlib
 import json
 import os
 import re
@@ -957,19 +958,33 @@ def _run_tavern_suites_parallel(suite_dirs, logger, project, reactor):
         )
         return
 
-    suite_sites = {}
-    for tavern_test_directory in runnable:
-        suite_sites[tavern_test_directory] = _install_tavern_suite_site(
-            tavern_test_directory, logger, project, reactor
-        )
-
     workers = _parallel_tavern_suite_workers(project, len(runnable))
     logger.info(f"Running {len(runnable)} tavern suites in parallel (workers={workers})")
+    fingerprint_by_dir = {suite_dir: _tavern_requirements_fingerprint(suite_dir) for suite_dir in runnable}
+    plugin_fingerprint = _plugin_venv_requirements_fingerprint(fingerprint_by_dir)
+    installer = _ParallelTavernRequirementInstaller(plugin_fingerprint)
+    if plugin_fingerprint is not None:
+        sample_dir = next(
+            suite_dir for suite_dir, fingerprint in fingerprint_by_dir.items()
+            if fingerprint == plugin_fingerprint
+        )
+        logger.info(
+            f"Installing tavern requirements from {os.path.basename(sample_dir)} "
+            f"into the plugin venv before parallel pytest"
+        )
+        installer.install(sample_dir, plugin_fingerprint, logger, project, reactor)
     failures = []
 
     def run_suite(tavern_test_directory):
         role = os.path.basename(tavern_test_directory)
         logger.info(f"Running {tavern_test_directory}")
+        site = installer.install(
+            tavern_test_directory,
+            fingerprint_by_dir[tavern_test_directory],
+            logger,
+            project,
+            reactor,
+        )
         _run_tavern_tests_in_dir(
             test_dir=tavern_test_directory,
             logger=logger,
@@ -978,7 +993,7 @@ def _run_tavern_suites_parallel(suite_dirs, logger, project, reactor):
             role=role,
             isolated=True,
             skip_install=True,
-            extra_pythonpath=suite_sites[tavern_test_directory],
+            extra_pythonpath=site,
         )
         return role
 
@@ -998,6 +1013,71 @@ def _run_tavern_suites_parallel(suite_dirs, logger, project, reactor):
         )
 
 
+class _ParallelTavernRequirementInstaller:
+    def __init__(self, plugin_venv_fingerprint):
+        self._plugin_venv_fingerprint = plugin_venv_fingerprint
+        self._lock = threading.Lock()
+        self._events = {}
+        self._results = {}
+        self._errors = {}
+
+    def install(self, test_dir, fingerprint, logger, project, reactor):
+        if fingerprint is None:
+            return None
+        with self._lock:
+            if fingerprint in self._results:
+                return self._results[fingerprint]
+            if fingerprint in self._errors:
+                raise self._errors[fingerprint]
+            event = self._events.get(fingerprint)
+            if event is None:
+                event = threading.Event()
+                self._events[fingerprint] = event
+                leader = True
+            else:
+                leader = False
+        if not leader:
+            event.wait()
+            with self._lock:
+                if fingerprint in self._errors:
+                    raise self._errors[fingerprint]
+                if fingerprint in self._results:
+                    return self._results[fingerprint]
+                raise BuildFailedException(
+                    f"Tavern requirements install for {os.path.basename(test_dir)} finished without a result"
+                )
+        try:
+            if fingerprint == self._plugin_venv_fingerprint:
+                _install_tavern_requirements(test_dir, logger, project, reactor)
+                site = None
+            else:
+                site = _install_tavern_suite_site(test_dir, logger, project, reactor, fingerprint)
+            with self._lock:
+                self._results[fingerprint] = site
+            return site
+        except Exception as error:
+            with self._lock:
+                self._errors[fingerprint] = error
+            raise
+        finally:
+            event.set()
+
+
+def _plugin_venv_requirements_fingerprint(fingerprint_by_dir):
+    counts = {}
+    order = []
+    for fingerprint in fingerprint_by_dir.values():
+        if fingerprint is None:
+            continue
+        if fingerprint not in counts:
+            order.append(fingerprint)
+            counts[fingerprint] = 0
+        counts[fingerprint] += 1
+    if not order:
+        return None
+    return max(order, key=lambda fingerprint: (counts[fingerprint], -order.index(fingerprint)))
+
+
 def verify_cypress(project: Project, logger: Logger, reactor: Reactor):
     # Get directories with test and cypress executable
     work_dir = project.expand_path(f"${CYPRESS_TEST_DIR}")
@@ -1227,10 +1307,18 @@ def _plugin_venv_python(reactor):
     raise BuildFailedException("plugin venv python is missing or not executable")
 
 
-def _tavern_suite_site_directory(project, role):
+def _tavern_requirements_fingerprint(test_dir):
+    requirements_file = os.path.join(test_dir, "requirements.txt")
+    if not os.path.isfile(requirements_file):
+        return None
+    with open(requirements_file, "rb") as requirements:
+        return hashlib.sha256(requirements.read()).hexdigest()
+
+
+def _tavern_suite_site_directory(project, fingerprint):
     root = project.expand_path("$dir_target/integration/tavern-sites")
     directory_utility._ensure_directory_exists(root)
-    site = os.path.join(root, role)
+    site = os.path.join(root, fingerprint)
     if os.path.islink(site) or os.path.isfile(site):
         os.unlink(site)
     elif os.path.isdir(site):
@@ -1239,37 +1327,45 @@ def _tavern_suite_site_directory(project, role):
     return site
 
 
+def _tavern_pip_cache_directory(project, fingerprint):
+    cache = os.path.join(project.expand_path("$dir_target/integration/tavern-pip-cache"), fingerprint)
+    return directory_utility._ensure_directory_exists(cache)
+
+
 def _isolated_suite_subprocess_env(extra_pythonpath=None):
     env = os.environ.copy()
     env.pop("GITHUB_STEP_SUMMARY", None)
     env.pop("PYTHONHOME", None)
     env.pop("VIRTUAL_ENV", None)
     env.pop("PYTHONPATH", None)
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
     if extra_pythonpath:
         env["PYTHONPATH"] = extra_pythonpath
     return env
 
 
-def _install_tavern_suite_site(test_dir, logger, project, reactor):
+def _install_tavern_suite_site(test_dir, logger, project, reactor, fingerprint):
     requirements_file = os.path.join(test_dir, "requirements.txt")
-    if not os.path.exists(requirements_file):
+    if not os.path.isfile(requirements_file):
         return None
     role = os.path.basename(test_dir)
-    site = _tavern_suite_site_directory(project, role)
+    site = _tavern_suite_site_directory(project, fingerprint)
+    env = _isolated_suite_subprocess_env()
+    env["PIP_CACHE_DIR"] = _tavern_pip_cache_directory(project, fingerprint)
     command = [_plugin_venv_python(reactor), "-m", "pip", "install", "-r", requirements_file, "-t", site]
-    logger.info(f"Installing tavern requirements for {role} into {site}")
+    logger.info(f"Installing tavern requirements from {role} into isolated site {site}")
     process = subprocess.Popen(
         command,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         cwd=test_dir,
-        env=_isolated_suite_subprocess_env(),
+        env=env,
         shell=False,
     )
     returncode = _stream_subprocess_output(process, logger, role)
     if returncode != 0:
         raise BuildFailedException(
-            f"Failed to install tavern requirements for {role} into isolated site {site}"
+            f"Failed to install tavern requirements from {role} into isolated site {site}"
         )
     return site
 

{pybuilder_integration-109 → pybuilder_integration-110}/pybuilder_integration.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-integration
-Version: 109
+Version: 110
 Summary: A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.
 Home-page: https://github.com/rspitler/pybuilder-integration
 Author: 

{pybuilder_integration-109 → pybuilder_integration-110}/setup.py

@@ -21,7 +21,7 @@ class install(_install):
 if __name__ == '__main__':
     setup(
         name = 'pybuilder-integration',
-        version = '109',
+        version = '110',
         description = 'A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.',
         long_description = 'A pybuilder plugin that runs integration tests against a target.  This is intended to be a broader scope than unit-tests encompassing dependant functionality.',
         long_description_content_type = None,