toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/test/__init__.py

@@ -28,8 +28,10 @@ import uuid
 import zoneinfo
 from abc import ABCMeta, abstractmethod
 from collections.abc import Generator
-from contextlib import contextmanager
+from contextlib import AbstractContextManager, contextmanager
+from importlib.resources import as_file, files
 from inspect import getsource
+from pathlib import Path
 from shutil import which
 from tempfile import mkstemp
 from textwrap import dedent
@@ -38,7 +40,10 @@ from unittest.util import strclass
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen
 
-from toil import ApplianceImageNotFound, applianceSelf, toilPackageDirPath
+import pytest
+from typing_extensions import Self
+
+from toil import ApplianceImageNotFound, applianceSelf
 from toil.lib.accelerators import (
     have_working_nvidia_docker_runtime,
     have_working_nvidia_smi,
@@ -49,14 +54,41 @@ from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
 from toil.version import distVersion
 
+try:
+    from botocore.exceptions import ProxyConnectionError
+except ImportError:
+
+    class ProxyConnectionError(BaseException):  # type: ignore[no-redef]
+        ...
+
 logger = logging.getLogger(__name__)
 
 
+@contextmanager
+def _fallback_get_data(filename: str) -> Generator[Path]:
+    try:
+        yield Path(os.path.dirname(__file__)) / ".." / ".." / ".." / filename
+    finally:
+        pass  # no cleanup needed
+
+
+def get_data(filename: str) -> AbstractContextManager[Path]:
+    """Returns an absolute path for a file from this package."""
+    # normalizing path depending on OS or else it will cause problem when joining path
+    filename = os.path.normpath(filename)
+    try:
+        return as_file(files("toil") / filename)
+    except ModuleNotFoundError:
+        pass
+    return _fallback_get_data(filename)
+
+
+@pytest.mark.usefixtures("rootpath")
 class ToilTest(unittest.TestCase):
     """
-    A common base class for Toil tests.
+    Legacy common base class for Toil tests.
 
-    Please have every test case directly or indirectly inherit this one.
+    New tests should be made in the 'pytest' style and not use this class.
 
     When running tests you may optionally set the TOIL_TEST_TEMP environment variable
     to the path of a directory where you want temporary test files be placed. The
@@ -68,6 +100,7 @@ class ToilTest(unittest.TestCase):
     Otherwise, left-over files will not be removed.
     """
 
+    _rootpath: Path
     _tempBaseDir: Optional[str] = None
     _tempDirs: list[str] = []
 
@@ -84,9 +117,7 @@ class ToilTest(unittest.TestCase):
         super().setUpClass()
         tempBaseDir = os.environ.get("TOIL_TEST_TEMP", None)
         if tempBaseDir is not None and not os.path.isabs(tempBaseDir):
-            tempBaseDir = os.path.abspath(
-                os.path.join(cls._projectRootPath(), tempBaseDir)
-            )
+            tempBaseDir = os.path.abspath(cls._rootpath / tempBaseDir)
             os.makedirs(tempBaseDir, exist_ok=True)
         cls._tempBaseDir = tempBaseDir
 
@@ -149,28 +180,6 @@ class ToilTest(unittest.TestCase):
         assert region
         return region.group(1)
 
-    @classmethod
-    def _getUtilScriptPath(cls, script_name: str) -> str:
-        return os.path.join(toilPackageDirPath(), "utils", script_name + ".py")
-
-    @classmethod
-    def _projectRootPath(cls) -> str:
-        """
-        Return the path to the project root.
-
-        i.e. the directory that typically contains the .git and src subdirectories.
-        This method has limited utility. It only works if in "develop"
-        mode, since it assumes the existence of a src subdirectory which, in a regular install
-        wouldn't exist. Then again, in that mode project root has no meaning anyways.
-        """
-        assert re.search(r"__init__\.pyc?$", __file__)
-        projectRootPath = os.path.dirname(os.path.abspath(__file__))
-        packageComponents = __name__.split(".")
-        expectedSuffix = os.path.join("src", *packageComponents)
-        assert projectRootPath.endswith(expectedSuffix)
-        projectRootPath = projectRootPath[: -len(expectedSuffix)]
-        return projectRootPath
-
     def _createTempDir(self, purpose: Optional[str] = None) -> str:
         return self._createTempDirEx(self._testMethodName, purpose)
 
@@ -198,57 +207,6 @@ class ToilTest(unittest.TestCase):
         os.rmdir(path)
         return path
 
-    @classmethod
-    def _getSourceDistribution(cls) -> str:
-        """
-        Find the sdist tarball for this project and return the path to it.
-
-        Also assert that the sdist is up-to date
-        """
-        sdistPath = os.path.join(
-            cls._projectRootPath(), "dist", "toil-%s.tar.gz" % distVersion
-        )
-        assert os.path.isfile(sdistPath), (
-            "Can't find Toil source distribution at %s. Run 'make sdist'." % sdistPath
-        )
-        excluded = set(
-            cast(
-                str,
-                cls._run(
-                    "git",
-                    "ls-files",
-                    "--others",
-                    "-i",
-                    "--exclude-standard",
-                    capture=True,
-                    cwd=cls._projectRootPath(),
-                ),
-            ).splitlines()
-        )
-        dirty = cast(
-            str,
-            cls._run(
-                "find",
-                "src",
-                "-type",
-                "f",
-                "-newer",
-                sdistPath,
-                capture=True,
-                cwd=cls._projectRootPath(),
-            ),
-        ).splitlines()
-        assert all(path.startswith("src") for path in dirty)
-        dirty_set = set(dirty)
-        dirty_set.difference_update(excluded)
-        assert (
-            not dirty_set
-        ), "Run 'make clean_sdist sdist'. Files newer than {}: {!r}".format(
-            sdistPath,
-            list(dirty_set),
-        )
-        return sdistPath
-
     @classmethod
     def _run(cls, command: str, *args: str, **kwargs: Any) -> Optional[str]:
         """
@@ -362,6 +320,27 @@ def needs_rsync3(test_item: MT) -> MT:
     return test_item
 
 
+def _has_rsync3() -> bool:
+    try:
+        versionInfo = subprocess.check_output(["rsync", "--version"]).decode("utf-8")
+        if int(versionInfo.split()[2].split(".")[0]) < 3:
+            return False
+    except subprocess.CalledProcessError:
+        return False
+    except ValueError:
+        # Don't have an int where we looked
+        return False
+    except IndexError:
+        # Don't have the field to look in
+        return False
+    return True
+
+
+pneeds_rsync3 = pytest.mark.skipif(
+    not _has_rsync3(), reason="This test depends on rsync version 3.0.0+."
+)
+
+
 def needs_online(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if we are meant to talk to the Internet."""
     test_item = _mark_test("online", test_item)
@@ -370,10 +349,17 @@ def needs_online(test_item: MT) -> MT:
     return test_item
 
 
+def _skip_online() -> bool:
+    return os.getenv("TOIL_SKIP_ONLINE", "").lower() == "true"
+
+
+pneeds_online = pytest.mark.skipif(_skip_online(), reason="Skipping online test.")
+
+
 def needs_aws_s3(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if AWS S3 is usable."""
     # TODO: we just check for generic access to the AWS account
-    test_item = _mark_test("aws-s3", needs_online(test_item))
+    test_item = _mark_test("aws_s3", needs_online(test_item))
     try:
         from boto3 import Session
 
@@ -383,6 +369,10 @@ def needs_aws_s3(test_item: MT) -> MT:
         return unittest.skip("Install Toil with the 'aws' extra to include this test.")(
             test_item
         )
+    except ProxyConnectionError as e:
+        return unittest.skip(f"Proxy error: {e}, skipping this test.")(
+            test_item
+        )
     from toil.lib.aws import running_on_ec2
 
     if not (
@@ -396,23 +386,55 @@ def needs_aws_s3(test_item: MT) -> MT:
     return test_item
 
 
+def _aws_s3_avail() -> bool:
+    """Use as a decorator before test classes or methods to run only if AWS S3 is usable."""
+    try:
+        from boto3 import Session
+
+        session = Session()
+        boto3_credentials = session.get_credentials()
+    except ImportError:
+        return False
+    from toil.lib.aws import running_on_ec2
+
+    if not (
+        boto3_credentials
+        or os.path.exists(os.path.expanduser("~/.aws/credentials"))
+        or running_on_ec2()
+    ):
+        return False
+    return True
+
+
+pneeds_aws_s3 = pytest.mark.skipif(
+    _skip_online() or not _aws_s3_avail(),
+    reason="Install Toil with the 'aws' extra or configure AWS credentials to include this test.",
+)
+
+
 def needs_aws_ec2(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if AWS EC2 is usable."""
     # Assume we need S3 as well as EC2
-    test_item = _mark_test("aws-ec2", needs_aws_s3(test_item))
+    test_item = _mark_test("aws_ec2", needs_aws_s3(test_item))
     # In addition to S3 we also need an SSH key to deploy with.
     # TODO: We assume that if this is set we have EC2 access.
     test_item = needs_env_var("TOIL_AWS_KEYNAME", "an AWS-stored SSH key")(test_item)
     return test_item
 
 
+pneeds_aws_ec2 = pytest.mark.skipif(
+    _skip_online() or not _aws_s3_avail() or not os.getenv("TOIL_AWS_KEYNAME"),
+    reason="Set 'TOIL_AWS_KEYNAME' to an AWS-stored SSH key to run this test",
+)
+
+
 def needs_aws_batch(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if AWS Batch
     is usable.
     """
     # Assume we need S3 as well as Batch
-    test_item = _mark_test("aws-batch", needs_aws_s3(test_item))
+    test_item = _mark_test("aws_batch", needs_aws_s3(test_item))
     # Assume we have Batch if the user has set these variables.
     test_item = needs_env_var("TOIL_AWS_BATCH_QUEUE", "an AWS Batch queue name or ARN")(
         test_item
@@ -443,9 +465,9 @@ def needs_google_storage(test_item: MT) -> MT:
     Cloud is installed and we ought to be able to access public Google Storage
     URIs.
     """
-    test_item = _mark_test("google-storage", needs_online(test_item))
+    test_item = _mark_test("google_storage", needs_online(test_item))
     try:
-        from google.cloud import storage  # noqa
+        import google.clould.storage  # type: ignore[import-untyped]
     except ImportError:
         return unittest.skip(
             "Install Toil with the 'google' extra to include this test."
@@ -458,7 +480,7 @@ def needs_google_project(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if we have a Google Cloud project set.
     """
-    test_item = _mark_test("google-project", needs_online(test_item))
+    test_item = _mark_test("google_project", needs_online(test_item))
     test_item = needs_env_var("TOIL_GOOGLE_PROJECTID", "a Google project ID")(test_item)
     return test_item
 
@@ -471,6 +493,11 @@ def needs_gridengine(test_item: MT) -> MT:
     return unittest.skip("Install GridEngine to include this test.")(test_item)
 
 
+pneeds_gridengine = pytest.mark.skipif(
+    not which("qhost"), reason="Install GridEngine to include this test."
+)
+
+
 def needs_torque(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if PBS/Torque is installed."""
     test_item = _mark_test("torque", test_item)
@@ -479,6 +506,11 @@ def needs_torque(test_item: MT) -> MT:
     return unittest.skip("Install PBS/Torque to include this test.")(test_item)
 
 
+pneeds_torque = pytest.mark.skipif(
+    not which("pbsnodes"), reason="Install PBS/Torque to include this test."
+)
+
+
 def needs_kubernetes_installed(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Kubernetes is installed."""
     test_item = _mark_test("kubernetes", test_item)
@@ -493,28 +525,40 @@ def needs_kubernetes_installed(test_item: MT) -> MT:
     return test_item
 
 
-def needs_kubernetes(test_item: MT) -> MT:
-    """Use as a decorator before test classes or methods to run only if Kubernetes is installed and configured."""
-    test_item = needs_kubernetes_installed(needs_online(test_item))
+def _is_kubernetes_installed_and_configured() -> bool:
     try:
         import kubernetes
 
         try:
-            kubernetes.config.load_kube_config()
-        except kubernetes.config.ConfigException:
+            kubernetes.config.load_kube_config()  # type: ignore[attr-defined]
+        except kubernetes.config.ConfigException:  # type: ignore[attr-defined]
             try:
-                kubernetes.config.load_incluster_config()
-            except kubernetes.config.ConfigException:
-                return unittest.skip(
-                    "Configure Kubernetes (~/.kube/config, $KUBECONFIG, "
-                    "or current pod) to include this test."
-                )(test_item)
+                kubernetes.config.load_incluster_config()  # type: ignore[attr-defined]
+            except kubernetes.config.ConfigException:  # type: ignore[attr-defined]
+                return False
     except ImportError:
-        # We should already be skipping this test
-        pass
+        return False
+    return True
+
+
+def needs_kubernetes(test_item: MT) -> MT:
+    """Use as a decorator before test classes or methods to run only if Kubernetes is installed and configured."""
+    test_item = needs_kubernetes_installed(needs_online(test_item))
+    if not _is_kubernetes_installed_and_configured():
+        return unittest.skip(
+            "Configure Kubernetes (~/.kube/config, $KUBECONFIG, "
+            "or current pod) to include this test."
+        )(test_item)
     return test_item
 
 
+pneeds_kubernetes = pytest.mark.skipif(
+    _skip_online() or not _is_kubernetes_installed_and_configured(),
+    reason="Configure Kubernetes (~/.kube/config, $KUBECONFIG, "
+    "or current pod) to include this test.",
+)
+
+
 def needs_mesos(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Mesos is installed."""
     test_item = _mark_test("mesos", test_item)
@@ -524,7 +568,16 @@ def needs_mesos(test_item: MT) -> MT:
         )(test_item)
     try:
         import psutil  # noqa
-        import pymesos  # noqa
+        # If pymesos is installed, because it isn't typed, mypy sees an
+        # import-untyped error here.
+        #
+        # If pymesos *isn't* installed, mypy sees an import-not-found error
+        # here.
+        #
+        # If we ignore mypy errors by name, we'll get a mypy error for ignoring
+        # whichever one isn't actually occurring on the current system. So we
+        # need a blanket ignore here, or a much cleverer mypy.
+        import pymesos  # type: ignore
     except ImportError:
         return unittest.skip(
             "Install Mesos (and Toil with the 'mesos' extra) to include this test."
@@ -532,6 +585,23 @@ def needs_mesos(test_item: MT) -> MT:
     return test_item
 
 
+def _mesos_avail() -> bool:
+    if not (which("mesos-master") or which("mesos-agent")):
+        return False
+    try:
+        import psutil
+        import pymesos
+    except ImportError:
+        return False
+    return True
+
+
+pneeds_mesos = pytest.mark.skipif(
+    not _mesos_avail(),
+    reason="Install Mesos (and Toil with the 'mesos' extra) to include this test.",
+)
+
+
 def needs_slurm(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Slurm is installed."""
     test_item = _mark_test("slurm", test_item)
@@ -540,11 +610,16 @@ def needs_slurm(test_item: MT) -> MT:
     return unittest.skip("Install Slurm to include this test.")(test_item)
 
 
+pneeds_slurm = pytest.mark.skipif(
+    not which("squeue"), reason="Install Slurm to include this test."
+)
+
+
 def needs_htcondor(test_item: MT) -> MT:
     """Use a decorator before test classes or methods to run only if the HTCondor is installed."""
     test_item = _mark_test("htcondor", test_item)
     try:
-        import htcondor
+        import htcondor  # type: ignore
 
         htcondor.Collector(os.getenv("TOIL_HTCONDOR_COLLECTOR")).query(
             constraint="False"
@@ -576,6 +651,11 @@ def needs_lsf(test_item: MT) -> MT:
         return unittest.skip("Install LSF to include this test.")(test_item)
 
 
+pneeds_lsf = pytest.mark.skipif(
+    not which("bsub"), reason="Install LSF to include this test."
+)
+
+
 def needs_java(test_item: MT) -> MT:
     """Use as a test decorator to run only if java is installed."""
     test_item = _mark_test("java", test_item)
@@ -599,6 +679,14 @@ def needs_docker(test_item: MT) -> MT:
         return unittest.skip("Install docker to include this test.")(test_item)
 
 
+pneeds_docker = pytest.mark.skipif(
+    _skip_online()
+    or os.getenv("TOIL_SKIP_DOCKER", "").lower() == "true"
+    or not which("docker"),
+    reason="Requested to skip docker test or docker is not installed.",
+)
+
+
 def needs_singularity(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if
@@ -641,6 +729,12 @@ def needs_local_cuda(test_item: MT) -> MT:
         )(test_item)
 
 
+pneeds_local_cuda = pytest.mark.skipif(
+    not have_working_nvidia_smi(),
+    reason="Install nvidia-smi, an nvidia proprietary driver, and a CUDA-capable nvidia GPU to include this test.",
+)
+
+
 def needs_docker_cuda(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if
@@ -655,6 +749,12 @@ def needs_docker_cuda(test_item: MT) -> MT:
         )(test_item)
 
 
+pneeds_docker_cuda = pytest.mark.skipif(
+    _skip_online() or not have_working_nvidia_docker_runtime(),
+    reason="Install nvidia-container-runtime on your Docker server and configure an 'nvidia' runtime to include this test.",
+)
+
+
 def needs_encryption(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if PyNaCl is installed
@@ -689,21 +789,46 @@ def needs_cwl(test_item: MT) -> MT:
         return test_item
 
 
+def _cwl_available() -> bool:
+    try:
+        import cwltool
+    except ImportError:
+        return False
+    return True
+
+
+pneeds_cwl = pytest.mark.skipif(
+    not _cwl_available(),
+    reason="Install Toil with the 'cwl' extra to include this test.",
+)
+
+
+def _wdl_available() -> bool:
+    try:
+        # noinspection PyUnresolvedReferences
+        import WDL  # noqa
+    except ImportError:
+        return False
+    return True
+
+
 def needs_wdl(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if miniwdl is installed
     and configured.
     """
     test_item = _mark_test("wdl", test_item)
-    try:
-        # noinspection PyUnresolvedReferences
-        import WDL  # noqa
-    except ImportError:
-        return unittest.skip("Install Toil with the 'wdl' extra to include this test.")(
-            test_item
-        )
-    else:
+    if _wdl_available():
         return test_item
+    return unittest.skip("Install Toil with the 'wdl' extra to include this test.")(
+        test_item
+    )
+
+
+pneeds_wdl = pytest.mark.skipif(
+    not _wdl_available(),
+    reason="Install Toil with the 'wdl' extra to include this test.",
+)
 
 
 def needs_server(test_item: MT) -> MT:
@@ -713,7 +838,7 @@ def needs_server(test_item: MT) -> MT:
     test_item = _mark_test("server_mode", test_item)
     try:
         # noinspection PyUnresolvedReferences
-        import connexion
+        import connexion  # type: ignore[import-untyped]
 
         print(connexion.__file__)  # keep this import from being removed.
     except ImportError:
@@ -756,6 +881,23 @@ def needs_wes_server(test_item: MT) -> MT:
     return test_item
 
 
+def _is_wes_server_avail() -> bool:
+    wes_url = os.environ.get("TOIL_WES_ENDPOINT")
+    if not wes_url:
+        return False
+    try:
+        urlopen(f"{wes_url}/ga4gh/wes/v1/service-info")
+        return True
+    except (HTTPError, URLError):
+        return False
+
+
+pneeds_wes_server = pytest.mark.skipif(
+    _skip_online() or not _is_wes_server_avail(),
+    reason="Set TOIL_WES_ENDPOINT, or run a WES server at that location to include this test.",
+)
+
+
 def needs_local_appliance(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if
@@ -833,6 +975,13 @@ def integrative(test_item: MT) -> MT:
         )(test_item)
 
 
+pintegrative = pytest.mark.skipif(
+    os.getenv("TOIL_TEST_INTEGRATIVE", "").lower() != "true",
+    reason="Set TOIL_TEST_INTEGRATIVE=True to include this integration test, "
+    "or run `make integration_test_local` to run all integration tests.",
+)
+
+
 def slow(test_item: MT) -> MT:
     """
     Use this decorator to identify tests that are slow and not critical.
@@ -845,6 +994,11 @@ def slow(test_item: MT) -> MT:
         return unittest.skip('Skipped because TOIL_TEST_QUICK is "True"')(test_item)
 
 
+pslow = pytest.mark.skipif(
+    os.environ.get("TOIL_TEST_QUICK", "").lower() == "true",
+    reason='Skipped because TOIL_TEST_QUICK is "True"',
+)
+
 methodNamePartRegex = re.compile("^[a-zA-Z_0-9]+$")
 
 
@@ -881,7 +1035,11 @@ def timeLimit(seconds: int) -> Generator[None, None, None]:
         signal.alarm(0)
 
 
-def make_tests(generalMethod, targetClass, **kwargs):
+def make_tests(
+    generalMethod: Callable[[Any], Any],
+    targetClass: Optional[Callable[[Any], Any]],
+    **kwargs: Any,
+) -> None:
     """
     This method dynamically generates test methods using the generalMethod as a template. Each
     generated function is the result of a unique combination of parameters applied to the
@@ -937,7 +1095,9 @@ def make_tests(generalMethod, targetClass, **kwargs):
 
     """
 
-    def permuteIntoLeft(left, rParamName, right):
+    def permuteIntoLeft(
+        left: dict[str, dict[str, str]], rParamName: str, right: dict[str, str]
+    ) -> None:
         """
         Permutes values in right dictionary into each parameter: value dict pair in the left
         dictionary. Such that the left dictionary will contain a new set of keys each of which is
@@ -972,10 +1132,10 @@ def make_tests(generalMethod, targetClass, **kwargs):
                 left[prmValName + nextPrmVal] = aggDict
             left.pop(prmValName)
 
-    def insertMethodToClass():
+    def insertMethodToClass() -> None:
         """Generate and insert test methods."""
 
-        def fx(self, prms=prms):
+        def fx(self: Any, prms: Any = prms) -> Any:
             if prms is not None:
                 return generalMethod(self, **prms)
             else:
@@ -1085,7 +1245,7 @@ class ApplianceTestSupport(ToilTest):
             self.containerName = str(uuid.uuid4())
             self.popen: Optional[subprocess.Popen[bytes]] = None
 
-        def __enter__(self) -> "Appliance":
+        def __enter__(self) -> Self:
             with self.lock:
                 image = applianceSelf()
                 # Omitting --rm, it's unreliable, see https://github.com/docker/docker/issues/16575