toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/lib/io.py

@@ -6,14 +6,37 @@ import stat
 import sys
 import tempfile
 import uuid
-from collections.abc import Iterator
+from collections.abc import Iterator, Iterable
 from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
+from toil.lib.memoize import memoize
+from toil.lib.misc import StrPath
+
 logger = logging.getLogger(__name__)
 
 
+@memoize
+def get_toil_home() -> str:
+    """
+    Get the Toil home directory for storing configuration and global state.
+
+    Raises an error if it does not exist and cannot be created. Safe to run
+    simultaneously in multiple processes.
+    """
+   
+    # TODO: should this use an XDG config directory or ~/.config to not clutter the
+    # base home directory?
+    toil_home_dir = os.path.join(os.path.expanduser("~"), ".toil")
+
+    dir_path = try_path(toil_home_dir)
+    if dir_path is None:
+        raise RuntimeError(
+            f"Cannot create or access Toil configuration directory {toil_home_dir}"
+        )
+    return dir_path
+
 TOIL_URI_SCHEME = "toilfile:"
 
 
@@ -22,6 +45,9 @@ REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
 def is_standard_url(filename: str) -> bool:
+    """
+    Return True if the given URL is a non-Toil, non-file: URL.
+    """
     return is_url_with_scheme(filename, STANDARD_SCHEMES)
 
 def is_remote_url(filename: str) -> bool:
@@ -49,16 +75,23 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     return False
 
 def is_toil_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toilfile: URL.
+    """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
 def is_file_url(filename: str) -> bool:
-    return is_url_with_scheme(filename, ["file:"])
+    """
+    Return True if a URL is a file: URL.
 
+    Will return False for bare paths.
+    """
+    return is_url_with_scheme(filename, ["file:"])
 
 def mkdtemp(
     suffix: Optional[str] = None,
     prefix: Optional[str] = None,
-    dir: Optional[str] = None,
+    dir: Optional[StrPath] = None,
 ) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
@@ -153,31 +186,31 @@ def robust_rmtree(path: Union[str, bytes]) -> None:
             raise
 
 
-def atomic_tmp_file(final_path: str) -> str:
+def atomic_tmp_file(final_path: StrPath) -> str:
     """Return a tmp file name to use with atomic_install.  This will be in the
     same directory as final_path. The temporary file will have the same extension
     as finalPath.  It the final path is in /dev (/dev/null, /dev/stdout), it is
     returned unchanged and atomic_tmp_install will do nothing."""
     final_dir = os.path.dirname(os.path.normpath(final_path))  # can be empty
     if final_dir == "/dev":
-        return final_path
+        return str(final_path)
     final_basename = os.path.basename(final_path)
     final_ext = os.path.splitext(final_path)[1]
-    base_name = f"{final_basename}.{uuid.uuid4()}.tmp{final_ext}"
+    base_name = f"{final_basename}.{str(uuid.uuid4())}.tmp{final_ext}"
     return os.path.join(final_dir, base_name)
 
 
-def atomic_install(tmp_path, final_path) -> None:
+def atomic_install(tmp_path: StrPath, final_path: StrPath) -> None:
     """atomic install of tmp_path as final_path"""
     if os.path.dirname(os.path.normpath(final_path)) != "/dev":
         os.rename(tmp_path, final_path)
 
 
 @contextmanager
-def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
+def AtomicFileCreate(final_path: StrPath, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
     the temporary file in the same directory as finalPath.  If the code in
-    context succeeds, the file renamed to its actually name.  If an error
+    context succeeds, the file renamed to its actual name.  If an error
     occurs, the file is not installed and is removed unless keep is specified.
     """
     tmp_path = atomic_tmp_file(final_path)
@@ -229,23 +262,23 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
     our old default.
     """
     if suggested_name is not None:
-        generated_dir_path: str = os.path.join(in_directory, suggested_name)
+        generated_dir_path1 = os.path.join(in_directory, suggested_name)
         try:
-            os.mkdir(generated_dir_path)
-            os.chmod(generated_dir_path, 0o777)
-            return generated_dir_path
+            os.mkdir(generated_dir_path1)
+            os.chmod(generated_dir_path1, 0o777)
+            return generated_dir_path1
         except FileExistsError:
             pass
     for i in range(
         4, 32 + 1
     ):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
-            truncated_uuid: str = str(uuid.uuid4()).replace("-", "")[:i]
-            generated_dir_path: str = os.path.join(in_directory, truncated_uuid)
+            truncated_uuid = str(uuid.uuid4()).replace("-", "")[:i]
+            generated_dir_path2 = os.path.join(in_directory, truncated_uuid)
             try:
-                os.mkdir(generated_dir_path)
-                os.chmod(generated_dir_path, 0o777)
-                return generated_dir_path
+                os.mkdir(generated_dir_path2)
+                os.chmod(generated_dir_path2, 0o777)
+                return generated_dir_path2
             except FileExistsError:
                 pass
     this_should_never_happen: str = os.path.join(in_directory, str(uuid.uuid4()))
@@ -307,7 +340,7 @@ class WriteWatchingStream:
 
         self.backingStream = backingStream
         # We have no write listeners yet
-        self.writeListeners = []
+        self.writeListeners: list[Callable[[int], None]] = []
 
     def onWrite(self, listener: Callable[[int], None]) -> None:
         """
@@ -318,7 +351,7 @@ class WriteWatchingStream:
 
     # Implement the file API from https://docs.python.org/2.4/lib/bltin-file-objects.html
 
-    def write(self, data):
+    def write(self, data: bytes) -> None:
         """
         Write the given data to the file.
         """
@@ -330,7 +363,7 @@ class WriteWatchingStream:
             # Send out notifications
             listener(len(data))
 
-    def writelines(self, datas):
+    def writelines(self, datas: Iterable[bytes]) -> None:
         """
         Write each string from the given iterable, without newlines.
         """
@@ -338,14 +371,14 @@ class WriteWatchingStream:
         for data in datas:
             self.write(data)
 
-    def flush(self):
+    def flush(self) -> None:
         """
         Flush the backing stream.
         """
 
         self.backingStream.flush()
 
-    def close(self):
+    def close(self) -> None:
         """
         Close the backing stream.
         """

toil/lib/misc.py

@@ -9,10 +9,16 @@ import sys
 import time
 from collections.abc import Iterator
 from contextlib import closing
-from typing import Optional
+from typing import Optional, Union
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
 
 logger = logging.getLogger(__name__)
 
+StrPath: TypeAlias = Union[str, os.PathLike[str]]
+FileDescriptorOrPath: TypeAlias = Union[int, bytes, os.PathLike[bytes], StrPath]
 
 def get_public_ip() -> str:
     """Get the IP that this machine uses to contact the internet.
@@ -63,6 +69,24 @@ def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+def unix_seconds_to_timestamp(timestamp: float) -> str:
+    """
+    Convert a time in seconds since the Unix epoch to an ISO 8601 string.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
+
+def unix_seconds_to_local_time(timestamp: float) -> datetime.datetime:
+    """
+    Returns a local time corresponding to the given number of seconds since the Unix epoch.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).astimezone()
+
+def seconds_to_duration(time_difference: float) -> str:
+    """
+    Convert a time difference in seconds to an ISO 8601 duration string.
+    """
+    return f"PT{time_difference:.3f}S"
+
 
 def slow_down(seconds: float) -> float:
     """

toil/lib/resources.py

@@ -17,6 +17,7 @@ import os
 import resource
 import sys
 
+from toil.lib.misc import StrPath
 
 class ResourceMonitor:
     """
@@ -89,7 +90,7 @@ class ResourceMonitor:
         )
 
 
-def glob(glob_pattern: str, directoryname: str) -> list[str]:
+def glob(glob_pattern: str, directoryname: StrPath) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].

toil/lib/retry.py

@@ -24,21 +24,21 @@ objects wrapping Exceptions to include additional conditions.
 
 For example, retrying on a one Exception (HTTPError)::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[HTTPError])
     def update_my_wallpaper():
-        return get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 Or::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[HTTPError, ValueError])
     def update_my_wallpaper():
-        return get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 The examples above will retry for the default interval on any errors specified
 the "errors=" arg list.
@@ -46,7 +46,7 @@ the "errors=" arg list.
 To retry on specifically 500/502/503/504 errors, you could specify an ErrorCondition
 object instead, for example::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -55,11 +55,11 @@ object instead, for example::
                    error_codes=[500, 502, 503, 504]
                )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on specifically errors containing the phrase "NotFound"::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -68,11 +68,11 @@ To retry on specifically errors containing the phrase "NotFound"::
             error_message_must_include="NotFound"
         )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on all HTTPError errors EXCEPT an HTTPError containing the phrase "NotFound"::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -83,7 +83,7 @@ To retry on all HTTPError errors EXCEPT an HTTPError containing the phrase "NotF
                    retry_on_this_condition=False
                )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on boto3's specific status errors, an example of the implementation is::
 

toil/lib/threading.py

@@ -36,12 +36,13 @@ import psutil
 
 from toil.lib.exceptions import raise_
 from toil.lib.io import robust_rmtree
+from toil.lib.misc import StrPath
 
 logger = logging.getLogger(__name__)
 
 
 def ensure_filesystem_lockable(
-    path: str, timeout: float = 30, hint: Optional[str] = None
+    path: StrPath, timeout: float = 30, hint: Optional[str] = None
 ) -> None:
     """
     Make sure that the filesystem used at the given path is one where locks are safe to use.
@@ -71,7 +72,7 @@ def ensure_filesystem_lockable(
             # Start a child process to stat the path. See <https://unix.stackexchange.com/a/402236>.
             # We really should call statfs but no bindings for it are in PyPI.
             completed = subprocess.run(
-                ["stat", "-f", "-c", "%T", path],
+                ["stat", "-f", "-c", "%T", str(path)],
                 check=True,
                 capture_output=True,
                 timeout=timeout,
@@ -85,7 +86,7 @@ def ensure_filesystem_lockable(
             # Stat didn't work. Maybe we don't have the right version of stat installed?
             logger.warning(
                 "Could not determine filesystem type at %s because of: %s",
-                path,
+                str(path),
                 e.stderr.decode("utf-8", errors="replace").strip(),
             )
             # If we don't know the filesystem type, keep going anyway.
@@ -107,7 +108,7 @@ def ensure_filesystem_lockable(
             # flaky with regard to locks actually locking anything).
             logger.debug(
                 "Detected that %s has lockable filesystem type: %s",
-                path,
+                str(path),
                 filesystem_type,
             )
 
@@ -518,7 +519,7 @@ def process_name_exists(base_dir: str, name: str) -> bool:
 # Similar to the process naming system above, we define a global mutex system
 # for critical sections, based just around file locks.
 @contextmanager
-def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
+def global_mutex(base_dir: StrPath, mutex: str) -> Iterator[None]:
     """
     Context manager that locks a mutex. The mutex is identified by the given
     name, and scoped to the given directory. Works across all containers that
@@ -527,7 +528,7 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
 
     Only works between processes, NOT between threads.
 
-    :param str base_dir: Base directory to work in. Defines the shared namespace.
+    :param base_dir: Base directory to work in. Defines the shared namespace.
     :param str mutex: Mutex to lock. Must be a permissible path component.
     """
 
@@ -674,7 +675,7 @@ class LastProcessStandingArena:
     Consider using a try/finally; this class is not a context manager.
     """
 
-    def __init__(self, base_dir: str, name: str) -> None:
+    def __init__(self, base_dir: StrPath, name: str) -> None:
         """
         Connect to the arena specified by the given base_dir and name.
 
@@ -683,12 +684,12 @@ class LastProcessStandingArena:
 
         Doesn't enter or leave the arena.
 
-        :param str base_dir: Base directory to work in. Defines the shared namespace.
-        :param str name: Name of the arena. Must be a permissible path component.
+        :param base_dir: Base directory to work in. Defines the shared namespace.
+        :param name: Name of the arena. Must be a permissible path component.
         """
 
         # Save the base_dir which namespaces everything
-        self.base_dir = base_dir
+        self.base_dir = str(base_dir)
 
         # We need a mutex name to allow only one process to be entering or
         # leaving at a time.