easylink 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

easylink/utilities/__init__.py

@@ -1,8 +1,9 @@
 """
 =========
-utilities
+Utilities
 =========
 
-Utility functions.
+This package contains a collection of helper modules and files. It serves as a
+central repository for common operations, filepaths, and configuration data.
 
 """

easylink/utilities/aggregator_utils.py

@@ -9,6 +9,7 @@ embarrassingly parallel manner.
 
 Note that it is critical that all data aggregating utility functions are definied
 in this module; easylink will not be able to find them otherwise.
+
 """
 
 import pandas as pd

easylink/utilities/data_utils.py

@@ -1,14 +1,43 @@
 # mypy: ignore-errors
+"""
+==============
+Data Utilities
+==============
+
+This module contains utility functions for handling data files and directories.
+
+"""
+
 import os
 import shutil
+from collections.abc import Callable
 from datetime import datetime
 from pathlib import Path
 
 import yaml
 
 
-def modify_umask(func):
-    """Decorator to wrap umask modification before making directories"""
+def modify_umask(func: Callable) -> Callable:
+    """Decorates a function to modify a process's umask temporarily before calling the function.
+
+    This decorator sets the umask to 0o002, which grants write permission to the
+    group while preserving the umask settings for the owner and others. It ensures
+    that any file or directory created by the decorated function has group write
+    permissions. After the function executes, the decorator restores the original
+    umask.
+
+    Parameters
+    ----------
+    func
+        The function to be decorated. It can be any callable that might create files
+        or directories during its execution.
+
+    Returns
+    -------
+        A wrapper function that, when called, modifies the umask, calls the original
+        function with the provided arguments, and finally restores the umask to its
+        original value.
+    """
 
     def wrapper(*args, **kwargs):
         old_umask = os.umask(0o002)
@@ -22,11 +51,28 @@ def modify_umask(func):
 
 @modify_umask
 def create_results_directory(results_dir: Path) -> None:
+    """Creates a results directory.
+
+    This creates the high-level results directory to be used for storing results
+    (including any missing sub-directories).
+
+    Parameters
+    ----------
+    results_dir
+        The directory to be created.
+    """
     results_dir.mkdir(parents=True, exist_ok=True)
 
 
 @modify_umask
 def create_results_intermediates(results_dir: Path) -> None:
+    """Creates required sub-directories within a given run's results directory.
+
+    Parameters
+    ----------
+    results_dir
+        The results directory for the current run.
+    """
     (results_dir / "intermediate").mkdir(exist_ok=True)
     (results_dir / "diagnostics").mkdir(exist_ok=True)
 
@@ -37,6 +83,21 @@ def copy_configuration_files_to_results_directory(
     computing_environment: Path | None,
     results_dir: Path,
 ) -> None:
+    """Copies all configuration files into the results directory.
+
+    Parameters
+    ----------
+    pipeline_specification
+        The filepath to the pipeline specification file.
+    input_data
+        The filepath to the input data specification file (_not_ the paths to the
+        input data themselves).
+    computing_environment
+        The filepath to the specification file defining the computing environment
+        to run the pipeline on.
+    results_dir
+       The directory to write results and incidental files (logs, etc.) to.
+    """
     shutil.copy(pipeline_specification, results_dir)
     shutil.copy(input_data, results_dir)
     if computing_environment:
@@ -44,18 +105,50 @@ def copy_configuration_files_to_results_directory(
 
 
 def get_results_directory(output_dir: str | None, no_timestamp: bool) -> Path:
+    """Determines the results directory path.
+
+    This function determines the filepath for storing results by (optionally) appending
+    a timestamp to the specified output directory. If no output directory is provided,
+    it defaults to a directory named 'results' in the current working directory.
+
+    Parameters
+    ----------
+    output_dir
+        The directory to write results and incidental files (logs, etc.) to. If no
+        value is provided, results will be written to a 'results/' directory in the
+        current working directory.
+    no_timestamp
+        Whether or not to save the results in a timestamped sub-directory.
+
+    Returns
+    -------
+        The fully resolved path to the results directory.
+    """
     results_dir = Path("results" if output_dir is None else output_dir).resolve()
     if not no_timestamp:
-        launch_time = _get_timestamp()
-        results_dir = results_dir / launch_time
+        results_dir = results_dir / _get_timestamp()
     return results_dir
 
 
-def _get_timestamp():
+def _get_timestamp() -> str:
     return datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
 
 
 def load_yaml(filepath: str | Path) -> dict:
+    """Loads and returns the contents of a YAML file.
+
+    This function uses `yaml.safe_load` to parse the YAML file, which is designed
+    to safely load a subset of YAML without executing arbitrary code.
+
+    Parameters
+    ----------
+    filepath
+        The path to the YAML file to be loaded.
+
+    Returns
+    -------
+        The contents of the YAML file.
+    """
     with open(filepath, "r") as file:
         data = yaml.safe_load(file)
     return data

easylink/utilities/general_utils.py

@@ -1,4 +1,13 @@
 # mypy: ignore-errors
+"""
+=================
+General Utilities
+=================
+
+This module contains various utility functions.
+
+"""
+
 import errno
 import functools
 import shutil
@@ -14,7 +23,26 @@ from loguru import logger
 def handle_exceptions(
     func: Callable, exceptions_logger: Any, with_debugger: bool
 ) -> Callable:
-    """Drops a user into an interactive debugger if func raises an error."""
+    """Wraps a function to handle exceptions by logging and optionally dropping into a debugger.
+
+    Parameters
+    ----------
+    func
+        The wrapped function that is executed and monitored for exceptions.
+    exceptions_logger
+        The logging object used to log exceptions that occur during function execution.
+    with_debugger
+        Whether or not to drop into an interactive debugger upon encountering an exception.
+
+    Returns
+    -------
+        A wrapped version of `func` that includes the exception handling logic.
+
+    Notes
+    -----
+    Exceptions `BdbQuit` and `KeyboardInterrupt` are re-raised _without_ logging
+    to allow for normal debugger and program exit behaviors.
+    """
 
     @functools.wraps(func)
     def wrapped(*args, **kwargs):
@@ -35,13 +63,18 @@ def handle_exceptions(
     return wrapped
 
 
-def configure_logging_to_terminal(verbose: int):
-    """Sets up logging to ``sys.stdout``.
+def configure_logging_to_terminal(verbose: int) -> None:
+    """Configures logging output to the terminal with optional verbosity levels.
 
     Parameters
     ----------
     verbose
-        Verbosity of the logger.
+        An integer indicating the verbosity level of the logging output. Higher
+        values produce more detailed logging information.
+
+    Notes
+    -----
+    This function clears any default logging configuration before applying the new settings.
     """
     logger.remove(0)  # Clear default configuration
     _add_logging_sink(sys.stdout, verbose, colorize=True)
@@ -49,21 +82,20 @@ def configure_logging_to_terminal(verbose: int):
 
 def _add_logging_sink(
     sink: TextIO, verbose: int, colorize: bool = False, serialize: bool = False
-):
+) -> None:
     """Adds a logging sink to the global process logger.
 
     Parameters
     ----------
     sink
-        Either a file or system file descriptor like ``sys.stdout``.
+        The output stream to which log messages will be directed, e.g. ``sys.stdout``.
     verbose
-        Verbosity of the logger.
+        Verbosity of the logger. The log level is set to INFO if 0 and DEBUG otherwise.
     colorize
         Whether to use the colorization options from :mod:`loguru`.
     serialize
         Whether the logs should be converted to JSON before they're dumped
         to the logging sink.
-
     """
     message_format = (
         "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <green>{elapsed}</green> | "
@@ -104,7 +136,6 @@ def exit_with_validation_error(error_msg: dict) -> None:
     SystemExit
         Exits the program with an EINVAL (invalid argument) code due to
         previously-determined validation errors.
-
     """
 
     logger.error(
@@ -118,5 +149,12 @@ def exit_with_validation_error(error_msg: dict) -> None:
 
 
 def is_on_slurm() -> bool:
-    """Returns True if the current environment is a SLURM cluster."""
+    """Returns True if the current environment is a SLURM cluster.
+
+    Notes
+    -----
+    This function simply checks for the presence of the `sbatch` command to _infer_
+    if SLURM is installed. It does _not_ check if SLURM is currently active or
+    managing jobs.
+    """
     return shutil.which("sbatch") is not None

easylink/utilities/paths.py

@@ -2,15 +2,21 @@
 =========
 Filepaths
 =========
+
+This module contains commonly-used filepaths and directories.
+
 """
 
 from pathlib import Path
 
 # TODO: We'll need to update this to be more generic for external users and have a way of configuring this
 CONTAINER_DIR = "/mnt/team/simulation_science/priv/engineering/er_ecosystem/images"
+"""Path to the directory where the container images are stored."""
 IMPLEMENTATION_METADATA = Path(__file__).parent.parent / "implementation_metadata.yaml"
-# Bind EasyLink temp dir to /tmp in the container.
-# For now, put slurm in /tmp to avoid creating a subdir with a prolog script
+"""Path to the implementation metadata file."""
 EASYLINK_TEMP = {"local": Path("/tmp/easylink"), "slurm": Path("/tmp")}
-
+"""Paths to the easylink tmp/ directory to get bound to the container's /tmp directory.
+When running on slurm, we bind /tmp (rather than /tmp/easylink) to avoid creating 
+a subdir with a prolog script"""
 SPARK_SNAKEFILE = Path(__file__).parent / "spark.smk"
+"""Path to the Snakemake snakefile containing spark-specific rules."""

easylink/utilities/splitter_utils.py

@@ -9,6 +9,7 @@ parallel manner.
 
 Note that it is critical that all data splitting utility functions are definied
 in this module; easylink will not be able to find them otherwise.
+
 """
 
 import math

easylink/utilities/validation_utils.py

@@ -1,3 +1,13 @@
+"""
+=========================
+Data Validation Utilities
+=========================
+
+This module contains utility functions for validating datasets, e.g. the validation
+function(s) for processed data being passed out of one pipeline step and into the next.
+
+"""
+
 from pathlib import Path
 
 import pandas as pd
@@ -5,6 +15,25 @@ from pyarrow import parquet as pq
 
 
 def validate_input_file_dummy(filepath: str) -> None:
+    """Validates an input file to a dummy :class:`~easylink.step.Step`.
+
+    This function is intended to be used as the :attr:`~easylink.graph_components.InputSlot.validator`
+    for _all_ input data at every step in the dummy/:mod:`easylink.pipeline_schema_constants.development`
+    pipeline schema. It simply checks for supported file types as well as the presence
+    of required columns.
+
+    Parameters
+    ----------
+    filepath
+        The path to the input data file to be validated.
+
+    Raises
+    ------
+    NotImplementedError
+        If the file type is not supported.
+    LookupError
+        If the file is missing required columns.
+    """
     extension = Path(filepath).suffix
     if extension == ".parquet":
         output_columns = set(pq.ParquetFile(filepath).schema.names)

{easylink-0.1.7.dist-info → easylink-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: easylink
-Version: 0.1.7
+Version: 0.1.9
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.7.dist-info → easylink-0.1.9.dist-info}/RECORD

@@ -1,22 +1,22 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=YpKDcdV7CqL8n45u267wKtyloM13FSVbOdrqgNZnSLM,22
+easylink/_version.py,sha256=XIaxbMbyiP-L3kguR1GhxirFblTXiHR1lMfDVITvHUI,22
 easylink/cli.py,sha256=ARSKAljepNOEYd1VCS_QqBJQIBLzE3IgKiOb5-OROdY,6380
 easylink/configuration.py,sha256=Ire2pMZNZ6wtSwhcWnQpYa-snX4KrhXgovlQwQ2Wxf4,12530
-easylink/graph_components.py,sha256=U6gbKjQVTBftdOGlH-oOKS6o0dyS88IL6MCpI_mhv3s,12354
+easylink/graph_components.py,sha256=6OipaUkCW2ESBW6bxwZVgpRAX8RuL15m4x_mGE7i4R8,12669
 easylink/implementation.py,sha256=AwGl5YCKCSQo91owWj-gg9_5lBz7H_4q2z7jF0BhXs4,8992
 easylink/implementation_metadata.yaml,sha256=VvlEu3Dvlmeh1MpzeYx91j22GiV-9mu3hZP5yVuW04o,6763
 easylink/pipeline.py,sha256=EyCXv5p9WzTqcndXK6ukBJE6jY_fWIP_DGZQUl1wRcY,12284
 easylink/pipeline_graph.py,sha256=vsY6nW_iEwZCNf_N_3CsixsKBUy_5JxGEi61-1Q-KAw,22842
-easylink/pipeline_schema.py,sha256=ckvA4deRYalY5dLLbJDrO_pKttMuWnEUvSn5fSdu4jc,5900
+easylink/pipeline_schema.py,sha256=kINpvy2Fl2S3FBqgdgZCCFHEk237_36X4ltLOtk5-dE,5862
 easylink/rule.py,sha256=W97LMI-vkEPipJbnSZLn2BxfYfFtvzGTKzq6YgDVri0,19913
 easylink/runner.py,sha256=k9ICTToHj2xr6MGIuvlWf6YMeZ47UGgseaMByMgUGac,6271
-easylink/step.py,sha256=tTlDbhtjd7vkKmsnq622WnwQgBAdTN1dapUJqhUlPjA,65664
+easylink/step.py,sha256=8EhoFOXBLWgDfb3OhmQu5g03fqElIJCWg8-Y_5azKEA,67100
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=RVUncdInRvafu10hbf7J9QQv7cE_pg3ylw_C0v1uIOY,684
-easylink/pipeline_schema_constants/development.py,sha256=-F2xaht9u66oJsXyJ8-9Mnx0PoibVZNpTC6ReRykD9w,11280
-easylink/pipeline_schema_constants/testing.py,sha256=icg7Vx0t8Wnic_Bx8tGkYB5wlZmrBqHYeQSRc9mR0Lo,10704
+easylink/pipeline_schema_constants/__init__.py,sha256=uRVjQw7_Ff5IBQw0_Jc93Fzfa-MnbPVPKsy18CCaW7E,1021
+easylink/pipeline_schema_constants/development.py,sha256=kOTEqfZD5pWqP9gu7E6r9Cubf3ILtWEUxCfJfrN8znc,11547
+easylink/pipeline_schema_constants/testing.py,sha256=ohcTlT_viZYxS1GkO46mjkb8IzXo6yIOqvBbb4YrOhA,10897
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
 easylink/steps/dev/build-containers-remote.sh,sha256=Hy-kaaXf-ta6n8SzOz_ahByjMY5T7J71MvzXRXDvQw8,271
@@ -35,16 +35,16 @@ easylink/steps/dev/python_pyspark/python_pyspark.def,sha256=j_RmVjspmXGOhJTr10ED
 easylink/steps/dev/r/README.md,sha256=dPjZdDTqcJsZCiwhddzlOj1ob0P7YocZUNFrLIGM1-0,1201
 easylink/steps/dev/r/dummy_step.R,sha256=1TWZY8CEkT6gavrulBxFsKbDSKJJjk0NtJrGH7TIikE,4975
 easylink/steps/dev/r/r-image.def,sha256=LrhXlt0C3k7d_VJWopRPEVARnFWSuq_oILlwo7g03bE,627
-easylink/utilities/__init__.py,sha256=EBk0rvRPZqwqzIqdVo8jkpSiZFFnj_fHaRB-P6EuCmk,59
-easylink/utilities/aggregator_utils.py,sha256=7-zg7znkUow3SuwbqOAX1H0j2GzqU1RjLs2B-hiQWls,971
-easylink/utilities/data_utils.py,sha256=D1Srj_2Ol5mAVt_D8k8fpm5E-GU8s6k1uKLatcQ1Oeo,1597
-easylink/utilities/general_utils.py,sha256=IM78EToICkmkZX1pvYsU6uZnVvXYDmS26H9Tjmg0XCM,3293
-easylink/utilities/paths.py,sha256=yl0cuWChJmB6YKMCQavTKw9jIl-VQhH6cnsM6D5c0Zk,599
+easylink/utilities/__init__.py,sha256=0U33kbv4hoMfFQ_lh5hLwifxRPzOgkLkjKLYxmaK10g,196
+easylink/utilities/aggregator_utils.py,sha256=pqBog6kEX4MXBBMjQtHFlE5gEMqRWb5VFl64u0Lr__g,972
+easylink/utilities/data_utils.py,sha256=CcnM3u0_MQDQo3jMs3E4IK_rz8wAsFdJ674fZxYEFZg,4620
+easylink/utilities/general_utils.py,sha256=El1W0nn4P27sRBGotNQb-9du-Gbhk9ggSuu4vmGDfwo,4591
+easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,924
 easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
-easylink/utilities/splitter_utils.py,sha256=riz3rflTrbkQ8uqMaqmXCY1BaWvgdxGzl8WN7Lb7eO8,2601
-easylink/utilities/validation_utils.py,sha256=qOgn1n3_m5blFN7eHJ9MbOt5DkFA6DWucAOUAjvGvco,764
-easylink-0.1.7.dist-info/METADATA,sha256=tNiHPs5mHZUjAYO4hHpBS8k-26MgUZZe_g6W6GXuVV8,2804
-easylink-0.1.7.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
-easylink-0.1.7.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.7.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.7.dist-info/RECORD,,
+easylink/utilities/splitter_utils.py,sha256=y4CbbTBgRaoXFxy-9Eu5eWx4lA4ZEcbrYpxgLIzG_kc,2602
+easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
+easylink-0.1.9.dist-info/METADATA,sha256=kc6QCMEr_QU7QtZNnQu-Ic20wmV2OzoN_23Ndw6ArEc,2804
+easylink-0.1.9.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+easylink-0.1.9.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.9.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.9.dist-info/RECORD,,

{easylink-0.1.7.dist-info → easylink-0.1.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.1)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any