easylink 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

easylink/utilities/spark.smk

@@ -70,7 +70,7 @@ rule wait_for_spark_master:
         while true; do
 
             if [[ -e {params.spark_master_log_file} ]]; then
-                found=`grep -o "\(spark://.*$\)" {params.spark_master_log_file} || true`
+                found=`grep -o "\\(spark://.*$\\)" {params.spark_master_log_file} || true`
 
                 if [[ ! -z $found ]]; then
                     echo "Spark master URL found: $found"
@@ -178,7 +178,7 @@ rule wait_for_spark_worker:
         while true; do
 
             if [[ -e {params.spark_worker_log_file} ]]; then
-                found=`grep -o "\(Worker: Successfully registered with master $MASTER_URL\)" {params.spark_worker_log_file} || true`
+                found=`grep -o "\\(Worker: Successfully registered with master $MASTER_URL\\)" {params.spark_worker_log_file} || true`
 
                 if [[ ! -z $found ]]; then
                     echo "Spark Worker {wildcards.scatteritem} registered successfully"

easylink/utilities/splitter_utils.py

@@ -70,3 +70,38 @@ def split_data_by_size(
             f"{chunk.index[-1]})"
         )
         chunk.to_parquet(os.path.join(chunk_dir, "result.parquet"))
+
+
+def split_data_in_two(input_files: list[str], output_dir: str, *args, **kwargs) -> None:
+    """Splits the data (from a single input slot) into two chunks of equal.
+
+    This function takes all datasets from a single input slot, concatenates them,
+    and then splits the resulting dataset into two chunks of similar size.
+
+    Parameters
+    ----------
+    input_files
+        A list of input file paths to be concatenated and split.
+    output_dir
+        The directory where the resulting chunks will be saved.
+    desired_chunk_size_mb
+        The desired size of each chunk, in megabytes.
+    """
+
+    # concatenate all input files
+    df = pd.DataFrame()
+    for file in input_files:
+        tmp = pd.read_parquet(file)
+        df = pd.concat([df, tmp], ignore_index=True)
+
+    # divide df into two and save each chunk out
+    num_chunks = 2
+    chunk_size = math.ceil(len(df) / num_chunks)
+    for i in range(num_chunks):
+        start = i * chunk_size
+        end = (i + 1) * chunk_size
+        chunk = df.iloc[start:end]
+        chunk_dir = os.path.join(output_dir, f"chunk_{i}")
+        if not os.path.exists(chunk_dir):
+            os.makedirs(chunk_dir)
+        chunk.to_parquet(os.path.join(chunk_dir, "result.parquet"))

{easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.13
+Version: 0.1.15
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers
@@ -19,11 +19,14 @@ Requires-Dist: pyarrow
 Requires-Dist: snakemake>=8.0.0
 Requires-Dist: snakemake-interface-executor-plugins<9.0.0
 Requires-Dist: snakemake-executor-plugin-slurm
+Requires-Dist: pandas-stubs
+Requires-Dist: pyarrow-stubs
 Provides-Extra: docs
 Requires-Dist: sphinx<8.2.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
 Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
 Requires-Dist: sphinx-click; extra == "docs"
+Requires-Dist: sphinx-autobuild; extra == "docs"
 Requires-Dist: typing_extensions; extra == "docs"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
@@ -34,6 +37,7 @@ Requires-Dist: sphinx<8.2.0; extra == "dev"
 Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Requires-Dist: sphinx-autodoc-typehints; extra == "dev"
 Requires-Dist: sphinx-click; extra == "dev"
+Requires-Dist: sphinx-autobuild; extra == "dev"
 Requires-Dist: typing_extensions; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -71,28 +75,32 @@ Installation
 
 There are a few things to install in order to use this package:
 
-- Install singularity. If this is not already installed on your system, you will 
-  likely need to request it from your system admin. 
-  Refer to https://docs.sylabs.io/guides/4.1/admin-guide/installation.html
+- Install singularity. 
 
+  You may need to request it from your system admin. 
+  Refer to https://docs.sylabs.io/guides/4.1/admin-guide/installation.html. 
+  You can check if you already have singularity installed by running the command ``singularity --version``. For an
+  existing installation, your singularity version number is printed.
 
-- Install graphviz via:
+- Install conda. 
+  
+  We recommend `miniforge <https://github.com/conda-forge/miniforge>`_. You can check if you already
+  have conda installed by running the command ``conda --version``. For an existing installation, a version 
+  will be displayed.
 
-  .. code-block:: console
-
-    $ conda install graphviz
-
-- Install EasyLink.
+- Install easylink, python and graphviz in a conda environment.
 
   Option 1 - Install from PyPI with pip::
 
+    $ conda create --name easylink -c conda-forge python=3.12 graphviz 'gcc<14' -y
+    $ conda activate easylink
     $ pip install easylink
 
   Option 2 - Build from source with pip::
-
-    $ git clone git@github.com:ihmeuw/easylink.git  # or git clone https://github.com/ihmeuw/easylink.git
-    $ cd easylink
-    $ pip install .
+    
+    $ conda create --name easylink -c conda-forge python=3.12 graphviz 'gcc<14' -y
+    $ conda activate easylink
+    $ pip install git+https://github.com/ihmeuw/easylink.git
 
 .. _end_installation:
 

{easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/RECORD

@@ -1,22 +1,22 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=khDKUuWafURKVs5EAZkpOMiUHI2-V7axlqrWLPUpuZo,23
+easylink/_version.py,sha256=qb0TalpSt1CbprnFyeLUKqgrqNtmnk9IoQQ7umAoXVY,23
 easylink/cli.py,sha256=ARSKAljepNOEYd1VCS_QqBJQIBLzE3IgKiOb5-OROdY,6380
-easylink/configuration.py,sha256=Ire2pMZNZ6wtSwhcWnQpYa-snX4KrhXgovlQwQ2Wxf4,12530
-easylink/graph_components.py,sha256=PhMKxpgZjorhubS7vcta1pgXgXSGplmPulQpV0YZhqo,14811
-easylink/implementation.py,sha256=AwGl5YCKCSQo91owWj-gg9_5lBz7H_4q2z7jF0BhXs4,8992
+easylink/configuration.py,sha256=lfm8ViUpr1-O-EovTjKZbAlIht2EBv3RndN1mzYbmDE,12565
+easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
+easylink/implementation.py,sha256=4u3QgLOrNttfU9Kd_9u_lg3in4ePoYUfO9u_udwiuh0,10878
 easylink/implementation_metadata.yaml,sha256=VvlEu3Dvlmeh1MpzeYx91j22GiV-9mu3hZP5yVuW04o,6763
-easylink/pipeline.py,sha256=EyCXv5p9WzTqcndXK6ukBJE6jY_fWIP_DGZQUl1wRcY,12284
-easylink/pipeline_graph.py,sha256=vsY6nW_iEwZCNf_N_3CsixsKBUy_5JxGEi61-1Q-KAw,22842
+easylink/pipeline.py,sha256=5KOYH5HyJjVlFoBRKGLs2hn5mpC3tPYG_ux3T1qSV9k,17504
+easylink/pipeline_graph.py,sha256=9ysX4wAkA-WkUoo15jSLAErncybE4tJwznVx7N_kwIA,23922
 easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
-easylink/rule.py,sha256=W97LMI-vkEPipJbnSZLn2BxfYfFtvzGTKzq6YgDVri0,19913
-easylink/runner.py,sha256=k9ICTToHj2xr6MGIuvlWf6YMeZ47UGgseaMByMgUGac,6271
-easylink/step.py,sha256=Hweg1OAGcmrNAt95C-M4ksOAtc_db0oeibbF3cnqhq0,74951
+easylink/rule.py,sha256=uoPj7yFFqiwvxlnhoejrZuPR3YX--y1k02uDDz3viTc,16196
+easylink/runner.py,sha256=cbCo5_NvvulmjjAaBCG6qCmbtJiHK-7NuDvbngdU_PY,6675
+easylink/step.py,sha256=u1AMPrYGNVb3ZH6uB_U0dUeJvOeQ2MoVHdlC8k63AA8,85226
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=uRVjQw7_Ff5IBQw0_Jc93Fzfa-MnbPVPKsy18CCaW7E,1021
-easylink/pipeline_schema_constants/development.py,sha256=0fc6xWRBr5e_xDaldR9sY2vMQJU1wnlhDQS_-yUOT6g,12339
-easylink/pipeline_schema_constants/testing.py,sha256=ohcTlT_viZYxS1GkO46mjkb8IzXo6yIOqvBbb4YrOhA,10897
+easylink/pipeline_schema_constants/__init__.py,sha256=HbN-NytoGuk8aTfe0Wal232UnLopFBQGe2uRjmg_igQ,1272
+easylink/pipeline_schema_constants/development.py,sha256=yRzkCiBqF_Jv3Y0GNvswVAWeZfKJRXk8Y8Q9ZhwCg_A,11596
+easylink/pipeline_schema_constants/testing.py,sha256=8vVGj7opZ9Uzj7EHGMbgXyZj3_SboIeUPB0XlZkmvrM,18901
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
 easylink/steps/dev/build-containers-remote.sh,sha256=Hy-kaaXf-ta6n8SzOz_ahByjMY5T7J71MvzXRXDvQw8,271
@@ -40,11 +40,11 @@ easylink/utilities/aggregator_utils.py,sha256=pqBog6kEX4MXBBMjQtHFlE5gEMqRWb5VFl
 easylink/utilities/data_utils.py,sha256=CcnM3u0_MQDQo3jMs3E4IK_rz8wAsFdJ674fZxYEFZg,4620
 easylink/utilities/general_utils.py,sha256=El1W0nn4P27sRBGotNQb-9du-Gbhk9ggSuu4vmGDfwo,4591
 easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,924
-easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
-easylink/utilities/splitter_utils.py,sha256=y4CbbTBgRaoXFxy-9Eu5eWx4lA4ZEcbrYpxgLIzG_kc,2602
+easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
+easylink/utilities/splitter_utils.py,sha256=UOz4hjkEPqaAz0RrDkDYYej79lLSaq0VVVSH_tF1z0o,3838
 easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
-easylink-0.1.13.dist-info/METADATA,sha256=ooL68LseA1cN5X2wLOB_uIFgXIjW0PCINie7aMHw6t0,2805
-easylink-0.1.13.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-easylink-0.1.13.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.13.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.13.dist-info/RECORD,,
+easylink-0.1.15.dist-info/METADATA,sha256=wV8zQKO5KUtsvJIaAv7wi_I1Nich7hqjfuKNzQLM-o4,3449
+easylink-0.1.15.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+easylink-0.1.15.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.15.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.15.dist-info/RECORD,,

{easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any