easylink 0.1.22__py3-none-any.whl → 0.1.23__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.22"
+__version__ = "0.1.23"

easylink/implementation_metadata.yaml

@@ -2,8 +2,8 @@ step_1_python_pandas:
   steps:
   - step_1
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -11,8 +11,8 @@ step_1a_python_pandas:
   steps:
   - step_1a
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -22,8 +22,8 @@ step_1b_python_pandas:
   steps:
   - step_1b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -33,8 +33,8 @@ step_2_python_pandas:
   steps:
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -42,8 +42,8 @@ step_3_python_pandas:
   steps:
   - step_3
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_3_main_output: result.parquet
@@ -51,8 +51,8 @@ step_4_python_pandas:
   steps:
   - step_4
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -62,8 +62,8 @@ step_5_python_pandas:
   steps:
   - step_5
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -73,8 +73,8 @@ step_6_python_pandas:
   steps:
   - step_6
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -84,8 +84,8 @@ step_4a_python_pandas:
   steps:
   - step_4a
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -95,8 +95,8 @@ step_4b_python_pandas:
   steps:
   - step_4b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -106,8 +106,8 @@ step_4b_r:
   steps:
   - step_4b
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -117,8 +117,8 @@ step_1_python_pyspark:
   steps:
   - step_1
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -127,8 +127,8 @@ step_2_python_pyspark:
   steps:
   - step_2
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -137,8 +137,8 @@ step_3_python_pyspark:
   steps:
   - step_3
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_3_main_output: result.parquet
@@ -147,8 +147,8 @@ step_4_python_pyspark:
   steps:
   - step_4
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -158,8 +158,8 @@ step_1_r:
   steps:
   - step_1
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_1_main_output: result.parquet
@@ -168,8 +168,8 @@ step_2_r:
   steps:
   - step_2
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_2_main_output: result.parquet
@@ -178,8 +178,8 @@ step_3_r:
   steps:
   - step_3
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_3_main_output: result.parquet
@@ -188,8 +188,8 @@ step_4_r:
   steps:
   - step_4
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -201,8 +201,8 @@ step_1_and_step_2_combined_python_pandas:
   - step_1
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -211,8 +211,8 @@ step_1_and_step_2_parallel_python_pandas:
   - step_1
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: STEP_1_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,STEP_2_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS
@@ -223,8 +223,8 @@ step_3_and_step_4_combined_python_pandas:
   - step_3
   - step_4
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_4_main_output: result.parquet
@@ -233,8 +233,8 @@ step_1a_and_step_1b_combined_python_pandas:
   - step_1a
   - step_1b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -362,3 +362,10 @@ update_clusters_by_connected_components:
   script_cmd: python /update_clusters_by_connected_components.py
   outputs:
     clusters: result.parquet
+middle_name_to_initial:
+  steps:
+  - pre-processing
+  image_name: main/middle_name_to_initial.sif
+  script_cmd: python /middle_name_to_initial.py
+  outputs:
+    dataset: dataset

easylink/pipeline_schema_constants/testing.py

@@ -607,7 +607,7 @@ NODES_OUTPUT_DIR = [
         input_slots=[
             InputSlot(
                 name="step_2_main_input",
-                env_var="DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH",
+                env_var="MAIN_INPUT_DIR_PATH",
                 validator=validate_dir,
             )
         ],

easylink/rule.py

@@ -182,15 +182,15 @@ rule:
         # TODO [MIC-5787]: handle multiple wildcards, e.g.
         #   output_paths = ",".join(self.output)
         #   wildcards_subdir = "/".join([f"{{wildcards.{wc}}}" for wc in self.wildcards])
-        #   and then in shell cmd: export DUMMY_CONTAINER_OUTPUT_PATHS={output_paths}/{wildcards_subdir}
+        #   and then in shell cmd: export OUTPUT_PATHS={output_paths}/{wildcards_subdir}
 
         # snakemake shell commands require wildcards to be prefaced with 'wildcards.'
         output_files = ",".join(self.output).replace("{chunk}", "{wildcards.chunk}")
         shell_cmd = f"""
     shell:
         '''
-        export DUMMY_CONTAINER_OUTPUT_PATHS={output_files}
-        export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY={self.diagnostics_dir}"""
+        export OUTPUT_PATHS={output_files}
+        export DIAGNOSTICS_DIRECTORY={self.diagnostics_dir}"""
         for input_slot_attrs in self.input_slots.values():
             # snakemake shell commands require wildcards to be prefaced with 'wildcards.'
             input_files = ",".join(input_slot_attrs["filepaths"]).replace(
@@ -200,8 +200,8 @@ rule:
         export {input_slot_attrs["env_var"]}={input_files}"""
         if self.requires_spark:
             shell_cmd += f"""
-        read -r DUMMY_CONTAINER_SPARK_MASTER_URL < {{input.master_url}}
-        export DUMMY_CONTAINER_SPARK_MASTER_URL"""
+        read -r SPARK_MASTER_URL < {{input.master_url}}
+        export  SPARK_MASTER_URL"""
         for var_name, var_value in self.envvars.items():
             shell_cmd += f"""
         export {var_name}={var_value}"""

easylink/step.py

@@ -1134,7 +1134,7 @@ class LoopStep(TemplatedStep):
     @property
     def config_key(self):
         """The pipeline specification key required for a ``LoopStep``."""
-        return "iterate"
+        return "iterations"
 
     @property
     def node_prefix(self):

easylink/steps/cascading/exclude_clustered.py

@@ -69,8 +69,8 @@ clustered_record_ids = set(dataset_df["Record ID"].unique()) & set(
 
 IDS_TO_REMOVE = pd.DataFrame({"Record ID": list(clustered_record_ids)})
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/exclude_none.py

@@ -69,8 +69,8 @@ clusters_df = load_file(clusters_filepath)
 
 IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/update_clusters_by_connected_components.py

@@ -50,8 +50,8 @@ if len(known_clusters_filepaths) == 0:
 known_clusters_filepath = known_clusters_filepaths[0]
 known_clusters_df = load_file(known_clusters_filepath)
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (clusters.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (clusters.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 Path(results_filepath).parent.mkdir(exist_ok=True, parents=True)
 
 new_clusters_df = load_file(new_clusters_filepath)

easylink/steps/default/default_clusters_to_links.py

@@ -80,8 +80,8 @@ if len(clusters_filepaths) == 0:
 
 clusters_filepath = clusters_filepaths[0]
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 clusters_df = load_file(clusters_filepath)
 links_df = clusters_to_links(clusters_df)

easylink/steps/default/default_determining_exclusions.py

@@ -74,8 +74,8 @@ if len(clusters_df) > 0:
 
 IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/default/default_removing_records.py

@@ -45,8 +45,8 @@ if dataset_path is None:
 
 # IDS_TO_REMOVE_FILE_PATH is a single filepath (Cloneable section)
 ids_filepath = os.environ["IDS_TO_REMOVE_FILE_PATH"]
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
-results_dir = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+# OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = Path(os.environ["OUTPUT_PATHS"])
 results_dir.mkdir(exist_ok=True, parents=True)
 
 dataset = load_file(dataset_path)

easylink/steps/default/default_schema_alignment.py

@@ -42,11 +42,12 @@ records = pd.concat(
     ignore_index=True,
     sort=False,
 )
+# TODO: check both datasets contain all the columns
 
 records = records.rename(columns={"Record ID": "Input Record ID"})
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single filepath
-output_path = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single filepath
+output_path = os.environ["OUTPUT_PATHS"]
 Path(output_path).parent.mkdir(exist_ok=True, parents=True)
 
 logging.info(f"Writing output to {output_path}")

easylink/steps/default/default_updating_clusters.py

@@ -54,8 +54,8 @@ if len(known_clusters_df) > 0:
         "Default implementation of updating_clusters passed a non-empty set of known clusters"
     )
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (clusters.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (clusters.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 Path(results_filepath).parent.mkdir(exist_ok=True, parents=True)
 
 clusters_df = load_file(new_clusters_filepath)

easylink/steps/dev/README.md

@@ -46,7 +46,7 @@ is `DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS`, but you can also specify *what* the
 You can (optionally) provide another input file at `/extra_implementation_specific_input_data/input*` (Parquet or CSV) or a different path passed as `DUMMY_CONTAINER_EXTRA_IMPLEMENTATION_SPECIFIC_INPUT_FILE_PATH`.
 This is meant to represent an input that is specific to a given implementation.
 
-Output is written to `/results/result.<ext>` or a different comma-separated list of paths passed as `DUMMY_CONTAINER_OUTPUT_PATHS`.
+Output is written to `/results/result.<ext>` or a different comma-separated list of paths passed as `OUTPUT_PATHS`.
 If `DUMMY_CONTAINER_OUTPUT_FILE_TYPE` is `csv` it will be in CSV format, otherwise it will be Parquet.
 
 The environment variable `DUMMY_CONTAINER_BROKEN` makes the container return data that does not meet the specification.

easylink/steps/dev/python_pandas/dummy_step.py

@@ -101,9 +101,9 @@ else:
     df.drop(columns=columns_to_drop, inplace=True)
 
 output_file_format = os.getenv("DUMMY_CONTAINER_OUTPUT_FILE_FORMAT", "parquet")
-output_file_paths = os.getenv(
-    "DUMMY_CONTAINER_OUTPUT_PATHS", f"/results/result.{output_file_format}"
-).split(",")
+output_file_paths = os.getenv("OUTPUT_PATHS", f"/results/result.{output_file_format}").split(
+    ","
+)
 
 diagnostics["num_output_files"] = len(output_file_paths)
 diagnostics["output_file_paths"] = output_file_paths
@@ -117,7 +117,7 @@ for output_file_path in output_file_paths:
     else:
         raise ValueError()
 
-diagnostics_dir = os.getenv("DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY", "/diagnostics")
+diagnostics_dir = os.getenv("DIAGNOSTICS_DIRECTORY", "/diagnostics")
 try:
     with open(f"{diagnostics_dir}/diagnostics.yaml", "w") as f:
         yaml.dump(diagnostics, f, default_flow_style=False)

easylink/steps/dev/python_pandas/python_pandas.def

@@ -1,3 +1,4 @@
+
 Bootstrap: docker
 From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
 
@@ -16,18 +17,6 @@ From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a72
 
 %environment
     export LC_ALL=C
-    export PYTHONPATH=/app:$PYTHONPATH
 
 %runscript
-    python /dummy_step.py "$@"
-
-%labels
-    Author Patrick Nast
-    Version v1.0
-    Description Python Pandas Implementation
-
-%startscript
-    # These directories should be bound when running the container
-    mkdir -p /results
-    mkdir -p /diagnostics
-    mkdir -p /input_data
+    python /dummy_step.py '$@'

easylink/steps/dev/python_pyspark/dummy_step.py

@@ -17,9 +17,7 @@ logging.basicConfig(
 pyspark_log = logging.getLogger("pyspark")
 pyspark_log.setLevel(logging.WARNING)
 
-spark = SparkSession.builder.master(
-    os.getenv("DUMMY_CONTAINER_SPARK_MASTER_URL")
-).getOrCreate()
+spark = SparkSession.builder.master(os.getenv("SPARK_MASTER_URL")).getOrCreate()
 
 
 def load_file(file_path, file_format=None):
@@ -115,9 +113,9 @@ else:
     df = df.drop(*columns_to_drop)
 
 output_file_format = os.getenv("DUMMY_CONTAINER_OUTPUT_FILE_FORMAT", "parquet")
-output_file_paths = os.getenv(
-    "DUMMY_CONTAINER_OUTPUT_PATHS", f"/results/result.{output_file_format}"
-).split(",")
+output_file_paths = os.getenv("OUTPUT_PATHS", f"/results/result.{output_file_format}").split(
+    ","
+)
 
 diagnostics["num_output_files"] = len(output_file_paths)
 diagnostics["output_file_paths"] = output_file_paths
@@ -132,7 +130,7 @@ for output_file_path in output_file_paths:
     else:
         raise ValueError()
 
-diagnostics_dir = os.getenv("DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY", "/diagnostics")
+diagnostics_dir = os.getenv("DIAGNOSTICS_DIRECTORY", "/diagnostics")
 try:
     with open(f"{diagnostics_dir}/diagnostics.yaml", "w") as f:
         yaml.dump(diagnostics, f, default_flow_style=False)

easylink/steps/dev/python_pyspark/python_pyspark.def

@@ -1,3 +1,4 @@
+
 Bootstrap: docker
 From: apache/spark-py@sha256:489f904a77f21134df4840de5f8bd9f110925e7b439ca6a04b7c033813edfebc
 
@@ -22,15 +23,4 @@ From: apache/spark-py@sha256:489f904a77f21134df4840de5f8bd9f110925e7b439ca6a04b7
 
 %runscript
     cd /workdir
-    python3 /code/dummy_step.py "$@"
-
-%labels
-    Author Patrick Nast
-    Version v1.0
-    Description Python Pyspark Implementation
-
-%startscript
-    # These directories should be bound when running the container
-    mkdir -p /results
-    mkdir -p /diagnostics
-    mkdir -p /input_data
+    python3 /code/dummy_step.py '$@'

easylink/steps/dev/r/dummy_step.R

@@ -106,7 +106,7 @@ if (broken) {
 }
 
 output_file_format <- Sys.getenv("DUMMY_CONTAINER_OUTPUT_FILE_FORMAT", "parquet")
-output_file_paths <- strsplit(Sys.getenv("DUMMY_CONTAINER_OUTPUT_PATHS", paste0("/results/result.", output_file_format)), ",")[[1]]
+output_file_paths <- strsplit(Sys.getenv("OUTPUT_PATHS", paste0("/results/result.", output_file_format)), ",")[[1]]
 
 diagnostics$num_output_files <- length(output_file_paths)
 diagnostics$output_file_paths <- output_file_paths
@@ -123,7 +123,7 @@ for (output_file_path in output_file_paths) {
     }
 }
 
-diagnostics_dir <- Sys.getenv("DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY", "/diagnostics")
+diagnostics_dir <- Sys.getenv("DIAGNOSTICS_DIRECTORY", "/diagnostics")
 if (dir.exists(diagnostics_dir) && file.access(diagnostics_dir, mode = 2) == 0) {
     write_yaml(diagnostics, file.path(diagnostics_dir, 'diagnostics.yaml'))
 }

easylink/steps/dev/r/r-image.def

@@ -1,3 +1,4 @@
+
 Bootstrap: docker
 From: rocker/tidyverse@sha256:6a7c913590e758b5fe2ad9921ccc5df7c7160e5de1db5f353630fe8e0ee2f876
 
@@ -15,15 +16,4 @@ From: rocker/tidyverse@sha256:6a7c913590e758b5fe2ad9921ccc5df7c7160e5de1db5f3536
     export LC_ALL=C
 
 %runscript
-    Rscript /dummy_step.R "$@"
-
-%labels
-    Author Patrick Nast
-    Version v1.0
-    Description R Implementation
-
-%startscript
-    # These directories should be bound when running the container
-    mkdir -p /results
-    mkdir -p /diagnostics
-    mkdir -p /input_data
+    Rscript /dummy_step.R '$@'

easylink/steps/example/middle_name_to_initial.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./middle_name_to_initial.py /middle_name_to_initial.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /middle_name_to_initial.py '$@'

easylink/steps/example/middle_name_to_initial.py

@@ -0,0 +1,60 @@
+# STEP_NAME: pre-processing
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# DATASET_DIR_PATHS is list of directories, each containing one file
+dataset_paths = os.environ["DATASET_DIR_PATHS"].split(",")
+logging.info(f"{dataset_paths=}")
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+logging.info(f"splitter_choice={splitter_choice}")
+dataset_path = None
+for path in dataset_paths:
+    path = Path(path)
+    # NOTE: We iterate the dir here, but it should only have one non-hidden
+    # file in it. We don't validate that here as it is checked in the validator.
+    for path_to_check in path.iterdir():
+        if path_to_check.stem == splitter_choice:
+            dataset_path = str(path_to_check)
+            break
+
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = Path(os.environ["OUTPUT_PATHS"])
+results_dir.mkdir(exist_ok=True, parents=True)
+
+output_path = results_dir / Path(dataset_path).name
+
+dataset = load_file(dataset_path)
+
+# add middle initial column from middle name
+dataset["middle_initial"] = dataset["middle_name"].str[0]
+
+logging.info(f"Writing output for dataset from input {dataset_path} to {output_path}")
+dataset.to_parquet(output_path)

easylink/steps/fastLink/fastLink_evaluating_pairs.R

@@ -10,8 +10,8 @@ library(stringr)
 # Check required environment variables
 required_env_vars <- c(
   "BLOCKS_DIR_PATH",
-  "DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY",
-  "DUMMY_CONTAINER_OUTPUT_PATHS",
+  "DIAGNOSTICS_DIRECTORY",
+  "OUTPUT_PATHS",
   "COMPARISONS",
   "THRESHOLD_MATCH_PROBABILITY"
 )
@@ -24,8 +24,8 @@ if (length(missing_vars) > 0) {
 }
 
 blocks_dir <- Sys.getenv("BLOCKS_DIR_PATH")
-diagnostics_dir <- Sys.getenv("DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY")
-output_path <- Sys.getenv("DUMMY_CONTAINER_OUTPUT_PATHS")
+diagnostics_dir <- Sys.getenv("DIAGNOSTICS_DIRECTORY")
+output_path <- Sys.getenv("OUTPUT_PATHS")
 comparisons <- strsplit(Sys.getenv("COMPARISONS"), ",")[[1]]
 
 all_predictions <- list()

easylink/steps/fastLink/fastLink_links_to_clusters.R

@@ -12,7 +12,7 @@ library(stringr)
 # Check required environment variables
 required_env_vars <- c(
     "LINKS_FILE_PATH",
-    "DUMMY_CONTAINER_OUTPUT_PATHS",
+    "OUTPUT_PATHS",
     "THRESHOLD_MATCH_PROBABILITY"
 )
 missing_vars <- required_env_vars[!nzchar(Sys.getenv(required_env_vars))]
@@ -24,7 +24,7 @@ if (length(missing_vars) > 0) {
 }
 
 links_file_path <- Sys.getenv("LINKS_FILE_PATH")
-output_path <- Sys.getenv("DUMMY_CONTAINER_OUTPUT_PATHS")
+output_path <- Sys.getenv("OUTPUT_PATHS")
 
 if (!file.exists(links_file_path)) {
     stop(sprintf("File not found: %s", links_file_path))

easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py

@@ -11,7 +11,7 @@ data = pd.read_parquet(os.environ["STEP_1_MAIN_INPUT_FILE_PATHS"])
 
 print(data)
 
-dir_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+dir_path = Path(os.environ["OUTPUT_PATHS"])
 dir_path.mkdir(parents=True, exist_ok=True)
 
 for i in range(3):

easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py

@@ -8,7 +8,7 @@ from pathlib import Path
 
 import pandas as pd
 
-dir_path = Path(os.environ["DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH"])
+dir_path = Path(os.environ["MAIN_INPUT_DIR_PATH"])
 saved = False
 
 for i, f in enumerate([f for f in dir_path.iterdir() if f.is_file()]):
@@ -16,7 +16,7 @@ for i, f in enumerate([f for f in dir_path.iterdir() if f.is_file()]):
         continue
 
     if not saved:
-        shutil.copy(f, os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+        shutil.copy(f, os.environ["OUTPUT_PATHS"])
         saved = True
 
     print(pd.read_parquet(f))

easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.py

@@ -33,8 +33,8 @@ def load_file(file_path, file_format=None):
 
 # CLUSTERS_FILE_PATH is a path to a single file
 clusters_path = os.environ["CLUSTERS_FILE_PATH"]
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 clusters_df = load_file(clusters_path)
 

easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.py

@@ -45,8 +45,8 @@ for path in dataset_paths:
 if dataset_path is None:
     raise ValueError(f"No dataset matching {splitter_choice} found")
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
-results_dir = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+# OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = Path(os.environ["OUTPUT_PATHS"])
 results_dir.mkdir(exist_ok=True, parents=True)
 
 output_path = results_dir / Path(dataset_path).name

easylink/steps/splink/splink_blocking_and_filtering.def

@@ -13,7 +13,7 @@ From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a72
     mkdir -p /diagnostics
 
     # Install Python packages with specific versions
-    pip install pandas pyarrow splink==4.0.7
+    pip install pandas pyarrow splink==4.0.7 vl-convert-python
 
 %environment
     export LC_ALL=C

easylink/steps/splink/splink_blocking_and_filtering.py

@@ -1,5 +1,5 @@
 # STEP_NAME: blocking_and_filtering
-# REQUIREMENTS: pandas pyarrow splink==4.0.7
+# REQUIREMENTS: pandas pyarrow splink==4.0.7 vl-convert-python
 
 import os
 
@@ -7,22 +7,27 @@ import pandas as pd
 
 records = pd.read_parquet(os.environ["RECORDS_FILE_PATH"])
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
-results_dir = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = os.environ["OUTPUT_PATHS"]
 
 import splink
 
 blocking_rules = os.environ["BLOCKING_RULES"].split(",")
 
-from splink import Linker, SettingsCreator
+link_only = os.getenv("LINK_ONLY", "false").lower() in ("true", "yes", "1")
+
+from splink import DuckDBAPI, Linker, SettingsCreator
 
 # Create the Splink linker in dedupe mode
 settings = SettingsCreator(
-    link_type="link_and_dedupe",
+    link_type="link_only" if link_only else "link_and_dedupe",
     blocking_rules_to_generate_predictions=blocking_rules,
     comparisons=[],
 )
 from splink import DuckDBAPI
+from splink.blocking_analysis import (
+    cumulative_comparisons_to_be_scored_from_blocking_rules_chart,
+)
 
 grouped = records.rename(columns={"Input Record ID": "unique_id"}).groupby(
     "Input Record Dataset"
@@ -52,6 +57,7 @@ blocking_input_tablename_r = "__splink__df_concat_with_tf"
 
 link_type = linker._settings_obj._link_type
 
+
 # If exploded blocking rules exist, we need to materialise
 # the tables of ID pairs
 from splink.internals.blocking import materialise_exploded_id_tables
@@ -98,7 +104,12 @@ blocked_pairs[["Left Record ID", "Right Record ID"]] = blocked_pairs[
 wrong_order_dataset = (
     blocked_pairs["Left Record Dataset"] > blocked_pairs["Right Record Dataset"]
 )
-id_cols = ["Left Record Dataset", "Left Record ID", "Right Record Dataset", "Right Record ID"]
+id_cols = [
+    "Left Record Dataset",
+    "Left Record ID",
+    "Right Record Dataset",
+    "Right Record ID",
+]
 switched_id_cols = [
     "Right Record Dataset",
     "Right Record ID",
@@ -128,3 +139,18 @@ output_path.mkdir(exist_ok=True, parents=True)
 
 records.to_parquet(output_path / "records.parquet", index=False)
 blocked_pairs.to_parquet(output_path / "pairs.parquet", index=False)
+
+records["unique_id"] = (
+    str(records["Input Record Dataset"]) + "_" + str(records["Input Record ID"])
+)
+db_api = DuckDBAPI()
+diagnostics_dir = Path(os.environ["DIAGNOSTICS_DIRECTORY"])
+chart_path = diagnostics_dir / f"blocking_cumulative_comparisons_chart_block_0.png"
+cumulative_comparisons_to_be_scored_from_blocking_rules_chart(
+    table_or_tables=records,
+    blocking_rules=blocking_rules,
+    db_api=db_api,
+    link_type=link_type,
+    unique_id_column_name="unique_id",
+    source_dataset_column_name="Input Record Dataset",
+).save(chart_path)

easylink/steps/splink/splink_evaluating_pairs.py

@@ -10,9 +10,10 @@ import splink.comparison_library as cl
 from splink import Linker, SettingsCreator
 
 blocks_dir = Path(os.environ["BLOCKS_DIR_PATH"])
-diagnostics_dir = Path(os.environ["DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY"])
-output_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+diagnostics_dir = Path(os.environ["DIAGNOSTICS_DIRECTORY"])
+output_path = Path(os.environ["OUTPUT_PATHS"])
 Path(output_path).parent.mkdir(exist_ok=True, parents=True)
+link_only = os.getenv("LINK_ONLY", "false").lower() in ("true", "yes", "1")
 
 all_predictions = []
 
@@ -30,17 +31,20 @@ for block_dir in blocks_dir.iterdir():
             comparisons.append(cl.NameComparison(column))
         elif method == "dob":
             comparisons.append(cl.DateOfBirthComparison(column))
+        elif method == "levenshtein":
+            comparisons.append(cl.LevenshteinAtThresholds(column))
         else:
             raise ValueError(f"Unknown comparison method {method}")
 
     # Create the Splink linker in dedupe mode
     settings = SettingsCreator(
-        link_type="link_and_dedupe",
+        link_type="link_only" if link_only else "link_and_dedupe",
         blocking_rules_to_generate_predictions=[],
         comparisons=comparisons,
         probability_two_random_records_match=float(
             os.environ["PROBABILITY_TWO_RANDOM_RECORDS_MATCH"]
         ),
+        retain_intermediate_calculation_columns=True,
     )
 
     grouped = (
@@ -59,7 +63,7 @@ for block_dir in blocks_dir.iterdir():
         input_table_aliases=[name for name, _ in grouped],
     )
 
-    linker.training.estimate_u_using_random_sampling(max_pairs=5e6)
+    linker.training.estimate_u_using_random_sampling(max_pairs=5e6, seed=1234)
 
     blocking_rules_for_training = os.environ["BLOCKING_RULES_FOR_TRAINING"].split(",")
 
@@ -143,6 +147,12 @@ for block_dir in blocks_dir.iterdir():
 
     all_predictions.append(predictions.as_pandas_dataframe())
 
+comparisons_path = diagnostics_dir / f"comparisons_chart_{block_dir}.html"
+comparisons_path.parent.mkdir(exist_ok=True, parents=True)
+linker.visualisations.comparison_viewer_dashboard(
+    predictions, comparisons_path, overwrite=True
+)
+
 all_predictions = pd.concat(all_predictions, ignore_index=True)[
     [
         "source_dataset_l",

easylink/steps/splink/splink_links_to_clusters.py

@@ -32,7 +32,7 @@ dummy_records_df = pd.DataFrame(
         )
     }
 )
-output_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+output_path = Path(os.environ["OUTPUT_PATHS"])
 
 db_api = DuckDBAPI()
 

{easylink-0.1.22.dist-info → easylink-0.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.22
+Version: 0.1.23
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.22.dist-info → easylink-0.1.23.dist-info}/RECORD

@@ -1,39 +1,39 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=zmP2TRnzKPjZJ1eiBcT-cRInsji6FW-OVD3FafQFCc4,23
+easylink/_version.py,sha256=0byemO6n6WCv41u9vBG2AIsOkVbxLvok7puvwy8EhfU,23
 easylink/cli.py,sha256=zQO4lOVoZ3eVgPVWT2sCF4zNoKgiDJP9ReTh2Myr9jc,10307
 easylink/configuration.py,sha256=TPd3WbqUcJMJDPJuHeo5ZebvZPQrRyfm8-laad2sOFk,12681
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
 easylink/implementation.py,sha256=lSF37g-aQYgdLI0lgFaDrBkp23te9O9B1V-CmmRtB-0,14514
-easylink/implementation_metadata.yaml,sha256=GoU_aWjVryG8-xjUHkC2nCUeznmYD0BwfJYnNrpZ8P4,10670
+easylink/implementation_metadata.yaml,sha256=pKu_H9fLnTsS8E4wCnYRitumW1-zs7mfE3z66BAyO30,10848
 easylink/pipeline.py,sha256=NJyMInbOCjJ_5kRzzuK5AcupvC7ecd5qLOC-f1Gy3Ss,17701
 easylink/pipeline_graph.py,sha256=jtjS7_2IVa189U8cL621600erC_0pa6BKPRRO8VqwiU,23760
 easylink/pipeline_schema.py,sha256=sj6YQqMFUS35RRFWdlrhPlud68d3Rb5y7iKfAEZy1qs,6877
-easylink/rule.py,sha256=QJPmrvQUZPnqGFD9UmMK8imdJ7VODzGlUOSnpJhb9AU,16677
+easylink/rule.py,sha256=MM7WyW56J7zT2FVjHlFtjuz62PfdSBGTD3MNcpLfEZM,16598
 easylink/runner.py,sha256=Z9GKgiUAWtp0dW1cyAp86MGthIDeABJtHDXUtzv0-kE,6714
-easylink/step.py,sha256=zQAoz4HlSVvgS7iMlfmCrXluOtPQxbSgPZOeyZwjdpo,91085
+easylink/step.py,sha256=-vdFhPvwAZ3d69LMQGmSIVdcMG8E8ZtSvTE0UWif7zs,91088
 easylink/devtools/implementation_creator.py,sha256=gZZpfpiOOh912nNjR_d5wR0mBO5c09jWMS0FSYRcE1o,19120
 easylink/pipeline_schema_constants/__init__.py,sha256=njUL2v67WFC3dW_FjOXWRLSeOhInox_ZINLEji0_7V8,1523
 easylink/pipeline_schema_constants/development.py,sha256=0x6lWqBmP1K9AebEmeZ4veSnLBcdQcZXzbV6lCU11bc,12622
 easylink/pipeline_schema_constants/main.py,sha256=kcAhdbK_BhS79LUMhKkn-Uttl2vA-MHVX4M1StTloJQ,22934
-easylink/pipeline_schema_constants/testing.py,sha256=ZFD19CpcidZPVUYBvh8LAa5sZEERT2yfoFa-3xmskFs,24595
+easylink/pipeline_schema_constants/testing.py,sha256=ti08DeUuF-eWrGKMj4BMyOFFJnEYooDaWX0DGiferbk,24579
 easylink/steps/cascading/exclude_clustered.def,sha256=GfoDqO2Vtsh7VI8SwGaJtv_KtKjs-UmBcivqQ7OPkjk,503
-easylink/steps/cascading/exclude_clustered.py,sha256=NSA6GZBzGa7e6CH4tacCGfr0Y9sUM29g9Nf8NquHB44,2612
+easylink/steps/cascading/exclude_clustered.py,sha256=Bpsyf9vAZ431Fh96RVzHkF7fy77NQjo1Cl6bHCIy69c,2580
 easylink/steps/cascading/exclude_none.def,sha256=iFUhUMocxtkA0NErkjVrBxY0MUdS3DIPNsbCpTJRP0k,488
-easylink/steps/cascading/exclude_none.py,sha256=KntBX3q-V47d96ztOlPNRY_kCFJNi1LNYQ7UNs5wB4c,2507
+easylink/steps/cascading/exclude_none.py,sha256=5DK5bNG4TneMwUKE49Kmz7VDnKBNZWjOERkuSJU3BmA,2475
 easylink/steps/cascading/update_clusters_by_connected_components.def,sha256=sAAAWOod8EuAnotR1cayaGAvs7x6xoMVlwmLso_a9Cc,578
-easylink/steps/cascading/update_clusters_by_connected_components.py,sha256=43D5GBmPXSgxcjgbJTvEoGFvPzBCGqYgBaT42pncNNw,3661
+easylink/steps/cascading/update_clusters_by_connected_components.py,sha256=sFZXMGXl17jcGt8Fu5hgQz1KW5bFvPYdCoQGZ9Erc0I,3629
 easylink/steps/default/default_clusters_to_links.def,sha256=9PjUygLvsoYMUZDznceuuv55t8fPs473P57J_RMl3U0,527
-easylink/steps/default/default_clusters_to_links.py,sha256=EIYeP0lj0plBl2OpTRuv3iDEQl-zNVJONUg0kgKSEF0,2848
+easylink/steps/default/default_clusters_to_links.py,sha256=Ckm53d3W-ETNlTvQJPOkpHmSqCmxSWknMPQrEAIoTBY,2816
 easylink/steps/default/default_determining_exclusions.def,sha256=zZUEHDdrpLxzx3gTm-dki2ge5ivCCg4ziIwTErqCII0,542
-easylink/steps/default/default_determining_exclusions.py,sha256=tF2lcga-6n99shgYEmhpNuqok33u7dcW9q5wV3xgp5w,2661
+easylink/steps/default/default_determining_exclusions.py,sha256=4diLfuqYm_Koj7gwifjwe_7mLZ6xb6RQiEdk-RRtB94,2629
 easylink/steps/default/default_removing_records.def,sha256=QqacmOu6myxFSULHRKeKsVD8l73KDm4VEkPkPlovwqs,524
-easylink/steps/default/default_removing_records.py,sha256=LIlFS8EvJ6h5XqEfgWZYyIAjcKj7Oo8_I5a-vXHOozs,1938
+easylink/steps/default/default_removing_records.py,sha256=P4mmX2D4mhSoWd_S5CaNT4hlHOMAeZiMhCScWQiR_fQ,1906
 easylink/steps/default/default_schema_alignment.def,sha256=hFHJkos0Fhe4LvpjLOCd6klIaIqOKqECDDSTVu3G03Y,524
-easylink/steps/default/default_schema_alignment.py,sha256=Uxi6uTFveFKSiiRZG9MnTXOklQngSKGMafqnvKDc0rY,1459
+easylink/steps/default/default_schema_alignment.py,sha256=oT5jbYQ3C3ocLgqqOnvH0SIJ6NeTtPBWWmCqr_frnAI,1479
 easylink/steps/default/default_updating_clusters.def,sha256=vDzSkTknDfeiXeHREpw4BkUxFcTWamxr81c3rZ7_enY,527
-easylink/steps/default/default_updating_clusters.py,sha256=A-lO3ussM1Ntffp-ZyPQGbbxZg4QNiZ8AvSOGVJDXnA,2139
-easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
+easylink/steps/default/default_updating_clusters.py,sha256=uwblSM9w4PoqPgWA_5cwmnU7ARSruickWxTfr2EZCM0,2107
+easylink/steps/dev/README.md,sha256=lEHtM48SkFTV1FL-B5bbrEGjSVa_mb690Ed08nnheww,4533
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
 easylink/steps/dev/build-containers-remote.sh,sha256=Hy-kaaXf-ta6n8SzOz_ahByjMY5T7J71MvzXRXDvQw8,271
 easylink/steps/dev/test.py,sha256=4iudKBD6CFz2CxbjSBUkc8LCWlMl-Nmw_rB35ZN6TrQ,6835
@@ -43,36 +43,38 @@ easylink/steps/dev/input_data/input_file_1.parquet,sha256=Km8jRyfGNdq0MFdz_-bewl
 easylink/steps/dev/input_data/input_file_2.csv,sha256=YqKLZDC4d-aYN8Dh9OB6iQWWUKmvueu5CszckH1AApU,100016
 easylink/steps/dev/input_data/input_file_2.parquet,sha256=Vpo0sUqQ78qlWLRk8p303Nh89BVcK4uvXJljRGHmsWk,60392
 easylink/steps/dev/python_pandas/README.md,sha256=c_FbtkKKOTjt2R_LfHUo5lBga1qHiYkxLdQeewRr45g,977
-easylink/steps/dev/python_pandas/dummy_step.py,sha256=NvhLUZu40B3Xbj_S-chQ6IkYUPr6X2aGBxYUa3DqwmY,4362
-easylink/steps/dev/python_pandas/python_pandas.def,sha256=24cxwGF8Cqkv2a1zVsu94MfC_bAXBqAINLwfW2zyB_0,769
+easylink/steps/dev/python_pandas/dummy_step.py,sha256=ASZIxk_d46zNz0xUtZ37OuTJUpzfi98EwQduZXDiwK0,4330
+easylink/steps/dev/python_pandas/python_pandas.def,sha256=umAUJL3RQZsCLIvFvsaif_-kCGfWzsnIH8d6Zwx_qYQ,482
 easylink/steps/dev/python_pyspark/README.md,sha256=di29SAfcdTTpar7gdoJRLqKrL8DEfNeayYUyaywdhUg,1563
-easylink/steps/dev/python_pyspark/dummy_step.py,sha256=wxHHI3Uv8MTipKG2ffHbT_eL4JkoNpx49bJoErXumdc,5003
-easylink/steps/dev/python_pyspark/python_pyspark.def,sha256=j_RmVjspmXGOhJTr10ED13RYfbimgxRU3WVTL7VOIUQ,915
+easylink/steps/dev/python_pyspark/dummy_step.py,sha256=6BJi-L6tlXGc6GJnGTI06AnzuRYh3KYBgU09tvId954,4949
+easylink/steps/dev/python_pyspark/python_pyspark.def,sha256=e-OXax5t96OEgAJdsKDqjGXPuCk12GpPYbwIdrUpBOU,666
 easylink/steps/dev/r/README.md,sha256=dPjZdDTqcJsZCiwhddzlOj1ob0P7YocZUNFrLIGM1-0,1201
-easylink/steps/dev/r/dummy_step.R,sha256=1TWZY8CEkT6gavrulBxFsKbDSKJJjk0NtJrGH7TIikE,4975
-easylink/steps/dev/r/r-image.def,sha256=LrhXlt0C3k7d_VJWopRPEVARnFWSuq_oILlwo7g03bE,627
-easylink/steps/fastLink/fastLink_evaluating_pairs.R,sha256=fQRrTPrgb1t5hrQi0V5H55J-PHdWjsATrVRYdXNYtdU,4603
+easylink/steps/dev/r/dummy_step.R,sha256=4eFZgmKaagydjYOVnrOB3W2vjHHcGDhJ8LXsWDrDNyI,4943
+easylink/steps/dev/r/r-image.def,sha256=gBNCfMb_HtgrGPhTt8qEws5tWVfjQIS_GCIHRGAHG9c,391
+easylink/steps/example/middle_name_to_initial.def,sha256=UmD3FCuK8CMD0gQRUqg1BFnGq5Mucu7x8eU19jq7pZ0,518
+easylink/steps/example/middle_name_to_initial.py,sha256=1Q7xaXIxkIvPN6jW98WKTvVhWB9qMC23mRIoO7NYRa8,1901
+easylink/steps/fastLink/fastLink_evaluating_pairs.R,sha256=ucbHibtoYJ4-GDg1mWv-dtv0r_1XomhdT-KC3Zkat2E,4539
 easylink/steps/fastLink/fastLink_evaluating_pairs.def,sha256=5rDi-cmWhyuFEsiGFPpTxtySMqq5TpgJG-y8g_MtEvA,509
-easylink/steps/fastLink/fastLink_links_to_clusters.R,sha256=exVzJl4r7k7cRlMCHSmigOqTlxShqzK-FO3EDhlPksg,4087
+easylink/steps/fastLink/fastLink_links_to_clusters.R,sha256=iM6bi27bHNJRxfOIBOCvZcFUuKT3VauOWbM6d0Ws5dk,4055
 easylink/steps/fastLink/fastLink_links_to_clusters.def,sha256=1xYjOMsHtSS2-AI4EC2r6kL8ZX5F2JhmvESefEKeJVY,512
 easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def,sha256=CkQVG-uDRQ9spAavdkZbhx2GD_fRsKZGELPrr8yltsc,550
-easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py,sha256=dI0OWugE35ABLcSwsI-T3C4dvuPTKXwjE52dtSsCo8Y,428
+easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py,sha256=sBmF-wMgTpcqeM9gVWbHZkcy_w0OzfDYo98-3P2WMaM,412
 easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def,sha256=9gShg1EDJEHZcz7Z5VfZ1A4Gpm9XQes8ezn6rAZDgDM,550
-easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py,sha256=DMJW5TXjhELxhY4U9q2RpLjqxlS1YSosTGL2AfRnaZM,521
+easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py,sha256=ExFljptolMiidU7LiOfQtH13ChbDUGIF3r5qM5paKsA,489
 easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.def,sha256=YOWtJZxoe-kHFeEyrgGcVGfdqcbD_Fg17A9shOaK-yc,584
-easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.py,sha256=skZUiZWcSXAOqq8TAlN5I0wztXgCWHQYA_xkuiL5s28,1202
+easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.py,sha256=zctz6LktX1BsVbeSR2gWuNCzRU7WkmWDGg68wlzZw0Q,1170
 easylink/steps/rl-dummy/input_data/create_input_files.ipynb,sha256=uXvJ8zTTPg0077HgA7MhQ_9jD-aeISFLeMeEBbSnOu8,54498
 easylink/steps/rl-dummy/input_data/input_file_1.parquet,sha256=GQ_7v7ucwdJn-9mTgKVcvqkJ5gTkwb0B7y38mfOYbic,15200
 easylink/steps/rl-dummy/input_data/input_file_2.parquet,sha256=Y4eseBm0HmFroksQr_VApgozRL8h8u7nQO6x_Utyns8,14902
 easylink/steps/rl-dummy/input_data/known_clusters.parquet,sha256=Ysodu65toHZN4AgjVJsm0ueUxPIZAJjbtRm9SVM08JE,2598
 easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.def,sha256=HeUSv2QvMOQzsyVktYR1xYoEqwiNpDo-p7IRcGSMspE,512
-easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.py,sha256=aeDgn9z2um0oTPNSwPcTkBou3-1ajud_MWhkuRoHdOU,1884
-easylink/steps/splink/splink_blocking_and_filtering.def,sha256=foAQAPvhDEkXkevpghS-uftsTbIQnQy9PvTkyldQeAA,539
-easylink/steps/splink/splink_blocking_and_filtering.py,sha256=8-_a9PkOmKSa-8TJ9YMjqI7gLo-YD9JCAO1f8uBhdoE,4469
+easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.py,sha256=I6kqG4e_H2yFW5MpsMXdpoY_NjHcBvVVAHWv89LUgXE,1852
+easylink/steps/splink/splink_blocking_and_filtering.def,sha256=umWzxJhsfdi8g3TD-r2mKpjC-FPAMDk6-IERiWigdQc,557
+easylink/steps/splink/splink_blocking_and_filtering.py,sha256=FO8YJ2_KgCLpQoq5xsM339bTSN1DhCXCL8XT1pb5STY,5259
 easylink/steps/splink/splink_evaluating_pairs.def,sha256=DN3Ohy9qJOAyK58v164neP23HDVYpedMqzCu4eQh4Hg,521
-easylink/steps/splink/splink_evaluating_pairs.py,sha256=JR2qVgb14cNZKozDyOrN11nr1mXOwWv69E6WP0pRlMw,5713
+easylink/steps/splink/splink_evaluating_pairs.py,sha256=m-j1QMRSvPCiSoWVSV1kzzzsK1c_xG8nqYKMd3cj7kM,6195
 easylink/steps/splink/splink_links_to_clusters.def,sha256=RurvOYyGjNs9tx64DTXwI-GSgHD4T7SzDfhAH18pTEM,524
-easylink/steps/splink/splink_links_to_clusters.py,sha256=z5ymdYl9ytp1e5MA6vn8wpGRFWVuhh23LqGq8NJJxZQ,1936
+easylink/steps/splink/splink_links_to_clusters.py,sha256=5Sw8yi0dVLuRB-trN2kXmxbHBR0VJBxYee6u4_usg2Y,1920
 easylink/utilities/__init__.py,sha256=0U33kbv4hoMfFQ_lh5hLwifxRPzOgkLkjKLYxmaK10g,196
 easylink/utilities/aggregator_utils.py,sha256=_DAHRAf9janbDsuj_jnAn5Dzz2s4R5Ni3YeytDpN9UE,954
 easylink/utilities/data_utils.py,sha256=XPRjq3qW_fN0xQ23Jms_xBzpTHbRwqZWDP1AW0nYkP0,6926
@@ -81,9 +83,9 @@ easylink/utilities/paths.py,sha256=9inDKMPorAaWh5vhjc3j1Tj_aXVKhLdodiJO9H1nNes,9
 easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=KXiVUYJ9TGxCQmrwos18pB1sxG_0Ay67qoDJT6vem2o,3828
 easylink/utilities/validation_utils.py,sha256=DBJB2TLVklgYw1WaaPse9vqtwPLMGmZNYM2cbCZsoHI,18417
-easylink-0.1.22.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
-easylink-0.1.22.dist-info/METADATA,sha256=hei9KKa0HUgy1Z4aU-nPEAs8KF2_TEe7J0-_esdCG40,3565
-easylink-0.1.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-easylink-0.1.22.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.22.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.22.dist-info/RECORD,,
+easylink-0.1.23.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
+easylink-0.1.23.dist-info/METADATA,sha256=u-oRyBse4M0AsFkMjTuy0JCpul-BwHJ1JaD9fIALrHU,3565
+easylink-0.1.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+easylink-0.1.23.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.23.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.23.dist-info/RECORD,,