easylink 0.1.19__py3-none-any.whl → 0.1.20__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.19"
+__version__ = "0.1.20"

easylink/devtools/implementation_creator.py

@@ -29,7 +29,7 @@ from easylink.step import (
     TemplatedStep,
 )
 from easylink.utilities.data_utils import load_yaml
-from easylink.utilities.paths import IMPLEMENTATION_METADATA
+from easylink.utilities.paths import DEV_IMAGES_DIR, IMPLEMENTATION_METADATA
 
 
 def main(script_path: Path, host: Path) -> None:
@@ -195,9 +195,24 @@ class ImplementationCreator:
                 f"Implementation '{self.implementation_name}' already exists in the registry. "
                 "Overwriting it with the latest data."
             )
+
+        # Handle the fact that developers might be saving to username subdirs
+        # If the host folder is a subdirectory of DEV_IMAGES_DIR (e.g., the default
+        # host directory when calling `easylink devtools create-implementation`
+        # is DEV_IMAGES_DIR/<username>), we want to include the relative path
+        # to the DEV_IMAGES_DIR in the image name. This is required because ultimately
+        # when running a pipeline, all images are expected to be in a single directory.
+        image_name = (
+            self.hosted_container_path.name
+            # Use just the image name if the hosted path is not a part of DEV_IMAGES_DIR
+            if not self.hosted_container_path.is_relative_to(DEV_IMAGES_DIR)
+            # Use the path relative to DEV_IMAGES_DIR as the image name
+            else str(self.hosted_container_path.relative_to(DEV_IMAGES_DIR))
+        )
+
         info[self.implementation_name] = {
             "steps": [self.step],
-            "image_path": str(self.hosted_container_path),
+            "image_name": str(image_name),
             "script_cmd": f"{self.script_base_command} /{self.script_path.name}",
             "outputs": {
                 self.output_slot: "result.parquet",

easylink/steps/cascading/update_clusters_by_connected_components.py

@@ -60,12 +60,14 @@ new_clusters_df = load_file(new_clusters_filepath)
 def merge_clusters(known_clusters_df, new_clusters_df):
     # Combine both dataframes
     combined_df = pd.concat([known_clusters_df, new_clusters_df], ignore_index=True)
-
-    # Drop records with missing cluster IDs
-    combined_df = combined_df.dropna(subset=["Cluster ID"])
+    combined_df["Input Record Key"] = (
+        combined_df["Input Record Dataset"]
+        + "-__-"
+        + combined_df["Input Record ID"].astype(int).astype(str)
+    )
 
     # Group by Cluster ID to get connected records
-    cluster_groups = combined_df.groupby("Cluster ID")["Input Record ID"].apply(list)
+    cluster_groups = combined_df.groupby("Cluster ID")["Input Record Key"].apply(list)
 
     # Build a graph of all connections implied by cluster IDs
     G = nx.Graph()
@@ -75,8 +77,8 @@ def merge_clusters(known_clusters_df, new_clusters_df):
                 G.add_edge(group[i], group[j])
 
     # Add isolated nodes (records with unique clusters)
-    all_ids = set(combined_df["Input Record ID"])
-    G.add_nodes_from(all_ids)
+    all_keys = set(combined_df["Input Record Key"])
+    G.add_nodes_from(all_keys)
 
     # Compute connected components
     components = list(nx.connected_components(G))
@@ -84,13 +86,19 @@ def merge_clusters(known_clusters_df, new_clusters_df):
     # Assign new cluster IDs
     merged_data = []
     for cluster_id, records in enumerate(components, start=1):
-        for record_id in records:
-            merged_data.append((record_id, cluster_id))
+        for record_key in records:
+            merged_data.append((record_key, cluster_id))
 
     # Build the final DataFrame
-    merged_df = pd.DataFrame(merged_data, columns=["Input Record ID", "Cluster ID"])
+    merged_df = pd.DataFrame(merged_data, columns=["Input Record Key", "Cluster ID"])
+
+    merged_df[["Input Record Dataset", "Input Record ID"]] = merged_df[
+        "Input Record Key"
+    ].str.split("-__-", n=1, expand=True)
+
+    merged_df["Input Record ID"] = merged_df["Input Record ID"].astype(int)
 
-    return merged_df
+    return merged_df[["Input Record Dataset", "Input Record ID", "Cluster ID"]]
 
 
 output_df = merge_clusters(known_clusters_df, new_clusters_df)

{easylink-0.1.19.dist-info → easylink-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.19
+Version: 0.1.20
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.19.dist-info → easylink-0.1.20.dist-info}/RECORD

@@ -1,6 +1,6 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=cAJAbAh288a9AL-3yxwFzEM1L26izSJ6wma5aiml_9Y,23
+easylink/_version.py,sha256=8XalsVoLEfXslFvdtUEmkNOuYShzOzYOcFbgmOz1oSk,23
 easylink/cli.py,sha256=zQO4lOVoZ3eVgPVWT2sCF4zNoKgiDJP9ReTh2Myr9jc,10307
 easylink/configuration.py,sha256=hgmG5SIbYqnHDHfk44Gr3QX7C3yTaEVW6GuKeMqvu6c,12689
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
@@ -12,7 +12,7 @@ easylink/pipeline_schema.py,sha256=FieJBa3rKgaCIB9QDuQEfWJ9joNBUUp6iHT6xmns-Vk,6
 easylink/rule.py,sha256=NusEUtBxx18L7UCcgDi3KKooFxSUgyS4eisVM5aPqFE,16770
 easylink/runner.py,sha256=Z9GKgiUAWtp0dW1cyAp86MGthIDeABJtHDXUtzv0-kE,6714
 easylink/step.py,sha256=NGy1KNqM4eXP7kP0kdfcfyGc4K_ExSCSidCdW3h0Qg8,89902
-easylink/devtools/implementation_creator.py,sha256=1WQOOrjQYOhjjp8MQM9j1xoeAp-SW51A1f1oW4G792I,18251
+easylink/devtools/implementation_creator.py,sha256=RkwnI1T0aEquRPgGjPOGtJo_87tjoKvDAElRcf6Vqqk,19140
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
 easylink/pipeline_schema_constants/__init__.py,sha256=xYymSjTeH3prvQL_rgGFVrriohANFtW_cy0vDwlF3ds,1355
@@ -24,7 +24,7 @@ easylink/steps/cascading/exclude_clustered.py,sha256=NSA6GZBzGa7e6CH4tacCGfr0Y9s
 easylink/steps/cascading/exclude_none.def,sha256=iFUhUMocxtkA0NErkjVrBxY0MUdS3DIPNsbCpTJRP0k,488
 easylink/steps/cascading/exclude_none.py,sha256=KntBX3q-V47d96ztOlPNRY_kCFJNi1LNYQ7UNs5wB4c,2507
 easylink/steps/cascading/update_clusters_by_connected_components.def,sha256=sAAAWOod8EuAnotR1cayaGAvs7x6xoMVlwmLso_a9Cc,578
-easylink/steps/cascading/update_clusters_by_connected_components.py,sha256=w7tAOs2QtIIcpTDxw2P_dqMIR-BFa-wi-OmZwrKyhmg,3309
+easylink/steps/cascading/update_clusters_by_connected_components.py,sha256=43D5GBmPXSgxcjgbJTvEoGFvPzBCGqYgBaT42pncNNw,3661
 easylink/steps/default/default_clusters_to_links.def,sha256=9PjUygLvsoYMUZDznceuuv55t8fPs473P57J_RMl3U0,527
 easylink/steps/default/default_clusters_to_links.py,sha256=EIYeP0lj0plBl2OpTRuv3iDEQl-zNVJONUg0kgKSEF0,2848
 easylink/steps/default/default_determining_exclusions.def,sha256=zZUEHDdrpLxzx3gTm-dki2ge5ivCCg4ziIwTErqCII0,542
@@ -83,9 +83,9 @@ easylink/utilities/paths.py,sha256=9inDKMPorAaWh5vhjc3j1Tj_aXVKhLdodiJO9H1nNes,9
 easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=UOz4hjkEPqaAz0RrDkDYYej79lLSaq0VVVSH_tF1z0o,3838
 easylink/utilities/validation_utils.py,sha256=DBJB2TLVklgYw1WaaPse9vqtwPLMGmZNYM2cbCZsoHI,18417
-easylink-0.1.19.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
-easylink-0.1.19.dist-info/METADATA,sha256=nFZA-jZKgZUG4DdiDqY-pNOTfdt1H3QeiwNzvo27vpg,3565
-easylink-0.1.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-easylink-0.1.19.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.19.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.19.dist-info/RECORD,,
+easylink-0.1.20.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
+easylink-0.1.20.dist-info/METADATA,sha256=aGNai6P-z5BQcQ0XYFTBr9JmuZAFTpZJYouFRlTJCzk,3565
+easylink-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+easylink-0.1.20.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.20.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.20.dist-info/RECORD,,