easylink 0.1.19__tar.gz → 0.1.20__tar.gz

{easylink-0.1.19 → easylink-0.1.20}/CHANGELOG.rst

@@ -1,3 +1,7 @@
+**0.1.20 - 6/11/25**
+
+ - Update implementation creator tool for new implementation_metadata.yaml format
+
 **0.1.19 - 6/11/25**
 
  - Implement support for images hosted on zenodo

{easylink-0.1.19 → easylink-0.1.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.19
+Version: 0.1.20
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

easylink-0.1.20/src/easylink/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.20"

{easylink-0.1.19 → easylink-0.1.20}/src/easylink/devtools/implementation_creator.py

@@ -29,7 +29,7 @@ from easylink.step import (
     TemplatedStep,
 )
 from easylink.utilities.data_utils import load_yaml
-from easylink.utilities.paths import IMPLEMENTATION_METADATA
+from easylink.utilities.paths import DEV_IMAGES_DIR, IMPLEMENTATION_METADATA
 
 
 def main(script_path: Path, host: Path) -> None:
@@ -195,9 +195,24 @@ class ImplementationCreator:
                 f"Implementation '{self.implementation_name}' already exists in the registry. "
                 "Overwriting it with the latest data."
             )
+
+        # Handle the fact that developers might be saving to username subdirs
+        # If the host folder is a subdirectory of DEV_IMAGES_DIR (e.g., the default
+        # host directory when calling `easylink devtools create-implementation`
+        # is DEV_IMAGES_DIR/<username>), we want to include the relative path
+        # to the DEV_IMAGES_DIR in the image name. This is required because ultimately
+        # when running a pipeline, all images are expected to be in a single directory.
+        image_name = (
+            self.hosted_container_path.name
+            # Use just the image name if the hosted path is not a part of DEV_IMAGES_DIR
+            if not self.hosted_container_path.is_relative_to(DEV_IMAGES_DIR)
+            # Use the path relative to DEV_IMAGES_DIR as the image name
+            else str(self.hosted_container_path.relative_to(DEV_IMAGES_DIR))
+        )
+
         info[self.implementation_name] = {
             "steps": [self.step],
-            "image_path": str(self.hosted_container_path),
+            "image_name": str(image_name),
             "script_cmd": f"{self.script_base_command} /{self.script_path.name}",
             "outputs": {
                 self.output_slot: "result.parquet",

{easylink-0.1.19 → easylink-0.1.20}/src/easylink/steps/cascading/update_clusters_by_connected_components.py

@@ -60,12 +60,14 @@ new_clusters_df = load_file(new_clusters_filepath)
 def merge_clusters(known_clusters_df, new_clusters_df):
     # Combine both dataframes
     combined_df = pd.concat([known_clusters_df, new_clusters_df], ignore_index=True)
-
-    # Drop records with missing cluster IDs
-    combined_df = combined_df.dropna(subset=["Cluster ID"])
+    combined_df["Input Record Key"] = (
+        combined_df["Input Record Dataset"]
+        + "-__-"
+        + combined_df["Input Record ID"].astype(int).astype(str)
+    )
 
     # Group by Cluster ID to get connected records
-    cluster_groups = combined_df.groupby("Cluster ID")["Input Record ID"].apply(list)
+    cluster_groups = combined_df.groupby("Cluster ID")["Input Record Key"].apply(list)
 
     # Build a graph of all connections implied by cluster IDs
     G = nx.Graph()
@@ -75,8 +77,8 @@ def merge_clusters(known_clusters_df, new_clusters_df):
                 G.add_edge(group[i], group[j])
 
     # Add isolated nodes (records with unique clusters)
-    all_ids = set(combined_df["Input Record ID"])
-    G.add_nodes_from(all_ids)
+    all_keys = set(combined_df["Input Record Key"])
+    G.add_nodes_from(all_keys)
 
     # Compute connected components
     components = list(nx.connected_components(G))
@@ -84,13 +86,19 @@ def merge_clusters(known_clusters_df, new_clusters_df):
     # Assign new cluster IDs
     merged_data = []
     for cluster_id, records in enumerate(components, start=1):
-        for record_id in records:
-            merged_data.append((record_id, cluster_id))
+        for record_key in records:
+            merged_data.append((record_key, cluster_id))
 
     # Build the final DataFrame
-    merged_df = pd.DataFrame(merged_data, columns=["Input Record ID", "Cluster ID"])
+    merged_df = pd.DataFrame(merged_data, columns=["Input Record Key", "Cluster ID"])
+
+    merged_df[["Input Record Dataset", "Input Record ID"]] = merged_df[
+        "Input Record Key"
+    ].str.split("-__-", n=1, expand=True)
+
+    merged_df["Input Record ID"] = merged_df["Input Record ID"].astype(int)
 
-    return merged_df
+    return merged_df[["Input Record Dataset", "Input Record ID", "Cluster ID"]]
 
 
 output_df = merge_clusters(known_clusters_df, new_clusters_df)

{easylink-0.1.19 → easylink-0.1.20}/src/easylink.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.19
+Version: 0.1.20
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.19 → easylink-0.1.20}/tests/e2e/test_pipelines_main_schema.py

@@ -18,18 +18,18 @@ from easylink.utilities.paths import DEV_IMAGES_DIR
 @pytest.mark.parametrize(
     "pipeline_specification, input_data, computing_environment, correct_results_csv",
     [
-        # slurm pipeline_splink_dummy.yaml
+        # local pipeline_splink_dummy.yaml
         (
             "tests/specifications/e2e/pipeline_splink_dummy.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/e2e/environment_slurm.yaml",
+            "tests/specifications/common/environment_local.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
-        # local pipeline_splink_dummy.yaml
+        # slurm pipeline_splink_dummy.yaml
         (
             "tests/specifications/e2e/pipeline_splink_dummy.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/common/environment_local.yaml",
+            "tests/specifications/e2e/environment_slurm.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
     ],
@@ -103,7 +103,6 @@ def test_pipeline_splink_dummy(
         )
 
 
-@pytest.mark.skip(reason="FIXME [SSCI-2312]: This test is sporadically failing")
 @pytest.mark.slow
 @pytest.mark.skipif(
     not is_on_slurm(),
@@ -112,32 +111,32 @@ def test_pipeline_splink_dummy(
 @pytest.mark.parametrize(
     "pipeline_specification, input_data, computing_environment, splink_results_csv",
     [
-        # slurm pipeline_with_fastLink.yaml
+        # local pipeline_with_fastLink.yaml
         (
             "tests/specifications/e2e/pipeline_with_fastLink.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/e2e/environment_slurm.yaml",
+            "tests/specifications/common/environment_local.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
-        # local pipeline_with_fastLink.yaml
+        # slurm pipeline_with_fastLink.yaml
         (
             "tests/specifications/e2e/pipeline_with_fastLink.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/common/environment_local.yaml",
+            "tests/specifications/e2e/environment_slurm.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
-        # slurm pipeline_cascade.yaml
+        # local pipeline_cascade.yaml
         (
             "tests/specifications/e2e/pipeline_cascade.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/e2e/environment_slurm.yaml",
+            "tests/specifications/common/environment_local.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
-        # local pipeline_cascade.yaml
+        # slurm pipeline_cascade.yaml
         (
             "tests/specifications/e2e/pipeline_cascade.yaml",
             "tests/specifications/e2e/input_data_dummy.yaml",
-            "tests/specifications/common/environment_local.yaml",
+            "tests/specifications/e2e/environment_slurm.yaml",
             "tests/e2e/pipeline_splink_dummy_results.csv",
         ),
     ],
@@ -158,6 +157,9 @@ def test_pipelines_same_output_relabeled(
     Jenkins logs.
     """
 
+    if "fastLink" in pipeline_specification:
+        pytest.skip(reason="FIXME [SSCI-2312]: This test is sporadically failing")
+
     with capsys.disabled():  # disabled so we can monitor job submissions
         print(
             "\n\n*** RUNNING TEST ***\n"

{easylink-0.1.19 → easylink-0.1.20}/tests/unit/test_implementation_creator.py

@@ -8,7 +8,7 @@ from pytest_mock import MockerFixture
 
 from easylink.devtools.implementation_creator import ImplementationCreator
 from easylink.utilities.data_utils import load_yaml
-from easylink.utilities.paths import IMPLEMENTATION_METADATA
+from easylink.utilities.paths import DEV_IMAGES_DIR, IMPLEMENTATION_METADATA
 
 GOOD_METADATA = """
 # STEP_NAME: step_1
@@ -181,31 +181,35 @@ def test_register(tmp_path: Path, mocker: MockerFixture) -> None:
     # copy the real implementation metadata file to the test directory
     shutil.copy(IMPLEMENTATION_METADATA, implementation_metadata)
 
+    # Don't accidentally write to the real implementation metadata file
     def _write_test_metadata(info: dict[str, str | dict[str, str]]) -> None:
-        # Don't accidentally write to the real implementation metadata file
         if implementation_metadata.resolve() == IMPLEMENTATION_METADATA.resolve():
             raise ValueError("Attempting to write to the real implementation metadata file")
         with open(str(implementation_metadata), "w") as f:
             yaml.dump(info, f, sort_keys=False)
 
+    mocker.patch(
+        "easylink.devtools.implementation_creator.ImplementationCreator._write_metadata",
+        side_effect=_write_test_metadata,
+    )
+    mocker.patch(
+        "easylink.devtools.implementation_creator.load_yaml",
+        return_value=load_yaml(implementation_metadata),
+    )
+
     # write the script to be used for the test
     with open(script_path, "w") as file:
         file.write(GOOD_METADATA)
 
     creator = ImplementationCreator(script_path, Path("some-host"))
-
     assert "test_implementation" not in load_yaml(implementation_metadata)
-    mocker.patch(
-        "easylink.devtools.implementation_creator.ImplementationCreator._write_metadata",
-        side_effect=_write_test_metadata,
-    )
     creator.register()
 
     # load the new metadata and check it
     details = load_yaml(implementation_metadata)["test_implementation"]
     assert details == {
         "steps": ["step_1"],
-        "image_path": "some-host/test_implementation.sif",
+        "image_name": "test_implementation.sif",
         "script_cmd": "python /test_implementation.py",
         "outputs": {"step_1_main_output": "result.parquet"},
     }
@@ -221,18 +225,59 @@ def test_register(tmp_path: Path, mocker: MockerFixture) -> None:
     md = load_yaml(implementation_metadata)
     assert "test_implementation" in md
     assert "test_new_implementation" not in md
-
     new_creator.register()
 
-    new_details = load_yaml(implementation_metadata)["test_new_implementation"]
-    assert new_details == {
+    md = load_yaml(implementation_metadata)
+    assert "test_implementation" in md
+    assert md["test_new_implementation"] == {
         "steps": ["step_1"],
-        "image_path": "some-other-host/test_new_implementation.sif",
+        "image_name": "test_new_implementation.sif",
         "script_cmd": "python /test_new_implementation.py",
         "outputs": {"step_1_main_output": "result.parquet"},
     }
 
 
+def test_register_to_dev_folder_subdir(tmp_path: Path, mocker: MockerFixture) -> None:
+
+    implementation_metadata = tmp_path / "test_implementation_metadata.yaml"
+    # copy the real implementation metadata file to the test directory
+    shutil.copy(IMPLEMENTATION_METADATA, implementation_metadata)
+
+    # Don't accidentally write to the real implementation metadata file
+    def _write_test_metadata(info: dict[str, str | dict[str, str]]) -> None:
+        if implementation_metadata.resolve() == IMPLEMENTATION_METADATA.resolve():
+            raise ValueError("Attempting to write to the real implementation metadata file")
+        with open(str(implementation_metadata), "w") as f:
+            yaml.dump(info, f, sort_keys=False)
+
+    mocker.patch(
+        "easylink.devtools.implementation_creator.ImplementationCreator._write_metadata",
+        side_effect=_write_test_metadata,
+    )
+    mocker.patch(
+        "easylink.devtools.implementation_creator.load_yaml",
+        return_value=load_yaml(implementation_metadata),
+    )
+
+    dev_script = tmp_path / "some_fancy_new_implementation.py"
+    with open(dev_script, "w") as file:
+        file.write(GOOD_METADATA)
+
+    dev_creator = ImplementationCreator(dev_script, Path(DEV_IMAGES_DIR) / "some-subdir")
+
+    md = load_yaml(implementation_metadata)
+    assert "some_fancy_new_implementation" not in md
+    dev_creator.register()
+
+    md = load_yaml(implementation_metadata)
+    assert md["some_fancy_new_implementation"] == {
+        "steps": ["step_1"],
+        "image_name": "some-subdir/some_fancy_new_implementation.sif",
+        "script_cmd": "python /some_fancy_new_implementation.py",
+        "outputs": {"step_1_main_output": "result.parquet"},
+    }
+
+
 ####################
 # Helper functions #
 ####################

easylink-0.1.19/src/easylink/_version.py

@@ -1 +0,0 @@
-__version__ = "0.1.19"