easylink 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./dummy_pre-processing.py /dummy_pre-processing.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /dummy_pre-processing.py '$@'

easylink/steps/rl-dummy/pre-processing/dummy_pre-processing.py

@@ -0,0 +1,59 @@
+# STEP_NAME: pre-processing
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# DATASET_DIR_PATHS is list of directories, each containing one file
+dataset_paths = os.environ["DATASET_DIR_PATHS"].split(",")
+logging.info(f"{dataset_paths=}")
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+logging.info(f"splitter_choice={splitter_choice}")
+dataset_path = None
+for path in dataset_paths:
+    path = Path(path)
+    # NOTE: We iterate the dir here, but it should only have one non-hidden
+    # file in it. We don't validate that here as it is checked in the validator.
+    for path_to_check in path.iterdir():
+        if path_to_check.stem == splitter_choice:
+            dataset_path = str(path_to_check)
+            break
+
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+results_dir.mkdir(exist_ok=True, parents=True)
+
+output_path = results_dir / Path(dataset_path).name
+
+dataset = load_file(dataset_path)
+
+# NOTE: No actual pre-processing here, we save as-is.
+
+logging.info(f"Writing output for dataset from input {dataset_path} to {output_path}")
+dataset.to_parquet(output_path)

easylink/steps/splink/splink_blocking_and_filtering.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./splink_blocking_and_filtering.py /splink_blocking_and_filtering.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas pyarrow splink==4.0.7
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /splink_blocking_and_filtering.py '$@'

easylink/steps/splink/splink_blocking_and_filtering.py

@@ -0,0 +1,130 @@
+# STEP_NAME: blocking_and_filtering
+# REQUIREMENTS: pandas pyarrow splink==4.0.7
+
+import os
+
+import pandas as pd
+
+records = pd.read_parquet(os.environ["RECORDS_FILE_PATH"])
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+import splink
+
+blocking_rules = os.environ["BLOCKING_RULES"].split(",")
+
+from splink import Linker, SettingsCreator
+
+# Create the Splink linker in dedupe mode
+settings = SettingsCreator(
+    link_type="link_and_dedupe",
+    blocking_rules_to_generate_predictions=blocking_rules,
+    comparisons=[],
+)
+from splink import DuckDBAPI
+
+grouped = records.rename(columns={"Input Record ID": "unique_id"}).groupby(
+    "Input Record Dataset"
+)
+
+db_api = DuckDBAPI()
+linker = Linker(
+    [df for _, df in grouped],
+    settings,
+    db_api=db_api,
+    input_table_aliases=[name for name, _ in grouped],
+)
+
+# Copied/adapted from https://github.com/moj-analytical-services/splink/blob/3eb1921eaff6b8471d3ebacd3238eb514f62c844/splink/internals/linker_components/inference.py#L86-L131
+from splink.internals.pipeline import CTEPipeline
+from splink.internals.vertically_concatenate import compute_df_concat_with_tf
+
+pipeline = CTEPipeline()
+
+# In duckdb, calls to random() in a CTE pipeline cause problems:
+# https://gist.github.com/RobinL/d329e7004998503ce91b68479aa41139
+df_concat_with_tf = compute_df_concat_with_tf(linker, pipeline)
+pipeline = CTEPipeline([df_concat_with_tf])
+
+blocking_input_tablename_l = "__splink__df_concat_with_tf"
+blocking_input_tablename_r = "__splink__df_concat_with_tf"
+
+link_type = linker._settings_obj._link_type
+
+# If exploded blocking rules exist, we need to materialise
+# the tables of ID pairs
+from splink.internals.blocking import materialise_exploded_id_tables
+
+exploding_br_with_id_tables = materialise_exploded_id_tables(
+    link_type=link_type,
+    blocking_rules=linker._settings_obj._blocking_rules_to_generate_predictions,
+    db_api=linker._db_api,
+    splink_df_dict=linker._input_tables_dict,
+    source_dataset_input_column=linker._settings_obj.column_info_settings.source_dataset_input_column,
+    unique_id_input_column=linker._settings_obj.column_info_settings.unique_id_input_column,
+)
+
+from splink.internals.blocking import block_using_rules_sqls
+
+sqls = block_using_rules_sqls(
+    input_tablename_l=blocking_input_tablename_l,
+    input_tablename_r=blocking_input_tablename_r,
+    blocking_rules=linker._settings_obj._blocking_rules_to_generate_predictions,
+    link_type=link_type,
+    source_dataset_input_column=linker._settings_obj.column_info_settings.source_dataset_input_column,
+    unique_id_input_column=linker._settings_obj.column_info_settings.unique_id_input_column,
+)
+
+pipeline.enqueue_list_of_sqls(sqls)
+
+blocked_pairs = (
+    linker._db_api.sql_pipeline_to_splink_dataframe(pipeline)
+    .as_pandas_dataframe()
+    .drop(columns=["match_key"])
+)
+
+blocked_pairs[["Left Record Dataset", "Left Record ID"]] = blocked_pairs.pop(
+    "join_key_l"
+).str.split("-__-", n=1, expand=True)
+blocked_pairs[["Right Record Dataset", "Right Record ID"]] = blocked_pairs.pop(
+    "join_key_r"
+).str.split("-__-", n=1, expand=True)
+blocked_pairs[["Left Record ID", "Right Record ID"]] = blocked_pairs[
+    ["Left Record ID", "Right Record ID"]
+].astype(int)
+
+# Now ensure correct ordering
+wrong_order_dataset = (
+    blocked_pairs["Left Record Dataset"] > blocked_pairs["Right Record Dataset"]
+)
+id_cols = ["Left Record Dataset", "Left Record ID", "Right Record Dataset", "Right Record ID"]
+switched_id_cols = [
+    "Right Record Dataset",
+    "Right Record ID",
+    "Left Record Dataset",
+    "Left Record ID",
+]
+blocked_pairs.loc[wrong_order_dataset, id_cols] = blocked_pairs.loc[
+    wrong_order_dataset, switched_id_cols
+].values
+
+wrong_order_ids = (
+    blocked_pairs["Left Record Dataset"] == blocked_pairs["Right Record Dataset"]
+) & (blocked_pairs["Left Record ID"] > blocked_pairs["Right Record ID"])
+blocked_pairs.loc[wrong_order_ids, id_cols] = blocked_pairs.loc[
+    wrong_order_ids, switched_id_cols
+].values
+blocked_pairs[["Left Record ID", "Right Record ID"]] = blocked_pairs[
+    ["Left Record ID", "Right Record ID"]
+].astype(int)
+
+print(blocked_pairs)
+
+from pathlib import Path
+
+output_path = Path(results_dir) / "block_0"
+output_path.mkdir(exist_ok=True, parents=True)
+
+records.to_parquet(output_path / "records.parquet", index=False)
+blocked_pairs.to_parquet(output_path / "pairs.parquet", index=False)

easylink/steps/splink/splink_evaluating_pairs.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./splink_evaluating_pairs.py /splink_evaluating_pairs.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas pyarrow splink==4.0.7
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /splink_evaluating_pairs.py '$@'

easylink/steps/splink/splink_evaluating_pairs.py

@@ -0,0 +1,164 @@
+# STEP_NAME: evaluating_pairs
+# REQUIREMENTS: pandas pyarrow splink==4.0.7
+
+import os
+from pathlib import Path
+
+import pandas as pd
+import splink
+import splink.comparison_library as cl
+from splink import Linker, SettingsCreator
+
+blocks_dir = Path(os.environ["BLOCKS_DIR_PATH"])
+diagnostics_dir = Path(os.environ["DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY"])
+output_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+Path(output_path).parent.mkdir(exist_ok=True, parents=True)
+
+all_predictions = []
+
+for block_dir in blocks_dir.iterdir():
+    if str(block_dir.stem).startswith("."):
+        continue
+    encoded_comparisons = os.environ["COMPARISONS"].split(",")
+
+    comparisons = []
+    for encoded_comparison in encoded_comparisons:
+        column, method = encoded_comparison.split(":")
+        if method == "exact":
+            comparisons.append(cl.ExactMatch(column))
+        elif method == "name":
+            comparisons.append(cl.NameComparison(column))
+        elif method == "dob":
+            comparisons.append(cl.DateOfBirthComparison(column))
+        else:
+            raise ValueError(f"Unknown comparison method {method}")
+
+    # Create the Splink linker in dedupe mode
+    settings = SettingsCreator(
+        link_type="link_and_dedupe",
+        blocking_rules_to_generate_predictions=[],
+        comparisons=comparisons,
+        probability_two_random_records_match=float(
+            os.environ["PROBABILITY_TWO_RANDOM_RECORDS_MATCH"]
+        ),
+    )
+
+    grouped = (
+        pd.read_parquet(block_dir / "records.parquet")
+        .rename(columns={"Input Record ID": "unique_id"})
+        .groupby("Input Record Dataset")
+    )
+
+    from splink import DuckDBAPI
+
+    db_api = DuckDBAPI()
+    linker = Linker(
+        [df for _, df in grouped],
+        settings,
+        db_api=db_api,
+        input_table_aliases=[name for name, _ in grouped],
+    )
+
+    linker.training.estimate_u_using_random_sampling(max_pairs=5e6)
+
+    blocking_rules_for_training = os.environ["BLOCKING_RULES_FOR_TRAINING"].split(",")
+
+    for blocking_rule_for_training in blocking_rules_for_training:
+        linker.training.estimate_parameters_using_expectation_maximisation(
+            blocking_rule_for_training
+        )
+
+    chart_path = diagnostics_dir / f"match_weights_chart_{block_dir}.html"
+    chart_path.parent.mkdir(exist_ok=True, parents=True)
+    linker.visualisations.match_weights_chart().save(chart_path)
+
+    # Copied/adapted from https://github.com/moj-analytical-services/splink/blob/3eb1921eaff6b8471d3ebacd3238eb514f62c844/splink/internals/linker_components/inference.py#L264-L293
+    from splink.internals.pipeline import CTEPipeline
+    from splink.internals.vertically_concatenate import compute_df_concat_with_tf
+
+    pipeline = CTEPipeline()
+
+    # In duckdb, calls to random() in a CTE pipeline cause problems:
+    # https://gist.github.com/RobinL/d329e7004998503ce91b68479aa41139
+    pairs = (
+        pd.read_parquet(block_dir / "pairs.parquet")
+        .assign(
+            join_key_l=lambda df: df["Left Record Dataset"]
+            + "-__-"
+            + df["Left Record ID"].astype(int).astype(str),
+            join_key_r=lambda df: df["Right Record Dataset"]
+            + "-__-"
+            + df["Right Record ID"].astype(int).astype(str),
+        )
+        .drop(
+            columns=[
+                "Left Record Dataset",
+                "Left Record ID",
+                "Right Record Dataset",
+                "Right Record ID",
+            ]
+        )
+        .assign(match_key=0)
+    )  # What is this?
+    db_api._table_registration(pairs, "__splink__blocked_id_pairs")
+    df_concat_with_tf = compute_df_concat_with_tf(linker, pipeline)
+    pipeline = CTEPipeline(
+        [
+            db_api.table_to_splink_dataframe(
+                "__splink__blocked_id_pairs", "__splink__blocked_id_pairs"
+            ),
+            df_concat_with_tf,
+        ]
+    )
+
+    from splink.internals.comparison_vector_values import (
+        compute_comparison_vector_values_from_id_pairs_sqls,
+    )
+
+    sqls = compute_comparison_vector_values_from_id_pairs_sqls(
+        linker._settings_obj._columns_to_select_for_blocking,
+        linker._settings_obj._columns_to_select_for_comparison_vector_values,
+        input_tablename_l="__splink__df_concat_with_tf",
+        input_tablename_r="__splink__df_concat_with_tf",
+        source_dataset_input_column=linker._settings_obj.column_info_settings.source_dataset_input_column,
+        unique_id_input_column=linker._settings_obj.column_info_settings.unique_id_input_column,
+    )
+    pipeline.enqueue_list_of_sqls(sqls)
+
+    from splink.internals.predict import (
+        predict_from_comparison_vectors_sqls_using_settings,
+    )
+
+    sqls = predict_from_comparison_vectors_sqls_using_settings(
+        linker._settings_obj,
+        float(os.environ["THRESHOLD_MATCH_PROBABILITY"]),
+        threshold_match_weight=None,
+        sql_infinity_expression=linker._infinity_expression,
+    )
+    pipeline.enqueue_list_of_sqls(sqls)
+
+    predictions = linker._db_api.sql_pipeline_to_splink_dataframe(pipeline)
+
+    linker._predict_warning()
+
+    all_predictions.append(predictions.as_pandas_dataframe())
+
+all_predictions = pd.concat(all_predictions, ignore_index=True)[
+    [
+        "source_dataset_l",
+        "unique_id_l",
+        "source_dataset_r",
+        "unique_id_r",
+        "match_probability",
+    ]
+].rename(
+    columns={
+        "source_dataset_l": "Left Record Dataset",
+        "unique_id_l": "Left Record ID",
+        "source_dataset_r": "Right Record Dataset",
+        "unique_id_r": "Right Record ID",
+        "match_probability": "Probability",
+    }
+)
+print(all_predictions)
+all_predictions.to_parquet(output_path)

easylink/steps/splink/splink_links_to_clusters.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./splink_links_to_clusters.py /splink_links_to_clusters.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas pyarrow splink==4.0.7
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /splink_links_to_clusters.py '$@'

easylink/steps/splink/splink_links_to_clusters.py

@@ -0,0 +1,63 @@
+# STEP_NAME: links_to_clusters
+# REQUIREMENTS: pandas pyarrow splink==4.0.7
+
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+# Adapted from example on https://moj-analytical-services.github.io/splink/api_docs/clustering.html
+from splink import DuckDBAPI
+from splink.clustering import cluster_pairwise_predictions_at_threshold
+
+links = pd.read_parquet(os.environ["LINKS_FILE_PATH"]).rename(
+    columns={
+        "Probability": "match_probability",
+    }
+)
+
+# Create unique record keys by concatenating Input Record Dataset and Record ID for both left and right
+links["Left Record Key"] = (
+    links["Left Record Dataset"].astype(str) + "-__-" + links["Left Record ID"].astype(str)
+)
+links["Right Record Key"] = (
+    links["Right Record Dataset"].astype(str) + "-__-" + links["Right Record ID"].astype(str)
+)
+
+dummy_records_df = pd.DataFrame(
+    {
+        "Record Key": np.unique(
+            list(links["Left Record Key"]) + list(links["Right Record Key"])
+        )
+    }
+)
+output_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+
+db_api = DuckDBAPI()
+
+cc = (
+    cluster_pairwise_predictions_at_threshold(
+        dummy_records_df,
+        links,
+        node_id_column_name='"Record Key"',
+        edge_id_column_name_left='"Left Record Key"',
+        edge_id_column_name_right='"Right Record Key"',
+        db_api=db_api,
+        threshold_match_probability=float(os.environ["THRESHOLD_MATCH_PROBABILITY"]),
+    )
+    .as_pandas_dataframe()
+    .rename(columns={"cluster_id": "Cluster ID"})
+)
+
+# Split "Record Key" back into "Input Record Dataset" and "Input Record ID"
+cc[["Input Record Dataset", "Input Record ID"]] = (
+    cc["Record Key"].astype(str).str.split("-__-", n=1, expand=True)
+)
+cc = cc.drop(columns=["Record Key"])
+cc["Input Record ID"] = cc["Input Record ID"].astype(int)
+cc = cc[["Input Record Dataset", "Input Record ID", "Cluster ID"]]
+
+print(cc)
+
+cc.to_parquet(output_path)

easylink/utilities/data_utils.py

@@ -8,13 +8,17 @@ This module contains utility functions for handling data files and directories.
 
 """
 
+import hashlib
 import os
 import shutil
 from collections.abc import Callable
 from datetime import datetime
 from pathlib import Path
 
+import requests
 import yaml
+from loguru import logger
+from tqdm import tqdm
 
 
 def modify_umask(func: Callable) -> Callable:
@@ -152,3 +156,71 @@ def load_yaml(filepath: str | Path) -> dict:
     with open(filepath, "r") as file:
         data = yaml.safe_load(file)
     return data
+
+
+@modify_umask
+def download_image(
+    images_dir: str | Path, record_id: int, filename: str, md5_checksum: str
+) -> None:
+    """Downloads an image from zenodo.
+
+    Parameters
+    ----------
+    images_dir
+        The directory to download the image to.
+    record_id
+        The zenodo record ID that the image is a part of.
+    filename
+        The name of the image file to download.
+    md5_checksum
+        The expected MD5 checksum of the image file.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the image file was not downloaded.
+    ValueError
+        If the MD5 checksum of the downloaded file does not match the expected checksum.
+    """
+
+    images_dir = Path(images_dir).resolve()
+    if not images_dir.exists():
+        images_dir.mkdir(parents=True, exist_ok=True)
+
+    url = f"https://zenodo.org/record/{record_id}/files/{filename}?download=1"
+
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    total_size = int(response.headers.get("Content-Length", 0))
+    output_path = images_dir / filename
+    logger.info(f"Downloading {filename} to {output_path}...")
+    with open(output_path, "wb") as file, tqdm(
+        total=total_size, unit="B", unit_scale=True, desc=filename
+    ) as progress_bar:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                file.write(chunk)
+                progress_bar.update(len(chunk))
+
+    if not output_path.exists():
+        raise FileNotFoundError(f"Failed to download the image: {filename}")
+
+    # Verify MD5 checksum
+    calculated_md5_checksum = calculate_md5_checksum(output_path)
+    if calculated_md5_checksum != md5_checksum:
+        raise ValueError(
+            f"MD5 checksum does not match for {filename}.\n"
+            f"Try manually downloading the image and then moving it to the {images_dir} directory.\n"
+            f"Download the image by visiting this link: {url}"
+        )
+
+
+def calculate_md5_checksum(output_path: Path) -> str:
+    md5_hash = hashlib.md5()
+    with open(output_path, "rb") as file:
+        while chunk := file.read(8192):
+            md5_hash.update(chunk)
+
+    calculated_md5_checksum = md5_hash.hexdigest()
+    return calculated_md5_checksum

easylink/utilities/paths.py

@@ -9,9 +9,10 @@ This module contains commonly-used filepaths and directories.
 
 from pathlib import Path
 
-# TODO: We'll need to update this to be more generic for external users and have a way of configuring this
-CONTAINER_DIR = "/mnt/team/simulation_science/priv/engineering/er_ecosystem/images"
-"""Path to the directory where the container images are stored."""
+DEV_IMAGES_DIR = "/mnt/team/simulation_science/priv/engineering/er_ecosystem/images"
+"""Path to the directory where the development/dummy pipeline images are stored."""
+DEFAULT_IMAGES_DIR = Path.home() / ".easylink_images"
+"""Default subdirectory for storing downloaded images."""
 IMPLEMENTATION_METADATA = Path(__file__).parent.parent / "implementation_metadata.yaml"
 """Path to the implementation metadata file."""
 EASYLINK_TEMP = {"local": Path("/tmp/easylink"), "slurm": Path("/tmp")}