easylink 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

easylink/steps/default/default_updating_clusters.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./default_updating_clusters.py /default_updating_clusters.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /default_updating_clusters.py '$@'

easylink/steps/default/default_updating_clusters.py

@@ -0,0 +1,67 @@
+# STEP_NAME: updating_clusters
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# NEW_CLUSTERS_FILE_PATH is a path to a single file
+new_clusters_filepath = os.environ["NEW_CLUSTERS_FILE_PATH"]
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+known_clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(known_clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(known_clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+known_clusters_filepath = known_clusters_filepaths[0]
+known_clusters_df = load_file(known_clusters_filepath)
+
+if len(known_clusters_df) > 0:
+    raise ValueError(
+        "Default implementation of updating_clusters passed a non-empty set of known clusters"
+    )
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (clusters.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+Path(results_filepath).parent.mkdir(exist_ok=True, parents=True)
+
+clusters_df = load_file(new_clusters_filepath)
+
+
+logging.info(
+    f"Writing output for dataset from input {new_clusters_filepath} to {results_filepath}"
+)
+clusters_df.to_parquet(results_filepath)

easylink/steps/fastLink/fastLink_evaluating_pairs.R

@@ -0,0 +1,136 @@
+# STEP_NAME: evaluating_pairs
+# HAS_CUSTOM_RECIPE: true
+# SCRIPT_BASE_COMMAND: Rscript
+
+library(fastLink)
+library(arrow)
+library(dplyr)
+library(stringr)
+
+# Check required environment variables
+required_env_vars <- c(
+  "BLOCKS_DIR_PATH",
+  "DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY",
+  "DUMMY_CONTAINER_OUTPUT_PATHS",
+  "COMPARISONS",
+  "THRESHOLD_MATCH_PROBABILITY"
+)
+missing_vars <- required_env_vars[!nzchar(Sys.getenv(required_env_vars))]
+if (length(missing_vars) > 0) {
+  stop(sprintf(
+    "The following required environment variables are not set or are empty: %s",
+    paste(missing_vars, collapse = ", ")
+  ))
+}
+
+blocks_dir <- Sys.getenv("BLOCKS_DIR_PATH")
+diagnostics_dir <- Sys.getenv("DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY")
+output_path <- Sys.getenv("DUMMY_CONTAINER_OUTPUT_PATHS")
+comparisons <- strsplit(Sys.getenv("COMPARISONS"), ",")[[1]]
+
+all_predictions <- list()
+
+block_dirs <- list.dirs(blocks_dir, recursive = FALSE, full.names = TRUE)
+
+for (block_dir in block_dirs) {
+  records_path <- file.path(block_dir, "records.parquet")
+  pairs_path <- file.path(block_dir, "pairs.parquet")
+  if (!file.exists(records_path)) {
+    stop(sprintf("File not found: %s", records_path))
+  }
+  if (!file.exists(pairs_path)) {
+    stop(sprintf("File not found: %s", pairs_path))
+  }
+
+  records <- read_parquet(records_path) %>%
+    mutate(
+      unique_id = paste0(`Input Record Dataset`, "::", `Input Record ID`)
+    )
+  pairs <- read_parquet(pairs_path) %>%
+    mutate(
+      join_key_l = paste0(`Left Record Dataset`, "::", `Left Record ID`),
+      join_key_r = paste0(`Right Record Dataset`, "::", `Right Record ID`)
+    )
+
+  # Subset records to only those that appear in pairs (left and right separately)
+  left_ids <- unique(pairs$join_key_l)
+  right_ids <- unique(pairs$join_key_r)
+  dfA <- records %>% filter(unique_id %in% left_ids)
+  dfB <- records %>% filter(unique_id %in% right_ids)
+
+  # Prepare comparison columns and fastLink match flags
+  comparison_cols <- sapply(comparisons, function(x) strsplit(x, ":")[[1]][1])
+  comparison_methods <- sapply(comparisons, function(x) strsplit(x, ":")[[1]][2])
+
+  stringdist.match <- comparison_cols[comparison_methods == "stringdist" | comparison_methods == "partial"]
+  partial.match <- comparison_cols[comparison_methods == "partial"]
+
+  # Estimate number of candidate pairs (after subsetting)
+  nA <- nrow(dfA)
+  nB <- nrow(dfB)
+  n_requested_pairs <- nrow(pairs)
+  n_possible_pairs <- nA * nB
+  if (n_possible_pairs > 10 * n_requested_pairs) {
+    warning(sprintf(
+      "fastLink will compute %d candidate pairs, which is more than 10x the %d requested pairs.",
+      n_possible_pairs, n_requested_pairs
+    ))
+  }
+
+  # Trick: add a dummy column to dfB to avoid fastLink dedupe restriction if dfA and dfB are identical
+  dfB_b <- dfB
+  if (identical(dfA, dfB)) {
+    dfB_b[["__dummy__"]] <- 1
+  }
+
+  # Run fastLink on all pairs (Cartesian product of reduced sets)
+  fl_out <- fastLink::fastLink(
+    dfA = dfA,
+    dfB = dfB_b,
+    varnames = comparison_cols,
+    stringdist.match = stringdist.match,
+    partial.match = partial.match,
+    threshold.match = as.numeric(Sys.getenv("THRESHOLD_MATCH_PROBABILITY")),
+    dedupe.matches = FALSE,
+    return.all = TRUE
+  )
+
+  inds_a <- fl_out$matches$inds.a
+  inds_b <- fl_out$matches$inds.b
+  posteriors <- fl_out$posterior
+
+  # Build matched pairs dataframe
+  dfA_match <- dfA[inds_a, , drop = FALSE]
+  dfB_match <- dfB[inds_b, , drop = FALSE]
+  matches <- data.frame(
+    join_key_l = dfA_match$unique_id,
+    join_key_r = dfB_match$unique_id,
+    Probability = posteriors
+  )
+
+  # Subset to only requested pairs
+  matches <- matches %>%
+    semi_join(pairs, by = c("join_key_l", "join_key_r"))
+
+  # Parse out dataset and record ID from join keys
+  predictions <- matches %>%
+    transmute(
+      `Left Record Dataset` = str_split_fixed(join_key_l, "::", 2)[,1],
+      `Left Record ID` = as.integer(str_split_fixed(join_key_l, "::", 2)[,2]),
+      `Right Record Dataset` = str_split_fixed(join_key_r, "::", 2)[,1],
+      `Right Record ID` = as.integer(str_split_fixed(join_key_r, "::", 2)[,2]),
+      Probability
+    )
+
+  all_predictions[[length(all_predictions) + 1]] <- predictions
+
+  # Optionally, save diagnostics (e.g., plot of match probabilities)
+  chart_path <- file.path(diagnostics_dir, paste0("match_weights_chart_", basename(block_dir), ".png"))
+  png(chart_path)
+  hist(predictions$Probability, main = "Match Probabilities", xlab = "Probability")
+  dev.off()
+}
+
+all_predictions_df <- bind_rows(all_predictions)
+print(all_predictions_df)
+write_parquet(all_predictions_df, output_path)

easylink/steps/fastLink/fastLink_evaluating_pairs.def

@@ -0,0 +1,21 @@
+
+Bootstrap: docker
+From: rocker/r2u@sha256:02311f32a3e58b73027a9f572836ed4772fc530c9678b953604d875ea58ddde4
+
+%files
+    ./fastLink_evaluating_pairs.R /fastLink_evaluating_pairs.R
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    Rscript -e 'install.packages(c("arrow", "dplyr", "stringr", "fastLink"))'
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    Rscript /fastLink_evaluating_pairs.R '$@'

easylink/steps/fastLink/fastLink_links_to_clusters.R

@@ -0,0 +1,128 @@
+# STEP_NAME: links_to_clusters
+# HAS_CUSTOM_RECIPE: true
+# SCRIPT_BASE_COMMAND: Rscript
+
+options(error = function() traceback(3))
+
+library(fastLink)
+library(arrow)
+library(dplyr)
+library(stringr)
+
+# Check required environment variables
+required_env_vars <- c(
+    "LINKS_FILE_PATH",
+    "DUMMY_CONTAINER_OUTPUT_PATHS",
+    "THRESHOLD_MATCH_PROBABILITY"
+)
+missing_vars <- required_env_vars[!nzchar(Sys.getenv(required_env_vars))]
+if (length(missing_vars) > 0) {
+    stop(sprintf(
+        "The following required environment variables are not set or are empty: %s",
+        paste(missing_vars, collapse = ", ")
+    ))
+}
+
+links_file_path <- Sys.getenv("LINKS_FILE_PATH")
+output_path <- Sys.getenv("DUMMY_CONTAINER_OUTPUT_PATHS")
+
+if (!file.exists(links_file_path)) {
+    stop(sprintf("File not found: %s", links_file_path))
+}
+
+links <- read_parquet(links_file_path)
+
+# Filter links by threshold
+threshold <- as.numeric(Sys.getenv("THRESHOLD_MATCH_PROBABILITY"))
+links <- links %>%
+    filter(Probability >= threshold)
+
+if (nrow(links) == 0) {
+    clusters <- data.frame(
+        `Input Record Dataset` = character(),
+        `Input Record ID` = character(),
+        `Cluster ID` = integer()
+    )
+} else {
+    # NOTE: fastLink's dedupeMatches function is quite tightly coupled with other parts
+    # of the fastLink pipeline.
+    # It takes a bunch of objects, usually created by earlier steps in the pipeline,
+    # which need to have particular formats.
+    # Here, we "spoof" these objects using carefully constructed data.frames in order to
+    # use the linear sum assignment algorithm in isolation (potentially with links not
+    # from fastLink).
+    # ChatGPT helped me write the below code.
+    
+    # Combine dataset and record ID for unique keys
+    links <- links %>%
+        mutate(
+            Left_Record_Key = paste0(`Left Record Dataset`, "::", `Left Record ID`),
+            Right_Record_Key = paste0(`Right Record Dataset`, "::", `Right Record ID`)
+        )
+
+    n <- nrow(links)
+
+    # 1) Build matchesA / matchesB using the combined keys without touching `links`
+    matchesA <- data.frame(
+        idA = links[["Left_Record_Key"]],
+        idB = links[["Right_Record_Key"]],
+        prob = links[["Probability"]],
+        stringsAsFactors = FALSE
+    )
+    matchesB <- matchesA  # identical copy
+
+    # 2) Dummy one‐to‐one pattern key
+    patterns <- data.frame(
+        patID = seq_len(n)
+    )
+
+    # 3) Fake EM object, with dummy gamma columns
+    EM <- list(
+        patterns.w = data.frame(
+            patID = patterns$patID,
+            counts = 1,
+            weights = 1,
+            p.gamma.j.m = 1,
+            p.gamma.j.u = 1
+        ),
+        # convert match‐probabilities into log‐odds scores
+        zeta.j = log(matchesA$prob / (1 - matchesA$prob))
+    )
+
+    # 4) Map each row back to its original record keys
+    matchesLink <- data.frame(
+        inds.a = links[["Left_Record_Key"]],
+        inds.b = links[["Right_Record_Key"]],
+        stringsAsFactors = FALSE
+    )
+
+    # 5) Call dedupeMatches with linear programming
+    deduped <- dedupeMatches(
+        matchesA = matchesA,
+        matchesB = matchesB,
+        EM = EM,
+        matchesLink = matchesLink,
+        patterns = patterns,
+        linprog = TRUE
+    )
+
+    # Parse out dataset and record ID from keys for both sides, then bind
+    clusters <- deduped$matchesLink %>%
+        transmute(
+            `Input Record Dataset` = str_split_fixed(inds.a, "::", 2)[,1],
+            `Input Record ID` = as.integer(str_split_fixed(inds.a, "::", 2)[,2]),
+            `Cluster ID` = paste0("cluster_", inds.a, "_", inds.b)
+        ) %>%
+        bind_rows(
+            deduped$matchesLink %>%
+                transmute(
+                    `Input Record Dataset` = str_split_fixed(inds.b, "::", 2)[,1],
+                    `Input Record ID` = as.integer(str_split_fixed(inds.b, "::", 2)[,2]),
+                    `Cluster ID` = paste0("cluster_", inds.a, "_", inds.b)
+                )
+        )
+}
+
+print(clusters)
+# Write clusters to output
+arrow::write_parquet(clusters, output_path)

easylink/steps/fastLink/fastLink_links_to_clusters.def

@@ -0,0 +1,21 @@
+
+Bootstrap: docker
+From: rocker/r2u@sha256:02311f32a3e58b73027a9f572836ed4772fc530c9678b953604d875ea58ddde4
+
+%files
+    ./fastLink_links_to_clusters.R /fastLink_links_to_clusters.R
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    Rscript -e 'install.packages(c("arrow", "dplyr", "stringr", "fastLink"))'
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    Rscript /fastLink_links_to_clusters.R '$@'

easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./dummy_canonicalizing_and_downstream_analysis.py /dummy_canonicalizing_and_downstream_analysis.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /dummy_canonicalizing_and_downstream_analysis.py '$@'

easylink/steps/rl-dummy/canonicalizing_and_downstream_analysis/dummy_canonicalizing_and_downstream_analysis.py

@@ -0,0 +1,42 @@
+# STEP_NAME: canonicalizing_and_downstream_analysis
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from itertools import chain, combinations
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# For dummy we will load the clusters and output (only) them as-is
+
+# CLUSTERS_FILE_PATH is a path to a single file
+clusters_path = os.environ["CLUSTERS_FILE_PATH"]
+# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (results.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+clusters_df = load_file(clusters_path)
+
+logging.info(f"Writing output for dataset from input {clusters_path} to {results_filepath}")
+clusters_df.to_parquet(results_filepath)