hlink 4.1.0__py3-none-any.whl

hlink/configs/load_config.py

@@ -0,0 +1,82 @@
+# This file is part of the ISRDI's hlink.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/ipums/hlink
+
+from pathlib import Path
+from typing import Any
+import json
+import toml
+import tomli
+
+from hlink.errors import UsageError
+
+
+def load_conf_file(
+    conf_name: str, *, use_legacy_toml_parser: bool = False
+) -> tuple[Path, dict[str, Any]]:
+    """Flexibly load a config file.
+
+    Given a path `conf_name`, look for a file at that path. If that file
+    exists and has a '.toml' extension or a '.json' extension, load it and
+    return its contents. If it doesn't exist, look for a file with the same
+    name with a '.toml' extension added and load it if it exists. Then do the
+    same for a file with a '.json' extension added.
+
+    `use_legacy_toml_parser` tells this function to use the legacy TOML library
+    which hlink used to use instead of the current default. This is provided
+    for backwards compatibility. Some previously written config files may
+    depend on bugs in the legacy TOML library, making it hard to migrate to the
+    new TOML v1.0 compliant parser. It is strongly recommended that new code
+    and config files use the default parser. Old code and config files should
+    also try to migrate to the default parser when possible.
+
+    Args:
+        conf_name: the file to look for
+        use_legacy_toml_parser: (Not Recommended) Use the legacy, buggy TOML
+        parser instead of the default parser.
+
+    Returns:
+        a tuple (absolute path to the config file, contents of the config file)
+
+    Raises:
+        FileNotFoundError: if none of the three checked files exist
+        UsageError: if the file at path `conf_name` exists, but it doesn't have a '.toml' or '.json' extension
+    """
+    candidate_files = [
+        Path(conf_name),
+        Path(conf_name + ".toml"),
+        Path(conf_name + ".json"),
+    ]
+
+    existing_files = filter((lambda file: file.exists()), candidate_files)
+
+    for file in existing_files:
+        if file.suffix == ".toml":
+            # Legacy support for using the "toml" library instead of "tomli".
+            #
+            # Eventually we should remove use_legacy_toml_parser and just use
+            # tomli or Python's standard library tomllib, which is available in
+            # Python 3.11+.
+            if use_legacy_toml_parser:
+                with open(file) as f:
+                    conf = toml.load(f)
+                    return file.absolute(), conf
+            else:
+                with open(file, "rb") as f:
+                    conf = tomli.load(f)
+                    return file.absolute(), conf
+
+        if file.suffix == ".json":
+            with open(file) as f:
+                conf = json.load(f)
+                return file.absolute(), conf
+
+        raise UsageError(
+            f"The file {file} exists, but it doesn't have a '.toml' or '.json' extension."
+        )
+
+    candidate_files_str = ", ".join(map(str, candidate_files))
+    raise FileNotFoundError(
+        f"Couldn't find any of these three files: {candidate_files_str}"
+    )

hlink/errors.py

@@ -0,0 +1,16 @@
+# This file is part of the ISRDI's hlink.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/ipums/hlink
+
+
+class SparkError(Exception):
+    """Catch any exceptions from Spark"""
+
+
+class UsageError(Exception):
+    """Incorrectly specified options"""
+
+
+class DataError(Exception):
+    """There is an issue in the source data that will cause a problem."""

hlink/linking/README.md

@@ -0,0 +1,10 @@
+## Overview
+
+There is one base class,`LinkTask`, defined in the file: `link_task.py`. Each subpackage contains a class which inherits from the base `LinkTask`.
+
+All classes inheriting from `LinkTask`, directly correspond to a task that the user can do. For example, the `Preprocessing` class contains all the code for the `preprocessing` user task.
+
+## Templates
+
+The `templates` directory in each subpackage contains SQL file templates that are written using [jinja2](http://jinja.pocoo.org/docs/2.10/templates/). Putting all the sql in this directory allows for a seperation between the python code and the SQL code used. Using jinja2 templating allows for reuse of an sql file with slightly different parameters. 
+

hlink/linking/core/classifier.py

@@ -0,0 +1,154 @@
+# This file is part of the ISRDI's hlink.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/ipums/hlink
+
+from typing import Any
+
+from pyspark.ml.feature import SQLTransformer
+from pyspark.ml.regression import GeneralizedLinearRegression
+from pyspark.ml.classification import (
+    RandomForestClassifier,
+    LogisticRegression,
+    DecisionTreeClassifier,
+    GBTClassifier,
+)
+import hlink.linking.transformers.rename_prob_column
+
+try:
+    import synapse.ml.lightgbm
+except ModuleNotFoundError:
+    _lightgbm_available = False
+else:
+    _lightgbm_available = True
+
+try:
+    import xgboost.spark
+except ModuleNotFoundError:
+    _xgboost_available = False
+else:
+    _xgboost_available = True
+
+
+def choose_classifier(model_type: str, params: dict[str, Any], dep_var: str):
+    """Given a model type and hyper-parameters for the model, return a
+    classifier of that type with those hyper-parameters, along with a
+    post-classification transformer to run after classification.
+
+    The post-classification transformer standardizes the output of the
+    classifier for further processing. For example, some classifiers create
+    models that output a probability array of [P(dep_var=0), P(dep_var=1)], and
+    the post-classification transformer extracts the single float P(dep_var=1)
+    as the probability for these models.
+
+    Parameters
+    ----------
+    model_type
+        the type of model, which may be random_forest, probit,
+        logistic_regression, decision_tree, gradient_boosted_trees, lightgbm
+        (requires the 'lightgbm' extra), or xgboost (requires the 'xgboost'
+        extra)
+    params
+        a dictionary of hyper-parameters for the model
+    dep_var
+        the dependent variable for the model, sometimes also called the "label"
+
+    Returns
+    -------
+    The classifier and a transformer to be used after classification, as a tuple.
+    """
+    post_transformer = SQLTransformer(statement="SELECT * FROM __THIS__")
+    features_vector = "features_vector"
+    if model_type == "random_forest":
+        classifier = RandomForestClassifier(
+            **params,
+            labelCol=dep_var,
+            featuresCol=features_vector,
+            seed=2133,
+            probabilityCol="probability_array",
+        )
+        post_transformer = SQLTransformer(
+            statement="SELECT *, parseProbVector(probability_array, 1) as probability FROM __THIS__"
+        )
+
+    elif model_type == "probit":
+        classifier = GeneralizedLinearRegression(
+            family="binomial",
+            link="probit",
+            labelCol=dep_var,
+            featuresCol=features_vector,
+            predictionCol="probability",
+        )
+
+    elif model_type == "logistic_regression":
+        classifier = LogisticRegression(
+            **params,
+            featuresCol=features_vector,
+            labelCol=dep_var,
+            predictionCol="prediction",
+            probabilityCol="probability_array",
+        )
+        post_transformer = SQLTransformer(
+            statement="SELECT *, parseProbVector(probability_array, 1) as probability FROM __THIS__"
+        )
+
+    elif model_type == "decision_tree":
+        classifier = DecisionTreeClassifier(
+            **params,
+            featuresCol=features_vector,
+            labelCol=dep_var,
+            probabilityCol="probability_array",
+            seed=2133,
+        )
+        post_transformer = SQLTransformer(
+            statement="SELECT *, parseProbVector(probability_array, 1) as probability FROM __THIS__"
+        )
+
+    elif model_type == "gradient_boosted_trees":
+        classifier = GBTClassifier(
+            **params,
+            featuresCol=features_vector,
+            labelCol=dep_var,
+            seed=2133,
+        )
+        post_transformer = (
+            hlink.linking.transformers.rename_prob_column.RenameProbColumn()
+        )
+    elif model_type == "lightgbm":
+        if not _lightgbm_available:
+            raise ModuleNotFoundError(
+                "To use the 'lightgbm' model type, you need to install the synapseml "
+                "Python package, which provides LightGBM-Spark integration, and "
+                "its dependencies. Try installing hlink with the lightgbm extra: "
+                "\n\n    pip install hlink[lightgbm]"
+            )
+        classifier = synapse.ml.lightgbm.LightGBMClassifier(
+            **params,
+            featuresCol=features_vector,
+            labelCol=dep_var,
+            probabilityCol="probability_array",
+        )
+        post_transformer = SQLTransformer(
+            statement="SELECT *, parseProbVector(probability_array, 1) as probability FROM __THIS__"
+        )
+    elif model_type == "xgboost":
+        if not _xgboost_available:
+            raise ModuleNotFoundError(
+                "To use the experimental 'xgboost' model type, you need to install "
+                "the xgboost library and its dependencies. Try installing hlink with "
+                "the xgboost extra:\n\n    pip install hlink[xgboost]"
+            )
+        classifier = xgboost.spark.SparkXGBClassifier(
+            **params,
+            features_col=features_vector,
+            label_col=dep_var,
+            probability_col="probability_array",
+        )
+        post_transformer = SQLTransformer(
+            statement="SELECT *, parseProbVector(probability_array, 1) as probability FROM __THIS__"
+        )
+    else:
+        raise ValueError(
+            "Model type not recognized! Please check your config, reload, and try again."
+        )
+    return classifier, post_transformer

hlink/linking/core/column_mapping.py

@@ -0,0 +1,53 @@
+# This file is part of the ISRDI's hlink.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/ipums/hlink
+from typing import Any
+
+from pyspark.sql.functions import col, lit
+from pyspark.sql import DataFrame
+import hlink.linking.core.transforms as transforms_core
+
+
+def select_column_mapping(
+    column_mapping: dict[str, Any],
+    df_selected: DataFrame,
+    is_a: bool,
+    column_selects: list[str],
+) -> tuple[DataFrame, list[str]]:
+    name = column_mapping["column_name"]
+    if "override_column_a" in column_mapping and is_a:
+        override_name = column_mapping["override_column_a"]
+        column_select = col(override_name)
+        if "override_transforms" in column_mapping:
+            for transform in column_mapping["override_transforms"]:
+                column_select = transforms_core.apply_transform(
+                    column_select, transform, is_a
+                )
+    elif "override_column_b" in column_mapping and not is_a:
+        override_name = column_mapping["override_column_b"]
+        column_select = col(override_name)
+        if "override_transforms" in column_mapping:
+            for transform in column_mapping["override_transforms"]:
+                column_select = transforms_core.apply_transform(
+                    column_select, transform, is_a
+                )
+    elif "set_value_column_a" in column_mapping and is_a:
+        value_to_set = column_mapping["set_value_column_a"]
+        column_select = lit(value_to_set)
+    elif "set_value_column_b" in column_mapping and not is_a:
+        value_to_set = column_mapping["set_value_column_b"]
+        column_select = lit(value_to_set)
+    elif "transforms" in column_mapping:
+        column_select = col(name)
+        for transform in column_mapping["transforms"]:
+            column_select = transforms_core.apply_transform(
+                column_select, transform, is_a
+            )
+    else:
+        column_select = col(name)
+
+    alias = column_mapping["alias"] if "alias" in column_mapping else name
+
+    column_selects.append(alias)
+    return df_selected.withColumn(alias, column_select), column_selects

hlink/linking/core/comparison.py

@@ -0,0 +1,91 @@
+# This file is part of the ISRDI's hlink.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/ipums/hlink
+from typing import Any
+
+import hlink.linking.core.comparison_feature as comparison_feature_core
+
+
+def get_comparison_leaves(comp: dict[str, Any]) -> list[dict[str, Any]]:
+    comp_leaves = []
+
+    def _get_comp_leaf(comp: dict[str, Any], comp_leaves: list[dict[str, Any]]) -> None:
+        if "comp_a" in comp:
+            _get_comp_leaf(comp["comp_a"], comp_leaves)
+            _get_comp_leaf(comp["comp_b"], comp_leaves)
+
+        else:
+            comp_leaves.append(comp)
+
+    if "comp_a" in comp:
+        _get_comp_leaf(comp["comp_a"], comp_leaves)
+        _get_comp_leaf(comp["comp_b"], comp_leaves)
+
+    elif "secondary" in comp:
+        _get_comp_leaf(comp["threshold_a"], comp_leaves)
+        _get_comp_leaf(comp["threshold_b"], comp_leaves)
+
+    else:
+        _get_comp_leaf(comp, comp_leaves)
+
+    return comp_leaves
+
+
+def generate_comparisons(
+    comp: dict[str, Any], features: list[dict[str, Any]], id_col: str
+) -> str:
+    """Creates the comparison SQL clause given a comparison and a list of comparison features.
+
+    Parameters
+    ----------
+    comp:
+        the config dictionary containing the comparison definition
+    features:
+        the config list containing the comparison features
+    id_col:
+        the id column
+
+    Returns
+    -------
+    The sql clause to be used for comparison filtering after blocking.
+    """
+    if comp != {}:
+        if "comp_a" in comp:
+            comp_a_clause = generate_comparisons(comp["comp_a"], features, id_col)
+            comp_b_clause = generate_comparisons(comp["comp_b"], features, id_col)
+            if comp["operator"] == "AND":
+                return f"""
+                ({comp_a_clause} AND {comp_b_clause})
+                """
+            elif comp["operator"] == "OR":
+                return f"""
+                ({comp_a_clause} OR {comp_b_clause})
+                """
+        elif "secondary" in comp:
+            comp_a = comp["threshold_a"]
+            comp_a_clause = f"{comp_a['feature_name']} >= {comp_a['threshold']}"
+            comp_b = comp["threshold_b"]
+            comp_b_clause = f"{comp_b['feature_name']} >= {comp_b['threshold']}"
+            if comp["operator"] == "AND":
+                return f"({comp_a_clause} AND {comp_b_clause})"
+
+        else:
+            if "column_name" in comp:
+                col = comp["column_name"]
+            else:
+                col = comparison_feature_core.generate_comparison_feature(
+                    [f for f in features if f["alias"] == comp["feature_name"]][0],
+                    id_col,
+                )
+            if "comparison_type" in comp:
+                comp_type = comp["comparison_type"]
+                if comp_type == "threshold":
+                    if comp.get("threshold_expr", False):
+                        return f"{col} {comp['threshold_expr']}"
+                    else:
+                        return f"{col} >= {comp['threshold']}"
+            else:
+                return f"{col}"
+    else:
+        return ""