synthyverse 0.1.0__py3-none-any.whl

synthyverse/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

synthyverse/benchmark/__init__.py

@@ -0,0 +1 @@
+from .benchmark import TabularBenchmark

synthyverse/benchmark/benchmark.py

@@ -0,0 +1,88 @@
+from sklearn.model_selection import train_test_split
+from ..evaluation.eval import MetricEvaluator
+from ..utils.utils import get_generator, free_up_memory
+from ..utils.reproducibility import set_seed
+import pandas as pd
+from time import time
+
+
+class TabularBenchmark:
+    def __init__(
+        self,
+        generator_name: str = "arf",
+        generator_params: dict = {},
+        n_random_splits: int = 1,
+        n_inits: int = 1,
+        n_generated_datasets: int = 1,
+        metrics: list = ["classifier_test", "mle", "dcr"],
+        test_size: float = 0.3,
+    ):
+
+        self.generator_name = generator_name
+        self.generator_params = generator_params
+        self.n_random_splits = n_random_splits
+        self.n_inits = n_inits
+        self.n_generated_datasets = n_generated_datasets
+        self.metrics = metrics
+        self.test_size = test_size
+
+    def run(self, X: pd.DataFrame, target_column: str, discrete_columns: list):
+
+        results = {}
+        generator_ = get_generator(self.generator_name)
+        for split_i in range(self.n_random_splits):
+            results[f"split_{split_i}"] = {}
+
+            # split data according to current seed
+            stratify = None
+            if target_column in discrete_columns:
+                stratify = X[target_column]
+            X_train, X_test = train_test_split(
+                X, stratify=stratify, test_size=self.test_size, random_state=split_i
+            )
+            X_train, X_test = X_train.reset_index(drop=True), X_test.reset_index(
+                drop=True
+            )
+
+            for init_i in range(self.n_inits):
+                results[f"split_{split_i}"][f"init_{init_i}"] = {}
+                set_seed(init_i)
+                generator = generator_(random_state=init_i, **self.generator_params)
+                start_time = time()
+                generator.fit(X_train, discrete_columns)
+                results[f"split_{split_i}"][f"init_{init_i}"]["training_time"] = (
+                    time() - start_time
+                )
+
+                # potentially generate multiple datasets
+                for generated_dataset_i in range(self.n_generated_datasets):
+                    results[f"split_{split_i}"][f"init_{init_i}"][
+                        f"generated_dataset_{generated_dataset_i}"
+                    ] = {}
+                    start_time = time()
+                    X_syn = generator.generate(len(X))
+                    results[f"split_{split_i}"][f"init_{init_i}"][
+                        f"generated_dataset_{generated_dataset_i}"
+                    ] = {}
+                    results[f"split_{split_i}"][f"init_{init_i}"][
+                        f"generated_dataset_{generated_dataset_i}"
+                    ]["inference_time"] = (time() - start_time)
+                    start_time = time()
+                    evaluator = MetricEvaluator(
+                        metrics=self.metrics,
+                        discrete_features=discrete_columns,
+                        target_column=target_column,
+                        random_state=init_i,
+                    )
+                    metric_results = evaluator.evaluate(X_train, X_test, X_syn)
+                    results[f"split_{split_i}"][f"init_{init_i}"][
+                        f"generated_dataset_{generated_dataset_i}"
+                    ]["evaluation_time"] = (time() - start_time)
+                    results[f"split_{split_i}"][f"init_{init_i}"][
+                        f"generated_dataset_{generated_dataset_i}"
+                    ].update(metric_results)
+
+                    # free up memory for next iteration
+                    free_up_memory()
+
+        return results

synthyverse/evaluation/__init__.py

@@ -0,0 +1 @@
+from .eval import MetricEvaluator

synthyverse/evaluation/eval.py

@@ -0,0 +1,120 @@
+from typing import Union
+import pandas as pd
+from ..utils.preprocessing import scale
+
+from .fidelity import (
+    ClassifierTest,
+    AlphaPrecisionBetaRecallAuthenticity,
+    Similarity,
+)
+
+from .utility import MLE
+
+from .privacy import DCR
+
+METRICS = {
+    "classifier_test": ClassifierTest,
+    "mle": MLE,
+    "dcr": DCR,
+    "similarity": Similarity,
+    "prauth": AlphaPrecisionBetaRecallAuthenticity,
+}
+
+
+class MetricEvaluator:
+
+    def __init__(
+        self,
+        metrics: Union[dict, list],
+        discrete_features: list = [],
+        target_column: str = "target",
+        random_state: int = 0,
+    ):
+
+        if isinstance(metrics, list):
+            self.metrics = {metric: {} for metric in metrics}
+        else:
+            self.metrics = metrics
+        self.discrete_features = discrete_features
+        self.target_column = target_column
+        self.random_state = random_state
+
+    def evaluate(
+        self, X_train: pd.DataFrame, X_test: pd.DataFrame, X_syn: pd.DataFrame
+    ):
+        X_train, X_test, X_syn = (
+            X_train.reset_index(drop=True),
+            X_test.reset_index(drop=True),
+            X_syn.reset_index(drop=True),
+        )
+
+        # ensure that we do not evaluate a larger real dataset than synthetic
+        X_train, X_test = X_train[: len(X_syn)], X_test[: len(X_syn)]
+
+        # one hot, label encode, standard scale
+        X_tr_scaled, X_te_scaled, X_syn_scaled = scale(
+            X_train,
+            X_test,
+            X_syn,
+            discrete_features=self.discrete_features,
+        )
+
+        dict_ = {}
+        for metric__ in self.metrics.keys():
+            metric_ = metric__.split("-")[0].strip().lower()
+            metric_cls = METRICS[metric_]
+            # Use class properties to determine which additional information needs to be passed to the metric
+            if hasattr(metric_cls, "needs_discrete_features") and getattr(
+                metric_cls, "needs_discrete_features", False
+            ):
+                self.metrics[metric__]["discrete_features"] = self.discrete_features
+            if hasattr(metric_cls, "needs_target_column") and getattr(
+                metric_cls, "needs_target_column", False
+            ):
+                self.metrics[metric__]["target_column"] = self.target_column
+            if hasattr(metric_cls, "needs_random_state") and getattr(
+                metric_cls, "needs_random_state", False
+            ):
+                self.metrics[metric__]["random_state"] = self.random_state
+
+            metric = metric_cls(**self.metrics[metric__])
+            # Use class property to determine which data to pass
+            data_req = getattr(metric_cls, "data_requirement", None)
+            if data_req == "test":
+                metric_result = metric.evaluate(
+                    X_test,
+                    X_syn[-len(X_test) :],
+                )
+            elif data_req == "train":
+                metric_result = metric.evaluate(
+                    X_train,
+                    X_syn[: len(X_train)],
+                )
+            elif data_req == "test_preprocessed":
+                metric_result = metric.evaluate(
+                    X_te_scaled,
+                    X_syn_scaled[-len(X_test) :],
+                )
+            elif data_req == "train_preprocessed":
+                metric_result = metric.evaluate(
+                    X_tr_scaled,
+                    X_syn_scaled[: len(X_train)],
+                )
+            elif data_req == "train_and_test":
+                metric_result = metric.evaluate(X_train, X_test, X_syn)
+            elif data_req == "train_and_test_preprocessed":
+                metric_result = metric.evaluate(
+                    X_tr_scaled,
+                    X_te_scaled,
+                    X_syn_scaled,
+                )
+            else:
+                raise Exception(
+                    f"Metric {metric_} not (fully) implemented or missing data_requirement property"
+                )
+
+            # add result to dict (note that quantitative metrics have to output a dict, else they won't get added here)
+            if type(metric_result) == dict:
+                dict_.update(metric_result)
+
+        return dict_

synthyverse/evaluation/fidelity.py

@@ -0,0 +1,230 @@
+import pandas as pd
+import numpy as np
+from sklearn.metrics import roc_auc_score
+from xgboost import XGBClassifier
+from ..utils.xgb_utils import get_xgb_tree_method
+from ..utils.oneclass import OneClassLayer
+from ..utils.utils import suppress_print
+import torch
+from sklearn.neighbors import NearestNeighbors
+from sdmetrics.reports.single_table import QualityReport
+
+
+class ClassifierTest:
+    """
+    AUC score of XGB classifier which aims to distinguish synthetic from real data.
+    """
+
+    data_requirement = "train_and_test"
+    needs_discrete_features = True
+    needs_random_state = True
+
+    def __init__(
+        self,
+        discrete_features: list = [],
+        random_state: int = 0,
+    ):
+        super().__init__()
+        self.random_state = random_state
+        self.discrete_features = discrete_features
+
+    def evaluate(
+        self,
+        train: pd.DataFrame,
+        test: pd.DataFrame,
+        sd: pd.DataFrame,
+    ):
+
+        numerical_features = [
+            col for col in train.columns if col not in self.discrete_features
+        ]
+
+        X = pd.concat((train, sd[: len(train)]))
+        X = X.reset_index(drop=True)
+        X[numerical_features] = X[numerical_features].astype(float)
+        X[self.discrete_features] = X[self.discrete_features].astype("category")
+        y = pd.concat(
+            (
+                pd.Series(0, index=list(range(len(train))), name="y"),
+                pd.Series(1, index=list(range(len(train))), name="y"),
+            )
+        )
+        y = y.reset_index(drop=True)
+
+        model = XGBClassifier(
+            tree_method=get_xgb_tree_method(),
+            enable_categorical=True,
+            random_state=self.random_state,
+            max_depth=3,
+        )
+
+        model.fit(X, y)
+
+        X_te = pd.concat((test, sd[-len(test) :]))
+        X_te[numerical_features] = X_te[numerical_features].astype(float)
+        X_te[self.discrete_features] = X_te[self.discrete_features].astype("category")
+        y_te = pd.concat(
+            (
+                pd.Series(0, index=list(range(len(test))), name="y"),
+                pd.Series(1, index=list(range(len(test))), name="y"),
+            )
+        )
+        y_te = y_te.reset_index(drop=True)
+
+        preds = model.predict_proba(X_te)
+        score = roc_auc_score(y_te, preds[:, 1])
+
+        return {f"classifiertest.auc": float(score)}
+
+
+class AlphaPrecisionBetaRecallAuthenticity:
+    """
+    alpha-Precision, Beta-Recall, Authenticity score from the Alaa et al. paper.
+    """
+
+    data_requirement = "train_preprocessed"
+    needs_random_state = True
+
+    def __init__(
+        self,
+        discrete_features: list = [],
+        random_state: int = 0,
+    ):
+        super().__init__()
+        self.random_state = random_state
+        self.discrete_features = discrete_features
+
+    def evaluate(
+        self,
+        rd: pd.DataFrame,
+        sd: pd.DataFrame,
+    ):
+
+        OC_params = {
+            "input_dim": rd.shape[1],
+            "rep_dim": rd.shape[1],
+            "num_layers": 4,
+            "num_hidden": 32,
+            "activation": "ReLU",
+            "dropout_prob": 0.2,
+            "dropout_active": False,
+            "LossFn": "SoftBoundary",
+            "lr": 2e-3,
+            "epochs": 1000,
+            "warm_up_epochs": 20,
+            "train_prop": 1.0,
+            "weight_decay": 2e-3,
+        }
+        OC_hyperparams = {"Radius": 1, "nu": 1e-2}
+        OC_hyperparams["center"] = (
+            torch.ones(OC_params["rep_dim"]) * 10
+        )  # *10 is what is used in synthcity
+        OC_model = OneClassLayer(params=OC_params, hyperparams=OC_hyperparams)
+        OC_model.fit(rd.values, verbosity=True)
+        real = OC_model.predict(rd.values)
+        syn = OC_model.predict(sd.values)
+        emb_center = OC_model.c.detach().cpu().numpy()
+
+        n_steps = 30
+        alphas = np.linspace(0, 1, n_steps)
+
+        Radii = np.quantile(np.sqrt(np.sum((real - emb_center) ** 2, axis=1)), alphas)
+
+        synth_center = np.mean(syn, axis=0)
+
+        alpha_precision_curve = []
+        beta_coverage_curve = []
+
+        synth_to_center = np.sqrt(np.sum((syn - emb_center) ** 2, axis=1))
+
+        nbrs_real = NearestNeighbors(n_neighbors=2, n_jobs=-1, p=2).fit(real)
+        real_to_real, _ = nbrs_real.kneighbors(real)
+
+        nbrs_synth = NearestNeighbors(n_neighbors=1, n_jobs=-1, p=2).fit(syn)
+        real_to_synth, real_to_synth_args = nbrs_synth.kneighbors(real)
+
+        real_to_real = real_to_real[:, 1].squeeze()
+        real_to_synth = real_to_synth.squeeze()
+        real_to_synth_args = real_to_synth_args.squeeze()
+
+        real_synth_closest = syn[real_to_synth_args]
+
+        real_synth_closest_d = np.sqrt(
+            np.sum((real_synth_closest - synth_center) ** 2, axis=1)
+        )
+        closest_synth_Radii = np.quantile(real_synth_closest_d, alphas)
+
+        for k in range(len(Radii)):
+            precision_audit_mask = synth_to_center <= Radii[k]
+            alpha_precision = np.mean(precision_audit_mask)
+
+            beta_coverage = np.mean(
+                (
+                    (real_to_synth <= real_to_real)
+                    * (real_synth_closest_d <= closest_synth_Radii[k])
+                )
+            )
+
+            alpha_precision_curve.append(alpha_precision)
+            beta_coverage_curve.append(beta_coverage)
+
+        authen = real_to_real[real_to_synth_args] < real_to_synth
+        authenticity = np.mean(authen)
+
+        Delta_precision_alpha = 1 - np.sum(
+            np.abs(np.array(alphas) - np.array(alpha_precision_curve))
+        ) / np.sum(alphas)
+
+        Delta_coverage_beta = 1 - np.sum(
+            np.abs(np.array(alphas) - np.array(beta_coverage_curve))
+        ) / np.sum(alphas)
+
+        return {
+            "alphaprecision.oc.score": float(Delta_precision_alpha),
+            "betacoverage.oc.score": float(Delta_coverage_beta),
+            "authenticity.oc.score": float(authenticity),
+        }
+
+
+class Similarity:
+    """
+    Column Shapes and Column Pair Trends from the SDMetrics library.
+    Indicates quality of marginal distributions and correlations in synthetic data, respectively.
+    """
+
+    data_requirement = "train"
+    needs_discrete_features = True
+
+    def __init__(
+        self,
+        discrete_features: list = [],
+    ):
+        super().__init__()
+        self.discrete_features = discrete_features
+
+    @suppress_print
+    def evaluate(
+        self,
+        rd: pd.DataFrame,
+        sd: pd.DataFrame,
+    ):
+        dtypes = [
+            "categorical" if x in self.discrete_features else "numerical"
+            for x in rd.columns
+        ]
+        metadata = {k: {"sdtype": v} for k, v in zip(rd.columns, dtypes)}
+        metadata = {"columns": metadata}
+        metadata["primary_key"] = "index"
+
+        report = QualityReport()
+        report.generate(rd, sd, metadata)
+        scores = report.get_properties()
+
+        return {
+            "similarity.shape": float(
+                scores.loc[scores["Property"] == "Column Shapes", "Score"]
+            ),
+            "similarity.trend": float(
+                scores.loc[scores["Property"] == "Column Pair Trends", "Score"]
+            ),
+        }

synthyverse/evaluation/privacy.py

@@ -0,0 +1,88 @@
+import pandas as pd
+import numpy as np
+from sklearn.metrics import pairwise_distances_argmin_min
+
+
+class DCR:
+    """
+    Distance to Closest Record scores.
+    Indicates closeness of synthetic data to the training data, and an independent holdout set.
+    """
+
+    data_requirement = "train_and_test_preprocessed"
+
+    def __init__(
+        self,
+        estimates: list = [
+            "mean",
+            0.01,
+            0.05,
+            0.1,
+            0.25,
+            0.5,
+        ],
+        batch_size: int = 16000,
+    ):
+        super().__init__()
+        self.estimates = estimates
+        self.batch_size = batch_size
+
+    def _compute_min_distances_batch(
+        self, query_data: pd.DataFrame, reference_data: pd.DataFrame
+    ) -> np.ndarray:
+        """
+        Compute minimum distances between query_data and reference_data in batches.
+
+        Args:
+            query_data: DataFrame containing query points
+            reference_data: DataFrame containing reference points
+
+        Returns:
+            Array of minimum distances for each query point
+        """
+        if self.batch_size is None:
+            # Use original method for small datasets or when batch_size is not specified
+            _, min_distances = pairwise_distances_argmin_min(
+                query_data, reference_data, metric="euclidean"
+            )
+            return min_distances
+
+        min_distances = []
+        n_query = len(query_data)
+
+        for i in range(0, n_query, self.batch_size):
+            end_idx = min(i + self.batch_size, n_query)
+            batch_query = query_data.iloc[i:end_idx]
+
+            _, batch_min_distances = pairwise_distances_argmin_min(
+                batch_query, reference_data, metric="euclidean"
+            )
+            min_distances.extend(batch_min_distances)
+
+        return np.array(min_distances)
+
+    def evaluate(self, train: pd.DataFrame, test: pd.DataFrame, sd: pd.DataFrame):
+
+        sd = sd[: len(train)]
+
+        # Use batch processing if batch_size is specified
+        min_distances_syn = self._compute_min_distances_batch(sd, train)
+        min_distances_test = self._compute_min_distances_batch(test, train)
+
+        dictionary = {}
+
+        for estimate in self.estimates:
+            if estimate == "mean":
+                score_train = min_distances_syn.mean()
+                score_test = min_distances_test.mean()
+            else:
+                score_train = np.quantile(min_distances_syn, estimate)
+                score_test = np.quantile(min_distances_test, estimate)
+            dictionary.update(
+                {
+                    f"dcr.train.{estimate}": float(score_train),
+                    f"dcr.test.{estimate}": float(score_test),
+                }
+            )
+
+        return dictionary

synthyverse/evaluation/utility.py

@@ -0,0 +1,115 @@
+import pandas as pd
+import numpy as np
+from sklearn.metrics import roc_auc_score, r2_score
+from xgboost import XGBClassifier, XGBRegressor
+from sklearn.preprocessing import LabelEncoder
+from ..utils.xgb_utils import get_xgb_tree_method
+
+
+class MLE:
+    """
+    Machine Learning Efficacy from a XGB classifier.
+    AUC score for discrete target columns, R^2 score for continuous target columns.
+    """
+
+    data_requirement = "train_and_test"
+    needs_discrete_features = True
+    needs_target_column = True
+    needs_random_state = True
+
+    def __init__(
+        self,
+        target_column: str = "target",
+        discrete_features: list = [],
+        random_state: int = 0,
+        train_set: str = "synthetic",
+    ):
+        super().__init__()
+        self.random_state = random_state
+        self.discrete_features = discrete_features
+        self.target_column = target_column
+        self.train_set = train_set
+
+    def evaluate(
+        self,
+        train: pd.DataFrame,
+        test: pd.DataFrame,
+        sd: pd.DataFrame,
+    ):
+
+        y_tr = train[self.target_column]
+        y_te = test[self.target_column]
+        y_s = sd[self.target_column]
+        x_tr = train.drop(columns=[self.target_column])
+        x_te = test.drop(columns=[self.target_column])
+        x_s = sd.drop(columns=[self.target_column])
+
+        numerical_features = [
+            col for col in train.columns if col not in self.discrete_features
+        ]
+        discrete_features = [
+            col for col in self.discrete_features if col != self.target_column
+        ]
+
+        x_tr[numerical_features], x_te[numerical_features], x_s[numerical_features] = (
+            x_tr[numerical_features].astype(float),
+            x_te[numerical_features].astype(float),
+            x_s[numerical_features].astype(float),
+        )
+        x_tr[discrete_features], x_te[discrete_features], x_s[discrete_features] = (
+            x_tr[discrete_features].astype("category"),
+            x_te[discrete_features].astype("category"),
+            x_s[discrete_features].astype("category"),
+        )
+
+        if self.target_column in self.discrete_features:
+            le = LabelEncoder()
+            le.fit(pd.concat((y_tr, y_te, y_s)))
+            y_tr = le.transform(y_tr)
+            y_te = le.transform(y_te)
+            y_s = le.transform(y_s)
+            model = XGBClassifier(
+                tree_method=get_xgb_tree_method(),
+                enable_categorical=True,
+                random_state=self.random_state,
+                max_depth=3,
+            )
+        else:
+            model = XGBRegressor(
+                tree_method=get_xgb_tree_method(),
+                enable_categorical=True,
+                random_state=self.random_state,
+                max_depth=3,
+            )
+
+        if self.train_set == "synthetic":
+            model.fit(x_s[: len(x_tr)], y_s[: len(x_tr)])
+            score = self._get_score(y_te, x_te, model)
+        else:
+            model.fit(x_tr, y_tr)
+            score = self._get_score(y_s[-len(x_te) :], x_s[-len(x_te) :], model)
+
+        # also add trtr score
+        model.fit(x_tr, y_tr)
+        score_trtr = self._get_score(y_te, x_te, model)
+
+        return {
+            f"mle.train-{self.train_set}-test-{'real' if self.train_set == 'synthetic' else 'synthetic'}.{'auc' if self.target_column in self.discrete_features else 'r2'}": float(
+                score
+            ),
+            f"mle.train-real-test-real.{'auc' if self.target_column in self.discrete_features else 'r2'}": float(
+                score_trtr
+            ),
+        }
+
+    def _get_score(self, y_te, X_te, model):
+        if self.target_column in self.discrete_features:
+            preds = model.predict_proba(X_te)
+            if np.unique(y_te).shape[0] > 2:
+                score = roc_auc_score(y_te, preds, multi_class="ovr", average="micro")
+            else:
+                score = roc_auc_score(y_te, preds[:, 1])
+        else:
+            preds = model.predict(X_te)
+            score = r2_score(y_te, preds)
+        return score

synthyverse/generators/__init__.py

@@ -0,0 +1,23 @@
+from .base import BaseGenerator
+
+try:
+    from .arf_generator import ARFGenerator
+except ImportError:
+    ARFGenerator = None
+
+try:
+    from .bn_generator import BNGenerator
+except ImportError:
+    BNGenerator = None
+
+try:
+    from .ctgan_generator import CTGANGenerator
+except ImportError:
+    CTGANGenerator = None
+
+try:
+    from .tvae_generator import TVAEGenerator
+except ImportError:
+    TVAEGenerator = None
+
+all_generators = [ARFGenerator, BNGenerator, CTGANGenerator, TVAEGenerator]

synthyverse/generators/arf_generator/__init__.py

@@ -0,0 +1 @@
+from .arf import ARFGenerator