synthyverse 0.2.0__tar.gz → 0.2.1__tar.gz

{synthyverse-0.2.0/synthyverse.egg-info → synthyverse-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synthyverse
-Version: 0.2.0
+Version: 0.2.1
 Summary: Synthetic data generation and evaluation library
 Home-page: https://github.com/synthyverse/synthyverse
 Author: Jim Achterberg, Saif Ul Islam, Zia Ur Rehman
@@ -106,24 +106,24 @@ Requires-Dist: xgboost; extra == "eval"
 Requires-Dist: optuna; extra == "eval"
 Requires-Dist: seaborn; extra == "eval"
 Provides-Extra: full
-Requires-Dist: ctgan==0.10.0; extra == "full"
 Requires-Dist: imbalanced-learn; extra == "full"
-Requires-Dist: pandas; extra == "full"
 Requires-Dist: pykeops; extra == "full"
-Requires-Dist: torch; extra == "full"
-Requires-Dist: scikit-learn; extra == "full"
-Requires-Dist: optuna; extra == "full"
-Requires-Dist: einops; extra == "full"
-Requires-Dist: seaborn; extra == "full"
-Requires-Dist: numpy; extra == "full"
-Requires-Dist: scipy; extra == "full"
+Requires-Dist: geomloss; extra == "full"
 Requires-Dist: tqdm; extra == "full"
-Requires-Dist: opacus==1.5.3; extra == "full"
+Requires-Dist: seaborn; extra == "full"
+Requires-Dist: torch-ema; extra == "full"
+Requires-Dist: optuna; extra == "full"
 Requires-Dist: arfpy; extra == "full"
+Requires-Dist: pandas; extra == "full"
 Requires-Dist: mostlyai-engine; extra == "full"
-Requires-Dist: torch-ema; extra == "full"
-Requires-Dist: geomloss; extra == "full"
+Requires-Dist: opacus==1.5.3; extra == "full"
 Requires-Dist: xgboost; extra == "full"
+Requires-Dist: scipy; extra == "full"
+Requires-Dist: einops; extra == "full"
+Requires-Dist: ctgan==0.10.0; extra == "full"
+Requires-Dist: numpy; extra == "full"
+Requires-Dist: torch; extra == "full"
+Requires-Dist: scikit-learn; extra == "full"
 Requires-Dist: synthcity==0.2.12; extra == "full"
 
 <table align="center" border="0">

{synthyverse-0.2.0 → synthyverse-0.2.1}/setup.py

@@ -9,7 +9,7 @@ if scripts_dir not in sys.path:
 
 from scripts.setup_utils import get_extras, read_requirements
 
-version = "0.2.0"
+version = "0.2.1"
 
 # Define extras dynamically from requirements folder
 extras = get_extras()

{synthyverse-0.2.0 → synthyverse-0.2.1}/synthyverse/evaluation/privacy.py

@@ -13,6 +13,7 @@ from sklearn.metrics import (
     r2_score,
     root_mean_squared_error,
     roc_auc_score,
+    roc_curve,
 )
 from sklearn.neighbors import KDTree, NearestNeighbors
 from scipy.stats import gaussian_kde, rankdata
@@ -54,6 +55,32 @@ def lift_at_k(y_true, y_score, k=0.1):
     return float(precision_at_k / prevalence)
 
 
+def tpr_at_fpr(y_true, y_score, max_fpr=0.1):
+    """
+    Return the highest true positive rate achievable at or below max_fpr.
+
+    This is a thresholded attack metric: it measures member recall while
+    constraining the fraction of non-members incorrectly flagged as members.
+    """
+    if not 0 <= max_fpr <= 1:
+        raise ValueError("max_fpr must be in the interval [0, 1].")
+
+    y_true = np.asarray(y_true, dtype=int)
+    y_score = np.nan_to_num(np.asarray(y_score, dtype=float))
+    if len(y_true) != len(y_score):
+        raise ValueError("y_true and y_score must have the same length.")
+    if len(y_true) == 0:
+        return 0.0
+    if not np.any(y_true == 1) or not np.any(y_true == 0):
+        return 0.0
+
+    fpr, tpr, _ = roc_curve(y_true, y_score)
+    valid = fpr <= max_fpr
+    if not np.any(valid):
+        return 0.0
+    return float(np.max(tpr[valid]))
+
+
 class DCR:
     """Distance to Closest Record (DCR) privacy metrics.
 
@@ -114,17 +141,17 @@ class DCR:
                 - "dcr.score": DCR score such that higher scores indicate better privacy
                 - "dcr.train": Proportion closer to train
                 - "dcr.test": Proportion closer to test
-                - "dcr.quantile_002": Proportion closer to train than the 2% train-test distance quantile
-                - "dcr.quantile_005": Proportion closer to train than the 5% train-test distance quantile
+                - "dcr.quantile_002": Proportion closer to train than the 2% test-to-train distance quantile
+                - "dcr.quantile_005": Proportion closer to train than the 5% test-to-train distance quantile
                 - "dcr.nndr_train": Mean NNDR score from synthetic records to train records
                 - "dcr.nndr_train_002": 2% NNDR quantile from synthetic records to train records
                 - "dcr.nndr_train_005": 5% NNDR quantile from synthetic records to train records
-                - "dcr.nndr_test": Mean NNDR baseline score from train records to test records
-                - "dcr.nndr_test_002": 2% NNDR baseline quantile from train records to test records
-                - "dcr.nndr_test_005": 5% NNDR baseline quantile from train records to test records
-                - "dcr.nndr_ratio": Mean pointwise ratio of each synthetic row's train NNDR score to the train-test NNDR baseline of its nearest training row
-                - "dcr.nndr_ratio_002": 2% quantile of pointwise nearest-training-row normalized NNDR ratios
-                - "dcr.nndr_ratio_005": 5% quantile of pointwise nearest-training-row normalized NNDR ratios
+                - "dcr.nndr_test": Mean NNDR score from synthetic records to test records
+                - "dcr.nndr_test_002": 2% NNDR quantile from synthetic records to test records
+                - "dcr.nndr_test_005": 5% NNDR quantile from synthetic records to test records
+                - "dcr.nndr_ratio": Mean pointwise ratio of each synthetic row's train NNDR score to its test NNDR score
+                - "dcr.nndr_ratio_002": 2% quantile of pointwise synthetic train/test NNDR ratios
+                - "dcr.nndr_ratio_005": 5% quantile of pointwise synthetic train/test NNDR ratios
 
         Raises:
             AssertionError: If test set is larger than train set.
@@ -136,37 +163,29 @@ class DCR:
             train
         ), "Test set must be smaller than or equal to train size to compute DCR"
 
+        num_rows_subsample = len(test) if self.subsample_test_size else len(train)
+        if len(sd) < num_rows_subsample:
+            raise ValueError(
+                "Synthetic data must contain at least "
+                f"{num_rows_subsample} rows to compute DCR with the current "
+                "subsampling settings."
+            )
+
         data = gower_like_transform(
-            {"train": train, "test": test, "syn": sd[: len(train)]},
+            {"train": train, "test": test, "syn": sd.iloc[: len(train)]},
             reference_data=train,
             discrete_features=self.discrete_features,
             categorical_fit_data=[train, test, sd],
         )
 
         # Optionally subsample train and synthetic data to match the test size.
-        num_rows_subsample = (
-            len(data["test"]) if self.subsample_test_size else len(data["train"])
-        )
         if len(data["train"]) < 2 or len(data["test"]) < 2 or num_rows_subsample < 2:
             raise ValueError(
                 "DCR with NNDR requires at least two train rows and two test rows "
                 "after applying subsampling."
             )
         num_iterations = int(np.ceil(len(data["train"]) / num_rows_subsample))
-        scores = []
-        closer_to_train = []
-        closer_to_test = []
-        dcr_002 = []
-        dcr_005 = []
-        nndr_train = []
-        nndr_train_002 = []
-        nndr_train_005 = []
-        nndr_test = []
-        nndr_test_002 = []
-        nndr_test_005 = []
-        nndr_ratio = []
-        nndr_ratio_002 = []
-        nndr_ratio_005 = []
+        metric_values = {}
 
         rng = np.random.default_rng(self.random_state)
 
@@ -177,7 +196,6 @@ class DCR:
             query: np.ndarray,
             reference: np.ndarray,
             n_neighbors: int = 1,
-            return_indices: bool = False,
         ):
             if len(reference) < n_neighbors:
                 raise ValueError(
@@ -189,87 +207,78 @@ class DCR:
                 metric="cityblock",
                 n_jobs=-1,
             ).fit(reference)
-            distances, indices = nbrs.kneighbors(query)
+            distances, _ = nbrs.kneighbors(query)
             if n_neighbors == 1:
                 distances = distances.ravel()
-                indices = indices.ravel()
-            if return_indices:
-                return distances, indices
             return distances
 
-        def nearest_distance_ratio(distances: np.ndarray) -> np.ndarray:
+        def safe_ratio(numerator: np.ndarray, denominator: np.ndarray) -> np.ndarray:
             return np.divide(
-                distances[:, 0],
-                distances[:, 1],
-                out=np.ones(len(distances), dtype=float),
-                where=distances[:, 1] > 0,
+                numerator,
+                denominator,
+                out=np.ones(len(numerator), dtype=float),
+                where=denominator > 0,
             )
 
+        def nearest_distance_ratio(distances: np.ndarray) -> np.ndarray:
+            return safe_ratio(distances[:, 0], distances[:, 1])
+
+        def distribution_metrics(prefix: str, values: np.ndarray) -> dict:
+            return {
+                prefix: np.mean(values),
+                f"{prefix}_002": np.quantile(values, 0.02),
+                f"{prefix}_005": np.quantile(values, 0.05),
+            }
+
+        def collect(metrics: dict):
+            for name, value in metrics.items():
+                metric_values.setdefault(name, []).append(value)
+
         for _ in range(num_iterations):
             syn_curr = choose(data["syn"], num_rows_subsample)
             train_curr = choose(data["train"], num_rows_subsample)
             test_curr = choose(data["test"], min(len(data["test"]), num_rows_subsample))
 
-            d_s_tr_neighbors, d_s_tr_indices = nearest_distances(
+            d_s_tr_neighbors = nearest_distances(
                 syn_curr,
                 train_curr,
                 n_neighbors=2,
-                return_indices=True,
             )
             d_s_tr = d_s_tr_neighbors[:, 0]
             # align test set size to never exceed subsampled set size
-            d_s_te = nearest_distances(syn_curr, test_curr)
-            d_tr_te_neighbors = nearest_distances(train_curr, test_curr, n_neighbors=2)
-            d_tr_te = d_tr_te_neighbors[:, 0]
+            d_s_te_neighbors = nearest_distances(syn_curr, test_curr, n_neighbors=2)
+            d_s_te = d_s_te_neighbors[:, 0]
+            d_te_tr = nearest_distances(test_curr, train_curr)
 
             closer_to_train_ = np.mean(d_s_tr < d_s_te)
             closer_to_test_ = 1 - closer_to_train_
-            closer_to_train.append(closer_to_train_)
-            closer_to_test.append(closer_to_test_)
-            score = min(1, closer_to_test_ * 2)
-            scores.append(score)
-            dcr_002.append(np.mean(d_s_tr < np.quantile(d_tr_te, 0.02)))
-            dcr_005.append(np.mean(d_s_tr < np.quantile(d_tr_te, 0.05)))
 
             nndr_train_scores = nearest_distance_ratio(d_s_tr_neighbors)
-            nndr_test_scores = nearest_distance_ratio(d_tr_te_neighbors)
-            nndr_train_ = np.mean(nndr_train_scores)
-            nndr_test_ = np.mean(nndr_test_scores)
-            nndr_train_q002 = np.quantile(nndr_train_scores, 0.02)
-            nndr_train_q005 = np.quantile(nndr_train_scores, 0.05)
-            nndr_test_q002 = np.quantile(nndr_test_scores, 0.02)
-            nndr_test_q005 = np.quantile(nndr_test_scores, 0.05)
-            nndr_train.append(nndr_train_)
-            nndr_train_002.append(nndr_train_q002)
-            nndr_train_005.append(nndr_train_q005)
-            nndr_test.append(nndr_test_)
-            nndr_test_002.append(nndr_test_q002)
-            nndr_test_005.append(nndr_test_q005)
-            nndr_test_scores_matched = nndr_test_scores[d_s_tr_indices[:, 0]]
-            nndr_ratio_scores = np.divide(
-                nndr_train_scores,
-                nndr_test_scores_matched,
-                out=np.ones(len(nndr_train_scores), dtype=float),
-                where=nndr_test_scores_matched > 0,
+            nndr_test_scores = nearest_distance_ratio(d_s_te_neighbors)
+            nndr_ratio_scores = safe_ratio(nndr_train_scores, nndr_test_scores)
+
+            iteration_metrics = {
+                "score": min(1, closer_to_test_ * 2),
+                "train": closer_to_train_,
+                "test": closer_to_test_,
+                "quantile_002": np.mean(d_s_tr < np.quantile(d_te_tr, 0.02)),
+                "quantile_005": np.mean(d_s_tr < np.quantile(d_te_tr, 0.05)),
+            }
+            iteration_metrics.update(
+                distribution_metrics("nndr_train", nndr_train_scores)
+            )
+            iteration_metrics.update(
+                distribution_metrics("nndr_test", nndr_test_scores)
             )
-            nndr_ratio.append(np.mean(nndr_ratio_scores))
-            nndr_ratio_002.append(np.quantile(nndr_ratio_scores, 0.02))
-            nndr_ratio_005.append(np.quantile(nndr_ratio_scores, 0.05))
+            iteration_metrics.update(
+                distribution_metrics("nndr_ratio", nndr_ratio_scores)
+            )
+
+            collect(iteration_metrics)
+
         return {
-            f"{self.name}.score": float(np.mean(scores)),
-            f"{self.name}.train": float(np.mean(closer_to_train)),
-            f"{self.name}.test": float(np.mean(closer_to_test)),
-            f"{self.name}.quantile_002": float(np.mean(dcr_002)),
-            f"{self.name}.quantile_005": float(np.mean(dcr_005)),
-            f"{self.name}.nndr_train": float(np.mean(nndr_train)),
-            f"{self.name}.nndr_train_002": float(np.mean(nndr_train_002)),
-            f"{self.name}.nndr_train_005": float(np.mean(nndr_train_005)),
-            f"{self.name}.nndr_test": float(np.mean(nndr_test)),
-            f"{self.name}.nndr_test_002": float(np.mean(nndr_test_002)),
-            f"{self.name}.nndr_test_005": float(np.mean(nndr_test_005)),
-            f"{self.name}.nndr_ratio": float(np.mean(nndr_ratio)),
-            f"{self.name}.nndr_ratio_002": float(np.mean(nndr_ratio_002)),
-            f"{self.name}.nndr_ratio_005": float(np.mean(nndr_ratio_005)),
+            f"{self.name}.{name}": float(np.mean(values))
+            for name, values in metric_values.items()
         }
 
 
@@ -484,6 +493,9 @@ class MIA(ABC):
         "lift_010",
         "lift_005",
         "lift_001",
+        "tpr_at_fpr_010",
+        "tpr_at_fpr_005",
+        "tpr_at_fpr_001",
     )
 
     def __init__(
@@ -535,6 +547,15 @@ class MIA(ABC):
                 "lift_010": lift_at_k(mia_data.y_eval, membership_scores, 0.10),
                 "lift_005": lift_at_k(mia_data.y_eval, membership_scores, 0.05),
                 "lift_001": lift_at_k(mia_data.y_eval, membership_scores, 0.01),
+                "tpr_at_fpr_010": tpr_at_fpr(
+                    mia_data.y_eval, membership_scores, 0.10
+                ),
+                "tpr_at_fpr_005": tpr_at_fpr(
+                    mia_data.y_eval, membership_scores, 0.05
+                ),
+                "tpr_at_fpr_001": tpr_at_fpr(
+                    mia_data.y_eval, membership_scores, 0.01
+                ),
             }
 
         avg_scores = {
@@ -1017,6 +1038,9 @@ class EnsembleMIA(MIA):
             "lift_010": lift_at_k(y_eval, scores, 0.10),
             "lift_005": lift_at_k(y_eval, scores, 0.05),
             "lift_001": lift_at_k(y_eval, scores, 0.01),
+            "tpr_at_fpr_010": tpr_at_fpr(y_eval, scores, 0.10),
+            "tpr_at_fpr_005": tpr_at_fpr(y_eval, scores, 0.05),
+            "tpr_at_fpr_001": tpr_at_fpr(y_eval, scores, 0.01),
         }
 
     def _component_membership_scores(

{synthyverse-0.2.0 → synthyverse-0.2.1/synthyverse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synthyverse
-Version: 0.2.0
+Version: 0.2.1
 Summary: Synthetic data generation and evaluation library
 Home-page: https://github.com/synthyverse/synthyverse
 Author: Jim Achterberg, Saif Ul Islam, Zia Ur Rehman
@@ -106,24 +106,24 @@ Requires-Dist: xgboost; extra == "eval"
 Requires-Dist: optuna; extra == "eval"
 Requires-Dist: seaborn; extra == "eval"
 Provides-Extra: full
-Requires-Dist: ctgan==0.10.0; extra == "full"
 Requires-Dist: imbalanced-learn; extra == "full"
-Requires-Dist: pandas; extra == "full"
 Requires-Dist: pykeops; extra == "full"
-Requires-Dist: torch; extra == "full"
-Requires-Dist: scikit-learn; extra == "full"
-Requires-Dist: optuna; extra == "full"
-Requires-Dist: einops; extra == "full"
-Requires-Dist: seaborn; extra == "full"
-Requires-Dist: numpy; extra == "full"
-Requires-Dist: scipy; extra == "full"
+Requires-Dist: geomloss; extra == "full"
 Requires-Dist: tqdm; extra == "full"
-Requires-Dist: opacus==1.5.3; extra == "full"
+Requires-Dist: seaborn; extra == "full"
+Requires-Dist: torch-ema; extra == "full"
+Requires-Dist: optuna; extra == "full"
 Requires-Dist: arfpy; extra == "full"
+Requires-Dist: pandas; extra == "full"
 Requires-Dist: mostlyai-engine; extra == "full"
-Requires-Dist: torch-ema; extra == "full"
-Requires-Dist: geomloss; extra == "full"
+Requires-Dist: opacus==1.5.3; extra == "full"
 Requires-Dist: xgboost; extra == "full"
+Requires-Dist: scipy; extra == "full"
+Requires-Dist: einops; extra == "full"
+Requires-Dist: ctgan==0.10.0; extra == "full"
+Requires-Dist: numpy; extra == "full"
+Requires-Dist: torch; extra == "full"
+Requires-Dist: scikit-learn; extra == "full"
 Requires-Dist: synthcity==0.2.12; extra == "full"
 
 <table align="center" border="0">

{synthyverse-0.2.0 → synthyverse-0.2.1}/synthyverse.egg-info/requires.txt

@@ -47,24 +47,24 @@ optuna
 seaborn
 
 [full]
-ctgan==0.10.0
 imbalanced-learn
-pandas
 pykeops
-torch
-scikit-learn
-optuna
-einops
-seaborn
-numpy
-scipy
+geomloss
 tqdm
-opacus==1.5.3
+seaborn
+torch-ema
+optuna
 arfpy
+pandas
 mostlyai-engine
-torch-ema
-geomloss
+opacus==1.5.3
 xgboost
+scipy
+einops
+ctgan==0.10.0
+numpy
+torch
+scikit-learn
 synthcity==0.2.12
 
 [smote]