BenchmarkDPFair 0.1.0__tar.gz

benchmarkdpfair-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Vinicius Gabriel Angelozzi Verona de Resende, Héber Hwang Arcolezi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchmarkdpfair-0.1.0/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: BenchmarkDPFair
+Version: 0.1.0
+Summary: A differentially private data synthesizer and fairness intervention benchmark framework
+Author-email: Vinicius Gabriel Angelozzi Verona de Resende <verona.projects@tutanota.com>, Héber Hwang Arcolezi <heber.hwang-arcolezi@etsmtl.ca>
+License: MIT License
+        
+        Copyright (c) 2025 Vinicius Gabriel Angelozzi Verona de Resende, Héber Hwang Arcolezi
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/vinicius-verona/dp-fair-intervention-benchmark
+Project-URL: Issues, https://github.com/vinicius-verona/dp-fair-intervention-benchmark/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aif360>=0.6.1
+Requires-Dist: aif360[inFairness]>=0.6.1
+Requires-Dist: fairlearn>=0.12.0
+Requires-Dist: inFairness>=0.2.3
+Requires-Dist: matplotlib>=3.9.4
+Requires-Dist: matplotlib-inline>=0.1.7
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: scikit-learn>=1.6.1
+Requires-Dist: scipy>=1.13.1
+Requires-Dist: smartnoise-sql>=1.0.6
+Requires-Dist: smartnoise-synth>=1.0.5
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: ucimlrepo>=0.0.7
+Requires-Dist: xgboost>=2.1.1
+Requires-Dist: tensorflow>=2.19.0
+Requires-Dist: tensorflow-io-gcs-filesystem>=0.37.1
+Requires-Dist: cvxpy>=1.6.5
+Requires-Dist: jax<0.5,>=0.4.30
+Requires-Dist: jaxlib<0.5,>=0.4.30
+Requires-Dist: chex>=0.1.87
+Dynamic: license-file
+
+# DP+Fair Benchmarking Framework
+
+This repository provides a Python framework for **benchmarking fairness mechanisms** on **Differentially Private Synthetic Data**.  
+
+
+---
+
+## Features
+
+- ⚡ Simple, reproducible setup for benchmarking algorithms  
+- 🧩 Flexible API to plug in any classifier implementing `fit`, `predict`, and `predict_proba`  
+- 📊 Pre-offered datasets included under `data/`  
+- 🔬 Configurable experiment settings: dataset schema, dataset synthesizer, seeds, privacy-budget, input/outputs, classifier, data pre-processing.  
+
+---
+
+## Installation
+
+To install, clone the repository and install dependencies:
+
+```bash
+git clone https://github.com/vinicius-verona/dp-fair-intervention-benchmark.git
+cd dp-fair-intervention-benchmark
+pip install -e .
+```
+
+Alternatively, you can install from **PyPI** (Yet to be made available):
+
+```bash
+pip install dp-fair-intervention-benchmark
+````
+
+---
+
+## Repository Structure
+
+```
+├── data/         # Pre-offered datasets
+├── src/          # Core source code
+├── examples/     # Some demo
+├── tests/        # Unit tests
+└── README.md
+```
+
+---
+
+## Quick Start
+
+Here is a minimal usage example:
+
+```python
+from BenchmarkDPFair.DataGenerator import generate_data, DatasetGeneratorConfig
+from BenchmarkDPFair.Benchmark import BenchmarkDatasetConfig, BenchmarkInfo
+
+from sklearn.ensemble import RandomForestClassifier
+
+# Generate Data
+data_conf = DatasetGeneratorConfig(
+    name = "Adult",
+    target= "...",
+    synthesizer = "aim",
+    root_dir="./data",
+    sensitive_attr = "...",
+    categorical_cols = [...],
+    sensitive_cols = [...],
+    privacy_budgets=[...],
+    binary_encoder=...
+)
+
+generate_data("adult.csv", data_conf, verbose=True) # Saves as CSV
+
+# Dataset configuration
+benchmark_config = BenchmarkInfo(
+    dp_method="aim",
+    output_dir="./data/Adult/output/",
+    seeds = [...],
+    eps = [...]
+)
+
+benchmark_dataset = BenchmarkDatasetConfig(
+    name = "Adult",
+    target= "income",
+    root_dir="./data",
+    sensitive_attr = "...",
+    index_col="...",
+    categorical_cols = [...],
+    sensitive_cols = [...],
+)
+
+benchmark(benchmark_info=benchmark_config, data_conf=benchmark_dataset)
+```
+
+More detailed examples can be found in the [`example/`](example/) directory.
+
+---
+
+## License
+
+License: **MIT**
+
+---
+
+## Contributing
+
+Contributions are welcome:
+
+* Open an issue for bug reports or feature requests
+* Submit a pull request to the `main` branch for code contributions
+
+---

benchmarkdpfair-0.1.0/README.md

@@ -0,0 +1,110 @@
+# DP+Fair Benchmarking Framework
+
+This repository provides a Python framework for **benchmarking fairness mechanisms** on **Differentially Private Synthetic Data**.  
+
+
+---
+
+## Features
+
+- ⚡ Simple, reproducible setup for benchmarking algorithms  
+- 🧩 Flexible API to plug in any classifier implementing `fit`, `predict`, and `predict_proba`  
+- 📊 Pre-offered datasets included under `data/`  
+- 🔬 Configurable experiment settings: dataset schema, dataset synthesizer, seeds, privacy-budget, input/outputs, classifier, data pre-processing.  
+
+---
+
+## Installation
+
+To install, clone the repository and install dependencies:
+
+```bash
+git clone https://github.com/vinicius-verona/dp-fair-intervention-benchmark.git
+cd dp-fair-intervention-benchmark
+pip install -e .
+```
+
+Alternatively, you can install from **PyPI** (Yet to be made available):
+
+```bash
+pip install dp-fair-intervention-benchmark
+````
+
+---
+
+## Repository Structure
+
+```
+├── data/         # Pre-offered datasets
+├── src/          # Core source code
+├── examples/     # Some demo
+├── tests/        # Unit tests
+└── README.md
+```
+
+---
+
+## Quick Start
+
+Here is a minimal usage example:
+
+```python
+from BenchmarkDPFair.DataGenerator import generate_data, DatasetGeneratorConfig
+from BenchmarkDPFair.Benchmark import BenchmarkDatasetConfig, BenchmarkInfo
+
+from sklearn.ensemble import RandomForestClassifier
+
+# Generate Data
+data_conf = DatasetGeneratorConfig(
+    name = "Adult",
+    target= "...",
+    synthesizer = "aim",
+    root_dir="./data",
+    sensitive_attr = "...",
+    categorical_cols = [...],
+    sensitive_cols = [...],
+    privacy_budgets=[...],
+    binary_encoder=...
+)
+
+generate_data("adult.csv", data_conf, verbose=True) # Saves as CSV
+
+# Dataset configuration
+benchmark_config = BenchmarkInfo(
+    dp_method="aim",
+    output_dir="./data/Adult/output/",
+    seeds = [...],
+    eps = [...]
+)
+
+benchmark_dataset = BenchmarkDatasetConfig(
+    name = "Adult",
+    target= "income",
+    root_dir="./data",
+    sensitive_attr = "...",
+    index_col="...",
+    categorical_cols = [...],
+    sensitive_cols = [...],
+)
+
+benchmark(benchmark_info=benchmark_config, data_conf=benchmark_dataset)
+```
+
+More detailed examples can be found in the [`example/`](example/) directory.
+
+---
+
+## License
+
+License: **MIT**
+
+---
+
+## Contributing
+
+Contributions are welcome:
+
+* Open an issue for bug reports or feature requests
+* Submit a pull request to the `main` branch for code contributions
+
+---

benchmarkdpfair-0.1.0/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "BenchmarkDPFair"
+version = "0.1.0"
+description = "A differentially private data synthesizer and fairness intervention benchmark framework"
+authors = [
+  { name="Vinicius Gabriel Angelozzi Verona de Resende", email="verona.projects@tutanota.com" },
+  { name="Héber Hwang Arcolezi", email="heber.hwang-arcolezi@etsmtl.ca" }
+]
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+  "aif360>=0.6.1",
+  "aif360[inFairness]>=0.6.1",
+  "fairlearn>=0.12.0",
+  "inFairness>=0.2.3",
+  "matplotlib>=3.9.4",
+  "matplotlib-inline>=0.1.7",
+  "numpy>=1.26.4",
+  "pandas>=2.2.3",
+  "scikit-learn>=1.6.1",
+  "scipy>=1.13.1",
+  "smartnoise-sql>=1.0.6",
+  "smartnoise-synth>=1.0.5",
+  "tabulate>=0.9.0",
+  "ucimlrepo>=0.0.7",
+  "xgboost>=2.1.1",
+  "tensorflow>=2.19.0",
+  "tensorflow-io-gcs-filesystem>=0.37.1",
+  "cvxpy>=1.6.5",
+  "jax>=0.4.30,<0.5",
+  "jaxlib>=0.4.30,<0.5",
+  "chex>=0.1.87"
+
+]
+license = { file = "LICENSE" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.urls]
+Homepage = "https://github.com/vinicius-verona/dp-fair-intervention-benchmark"
+Issues = "https://github.com/vinicius-verona/dp-fair-intervention-benchmark/issues"

benchmarkdpfair-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

benchmarkdpfair-0.1.0/src/BenchmarkDPFair/Benchmark/__init__.py

@@ -0,0 +1,4 @@
+from .benchmark import BenchmarkInfo,benchmark
+from .dataconf import BenchmarkDatasetConfig
+
+__all__ = ["BenchmarkInfo", "BenchmarkDatasetConfig", "benchmark"]

benchmarkdpfair-0.1.0/src/BenchmarkDPFair/Benchmark/benchmark.py

@@ -0,0 +1,282 @@
+import os
+import warnings
+import pandas as pd
+import numpy as np
+import inspect
+
+from typing import Callable, List, Any, Optional, Tuple, Union
+from sklearn.model_selection import train_test_split
+from tabulate import tabulate
+
+from .dataconf import BenchmarkDatasetConfig
+from .utils.types import FloatOrTuple, DFTuple
+from .utils.verifiers import check_data_loader, check_splitdata, check_target, read_verification, check_dict
+
+from .utils.benchmark import Benchmark
+from .utils.auxiliar import save_experiment
+
+DEFAULT_SEEDS : List[float]= [5,42,253,4112,32645,602627,153073,53453,178753,243421,767707,113647,796969,553067,96797,133843,6977,460403,126613,583879]
+DEFAULT_EPS  : List[float] = [0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 3, 5, 10, 15, 20]
+DP_ALGORITHM  : str = ""
+
+class BenchmarkInfo:
+    def __init__(self, dp_method:str, output_dir: str, data_loader: Optional[Callable[..., DFTuple]] = None, dlkwargs: Union[dict, set] = {},
+                 split_data: Optional[FloatOrTuple] = None, normalize: bool = True, seeds: List[float] = DEFAULT_SEEDS,
+                 eps: List[Union[float,int]] = DEFAULT_EPS, classifier: Any = None, classifier_kwargs: Optional[Union[dict,set]] = None):
+        """
+        Set of possible confiigurations for the Benchmark experiments. 
+
+        **In case you do not use our own generator, read the documentation first to understand how the benchmark expects the data to be organized.**
+
+        Parameters
+        ----------
+        dp_method : str
+            Which DP symthetic data generator was used
+        output_dir : str
+            Directory to save the experiment logs and metrics.
+        data_loader : Callable, optional
+            In case a new data loader needs to be used, refer to the documentation to understand the default data loader's behaviour. data_loader must accept seed as an argument and also kwargs.
+        dlkwargs : dict | set, optional
+            Custom parameters for the data loader.
+        split_data : FloatOrTuple, optional
+            Split distributions used while loading data. If not provided, the final distributions are **0.6, 0.2 and 0.2**, which is `split_data = (0.4, 0.5)`.
+        normalize : bool, optional
+            Allow MinMax normalization of the data. Default is **True**.
+        seeds : List[int], optional
+            List of seeds for the benchmark. Used to increase reproducibility.
+        eps : List[float|int], optional
+            List of DP epsilons (privacy budget) analysed during the benchmark.
+        classifier : Any, optional
+            Custom classifier. **Must implement fit, predict and predict_proba**. Default is [XGBoost](https://xgboost.readthedocs.io/en/stable/).
+        classifier_kwargs : dict | set, optional
+            Custom parameters for the classifier.
+        """
+        
+        self.dp_method    = dp_method
+        self.output_dir   = output_dir
+        self.normalize    = normalize
+        self.seeds = seeds
+        self.eps   = eps
+
+        global DP_ALGORITHM
+        DP_ALGORITHM = self.dp_method 
+
+        check_splitdata(split_data)
+        self.split = split_data
+
+        # Wrap user-supplied function with enforcement
+        self.data_loader = check_data_loader(data_loader) if data_loader is not None else self.__data_loader
+        self.custom_loader = False if data_loader is None else True
+        self.dlkwargs = dlkwargs
+
+        self.classifier = classifier
+        self.classifier_kwargs = classifier_kwargs
+
+    def dataloader(self, **kwargs) -> DFTuple:
+        """
+        Data loader, by default assumes that within the `baseline_dir` there exists a CSV file with the name set in `filename` parameter.
+        
+        If the `split_data` has been set before, it will look for the file mentioned and split it into three sets following the provided distribution.
+        
+        The split happens sequentially, if two values has been provided to split, the first split (train+test) happens normally, and then the test set is split following the second distribution.
+        
+        If only one number has been provided and no test directory found, the split happens sequentially following the distribution of the test set.
+      
+        **Please refer to the documentation to understand how the default dataloader expects the directory structure to be like.**
+
+        Parameters
+        ----------
+        data_conf : DatasetConf
+            Configuration of the desired dataset.
+        filename : str
+            The name of the CSV file to load. 
+        seed : int
+            The current seed used to load the file and split the data.
+        verbose : bool, optional
+            If `true` prints information on the laoded dataset.
+        extra_processing : Callable, optional
+            Custom (users) porcessing function applied to loaded data. Will be called using kwargs and the loaded data as arguments.
+        kwargs : Any, optional,
+            If an extra processing function is provided, will be forwarded while calling, with the loaded dataset.
+        
+        Returns
+        ----------
+        Three tuple[pd.DataFrame, pd.DataFrame]
+            - A 2-tuple of pandas DataFrames `(X, y)`.
+        """
+        return self.data_loader(**kwargs)
+
+
+    @check_data_loader
+    def __data_loader(self, data_conf: BenchmarkDatasetConfig, filename: str, seed: int,  **kwargs) -> DFTuple:
+        return _load_data(data_conf, filename, seed, split=self.split, **kwargs)
+
+
+def _load_data(data_conf: BenchmarkDatasetConfig, filename: str, seed: int, epsilon: Optional[float] = None, 
+               verbose: bool=True, split: Optional[FloatOrTuple] = None, extra_processing: Optional[Callable] = None, **kwargs) -> DFTuple:
+    
+    if verbose:
+        print(f"** Loading dataset {data_conf.name.upper()} **")
+    
+    if split is None:
+        split = (0.4, 0.5)
+
+    base, ext = os.path.splitext(filename)
+    base_pattern = base.rsplit("_", 1)
+
+    if (os.path.dirname(filename)):
+        test_path = os.path.dirname(os.path.dirname(filename)) + "DP-dataset-test/"
+    else:
+        test_path = f"{data_conf.dir}/{data_conf.name}/{DP_ALGORITHM}/DP-dataset-test/"
+        filename = f"{data_conf.dir}/{data_conf.name}/{DP_ALGORITHM}/DP-dataset-{f'epsilon-{epsilon}' if epsilon is not None else 'train'}/{filename}"
+
+    test_filename = f"{base_pattern[0]}_test{ext}"
+
+    cols = list(dict.fromkeys(data_conf.usecols + [data_conf.index_col] if data_conf.index_col else data_conf.usecols))
+    ds = pd.read_csv(filename, usecols=lambda col: col in cols)
+
+    if data_conf.index_col:
+        ds.set_index(data_conf.index_col, inplace=True)
+    
+    # Verify if data was read successfully
+    read_verification(ds, data_conf.usecols)
+
+    # Apply extra processing to dataset if the user wants it
+    if extra_processing is not None:
+        extra_processing(ds, **kwargs)
+
+    # Ensure all dataset is numerical
+    for col in data_conf.categorical_cols:
+        if not pd.api.types.is_numeric_dtype(ds[col]):
+            ds[col] = ds[col].astype('category').cat.codes # Int encode
+
+    X = ds.drop(columns=[data_conf.target])
+    y = ds[data_conf.target]
+
+    # Split data
+    if not os.path.exists(test_path) or not os.path.exists(test_path + "/" + test_filename):
+        if verbose:
+            train_split_distrib = 1 - split[0] if isinstance(split, Tuple) else split
+            val_split_distrib = split[0] * (1 - split[1]) if isinstance(split, Tuple) else split * (1 - split)
+            test_split_distrib = split[0] * split[1] if isinstance(split, Tuple) else split * split
+            print(f"[WARN] Test directory and/or file with test set not found, the provided {filename} will be split into three sets with distributions {(train_split_distrib, val_split_distrib, test_split_distrib)}.")
+            print(f"       This is the path we are looking for: {test_path + '/' + test_filename}.\n")
+
+        # No test path found, so split the data from filename
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=split[0] if isinstance(split, Tuple) else split, random_state=seed)
+        X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=split[1] if isinstance(split, Tuple) else split, random_state=seed)
+
+    else:
+        X_train = X
+        y_train = y
+
+        test_ds = pd.read_csv(test_path + "/" + test_filename, usecols=lambda col: col in cols)
+
+        if data_conf.index_col:
+            test_ds.set_index(data_conf.index_col, inplace=True)
+    
+        # Verify if data was read successfully
+        read_verification(test_ds, data_conf.usecols)
+
+        # Apply extra processing to dataset if the user wants it
+        if extra_processing is not None:
+            extra_processing(test_ds, **kwargs)
+            
+        X_test = test_ds.drop(columns=[data_conf.target])
+        y_test = test_ds[data_conf.target]
+
+        if isinstance(split, Tuple):
+            print(f"[WARN] You provided a tuple {split} of splitting distribution and a test directory and file has been found in {test_path}, the second value of the tuple will be used.\n")
+
+        X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=split[1] if isinstance(split, Tuple) else split, random_state=seed)
+
+    if verbose:
+        data = [
+            ["X_train", X_train.shape],
+            ["X_val",   X_val.shape],
+            ["X_test",  X_test.shape],
+            ["y_train", y_train.shape],
+            ["y_val",   y_val.shape],
+            ["y_test",  y_test.shape],
+        ]
+        print("\n#### Data  Information ####")
+        print(tabulate(data, headers=["Dataset", "Shape"], tablefmt="github"))
+        print("###########################\n")
+
+    # Check that the target column is binary
+    check_target(y_train, data_conf.target)
+    check_target(y_val, data_conf.target)
+    check_target(y_test, data_conf.target)
+
+    return (X_train, y_train), (X_val, y_val), (X_test, y_test)
+
+
+############# Experiments #############
+def _experiment(seed, dataset_conf: BenchmarkDatasetConfig, benchmark_info: BenchmarkInfo, savefile):
+    np.random.seed(seed)
+    output_dir = f"{benchmark_info.output_dir}/{dataset_conf.name}/{benchmark_info.dp_method}/results/"
+
+    print(f"\n*********************** Fair-only - seed = {seed} ***********************\n")
+    extra_kwargs = {
+        "data_conf": dataset_conf,
+        "filename": dataset_conf.name + f"_split_dataset_seed_{seed}_train.csv",
+        "custom_loader": benchmark_info.custom_loader,
+        "epsilon": None,
+        "seed": seed,
+        "classifier": benchmark_info.classifier,
+        "classifier_kwargs": benchmark_info.classifier_kwargs
+    }
+        
+    original_experiment = Benchmark(
+        name="baseline", data_loader=benchmark_info.data_loader, 
+        normalize=benchmark_info.normalize, seed=seed, dlkwargs=benchmark_info.dlkwargs, ekwargs = extra_kwargs
+    )
+    original_experiment.run()
+
+    save_experiment(original_experiment, seed, filename=savefile, path=output_dir,synth=benchmark_info.dp_method)
+
+    del original_experiment
+
+    for epsilon in benchmark_info.eps:
+        print(f"\n*********************** DP & DP+Fair | ε={epsilon} ***********************\n")
+        extra_kwargs = {
+            "data_conf": dataset_conf,
+            "filename": dataset_conf.name + f"_split_dataset_seed_{seed}_epsilon-{epsilon}.csv",
+            "custom_loader": benchmark_info.custom_loader,
+            "epsilon": epsilon,
+            "seed": seed,
+            "classifier": benchmark_info.classifier,
+            "classifier_kwargs": benchmark_info.classifier_kwargs
+        }
+        dp_experiment = Benchmark(
+            name="dp", data_loader=benchmark_info.data_loader, 
+            normalize=benchmark_info.normalize, seed=seed, dlkwargs=benchmark_info.dlkwargs, ekwargs=extra_kwargs
+        )
+        dp_experiment.run()
+
+        save_experiment(dp_experiment, seed, epsilon, filename=savefile, path=output_dir,synth=benchmark_info.dp_method)
+
+        del dp_experiment.data_loader, dp_experiment
+
+
+def benchmark(data_conf: BenchmarkDatasetConfig, benchmark_info: BenchmarkInfo):
+    """
+    Execute benchmark of Fairness interventions on models trained on original data and differentially private synthetic data.
+    
+    **The results obtained are output into a csv file in the defined output directory.**
+
+    Parameters
+    -----------
+    data_conf: BenchmarkDatasetConfig
+        Configurations on the dataset used
+    
+    benchmark_info: BenchmarkInfo
+        Configurations about the experiments
+    """
+    
+    print(f"Running DP Benchmark on dataset: '{data_conf.name}' with target: '{data_conf.target}' and sensitive attribute: '{data_conf.sensitive_attr}'")
+
+    savefile = f"benchmark_results_seeds_{'_'.join(str(seed) for seed in benchmark_info.seeds)}_eps_{'_'.join(str(e) for e in benchmark_info.eps)}_synth_{benchmark_info.dp_method}.csv"
+
+    for seed in benchmark_info.seeds:
+        _experiment(seed, data_conf, benchmark_info, savefile)