clip-protocol 2.0.0__py3-none-any.whl

clip_protocol/cli.py

@@ -0,0 +1,55 @@
+import argparse
+import os
+import pandas as pd
+from colorama import Style
+
+from clip_protocol.main.individual_method import run_individual_method  
+from clip_protocol.main.general_method import run_general_method
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the individual method for private frequency estimation.")
+    parser.add_argument("file_path", type=str, help="The path to the input dataset file.")
+    parser.add_argument("output_path", type=str, help="The path to the output where you want the final database to be saved.")
+    args = parser.parse_args()
+
+    if not os.path.exists(args.file_path):
+        raise FileNotFoundError(f"File not found at {args.file_path}")
+    elif not os.path.exists(args.output_path):
+        raise FileNotFoundError(f"Output path not found at {args.output_path}")
+    
+    file_name = os.path.basename(args.file_path)
+    print(f"Processing {Style.BRIGHT}{file_name}{Style.RESET_ALL}")
+    df = pd.read_excel(args.file_path)
+
+    priv_df = run_individual_method(df)
+    output = os.path.join(args.output_path, 'private_database.csv')
+    priv_df.to_csv(output, index=False)
+    print(f"{Style.BRIGHT}Private dataset saved at {args.output_path}{Style.RESET_ALL}")
+
+def main_general():
+    parser = argparse.ArgumentParser(description="Run the individual method for private frequency estimation.")
+    parser.add_argument("file_path", type=str, help="The path to the input dataset file.")
+    parser.add_argument("output_path", type=str, help="The path to the output where you want the final database to be saved.")
+    args = parser.parse_args()
+
+    if not os.path.exists(args.file_path):
+        raise FileNotFoundError(f"File not found at {args.file_path}")
+    elif not os.path.exists(args.output_path):
+        raise FileNotFoundError(f"Output path not found at {args.output_path}")
+    
+    file_name = os.path.basename(args.file_path)
+    print(f"Processing {Style.BRIGHT}{file_name}{Style.RESET_ALL}")
+    df = pd.read_excel(args.file_path)
+
+    priv = run_general_method(df)
+
+    for user, privatized_data in priv.items():
+        output_file = os.path.join(args.output_path, f'{user}.csv')
+        privatized_data.to_csv(args.output_path, index=False)
+        
+    print(f"{Style.BRIGHT}Private datasets saved at {args.output_path}{Style.RESET_ALL}")
+
+
+
+
+

clip_protocol/count_mean/__init__.py

@@ -0,0 +1,18 @@
+"""
+This subpackage contains implementations of algorithms for calculating private count means
+using the Count-Mean-Sketch (CMS) approach. The Count-Mean-Sketch is a probabilistic data
+structure that allows for efficient frequency estimation while providing differential privacy
+guarantees. This subpackage includes both client-side and server-side implementations for
+privacy-preserving data aggregation.
+
+Modules:
+- cms_client_mean.py: Implements the client-side logic for generating private count means.
+- private_cms_client.py: Contains the client-side logic for perturbing data before sending it to the server.
+- private_cms_server.py: Implements the server-side logic for aggregating and analyzing perturbed data.
+
+Main Functions:
+- execute_client: Simulates the client side of the privatized Count-Min Sketch for all elements in the dataset.
+- server_simulator: Simulates the server side of the privatized Count-Min Sketch, processes the privatized data, and estimates frequencies.
+- update_sketch_matrix: Updates the sketch matrix based on the privatized data received from the client.
+- estimate_client: Estimates the frequency of an element based on the private CMS sketch matrix.
+"""

clip_protocol/count_mean/cms_client_mean.py

@@ -0,0 +1,144 @@
+import random
+import numpy as np
+from sympy import primerange
+from rich.progress import Progress
+
+from clip_protocol.utils.utils import generate_hash_functions, display_results
+
+class CMSClient:
+    """
+    A class to represent the Count-Min Sketch (CMS) Client.
+
+    Attributes:
+        df: DataFrame containing the dataset.
+        k: Number of hash functions used in the CMS.
+        m: Size of the sketch matrix.
+        dataset: List of values from the dataset.
+        domain: Unique values in the dataset.
+        N: Total number of elements in the dataset.
+        M: Count-Min Sketch matrix.
+        H: List of hash functions.
+    
+    Methods:
+        client(d):
+            Simulates the client side of the CMS, returning a vector with hash values.
+        update_sketch_matrix(d):
+            Updates the sketch matrix based on the given element.
+        estimate_client(d):
+            Estimates the frequency of an element using the CMS sketch matrix.
+        server_simulator():
+            Simulates the server side of the CMS, processes the data, and estimates frequencies.
+    """
+
+    def __init__(self, k, m, df):
+        """
+        Initializes the CMSClient with the given parameters.
+        """
+        self.df = df
+        self.k = k 
+        self.m = m
+        self.dataset = self.df['value'].tolist()
+        self.domain = self.df['value'].unique().tolist()
+        self.N = len(self.dataset)
+        
+        # Creation of the sketch matrix
+        self.M = np.zeros((self.k, self.m))
+
+        # Definition of the hash family 3 by 3
+        primes = list(primerange(10**6, 10**7))
+        p = primes[random.randint(0, len(primes)-1)]
+        self.H = generate_hash_functions(self.k,p, 3,self.m)
+
+    def client(self, d):
+        """
+        Simulates the client side of the Count-Min Sketch.
+
+        Args:
+            d (element): The element for which the sketch vector is generated.
+
+        Returns:
+            tuple: A tuple containing the sketch vector and the index of the chosen hash function.
+        """
+        j = random.randint(0, self.k-1)
+        v = np.full(self.m, -1)
+        selected_hash = self.H[j]
+        v[selected_hash(d)] = 1
+        return v, j
+   
+    def update_sketch_matrix(self, d):
+        """
+        Updates the sketch matrix based on the given element.
+
+        Args:
+            d (element): The element to be used for updating the sketch matrix.
+        """
+        for i in range (self.k):
+            selected_hash = self.H[i]
+            hash_index = selected_hash(d)
+            self.M[i ,hash_index] += 1
+
+    def estimate_client(self,d):
+        """
+        Estimates the frequency of an element based on the sketch matrix.
+
+        Args:
+            d (element): The element whose frequency is estimated.
+
+        Returns:
+            float: The estimated frequency of the element.
+        """
+        mean = 0
+        for i in range(self.k):
+            selected_hash = self.H[i]
+            mean += self.M[i,selected_hash(d)]
+        return mean/self.k
+    
+    def server_simulator(self):
+        """
+        Simulates the server side of the CMS by processing the dataset 
+        and estimating the frequencies of each element.
+
+        Returns:
+            dict: A dictionary with the elements and their estimated frequencies.
+        """
+        with Progress() as progress:
+            bar = progress.add_task("[cyan]Processing client data...", total=len(self.dataset))
+
+            for d in self.dataset:
+                self.update_sketch_matrix(d)
+                progress.update(bar, advance=1)
+
+            F_estimated = {}
+            bar = progress.add_task("[cyan]Obtaining histogram of estimated frequencies...", total=len(self.domain))
+            for x in self.domain:
+                F_estimated[x] = self.estimate_client(x)
+                progress.update(bar, advance=1)
+        return F_estimated
+
+def run_cms_client_mean(k, m, df):
+    """
+    Runs the Count-Min Sketch algorithm and displays the results.
+
+    Args:
+        k (int): Number of hash functions.
+        m (int): Size of the sketch matrix.
+        df (DataFrame): Dataset to be processed.
+
+    Returns:
+        DataFrame: A table containing the elements and their estimated frequencies.
+    """
+    # Initialize the CMSClient
+    PCMS = CMSClient(k, m, df)
+
+    # Simulate the server side
+    f_estimated = PCMS.server_simulator()
+
+    # Show the results
+    data_table = display_results(df, f_estimated)
+
+    return data_table
+
+
+
+
+  

clip_protocol/count_mean/private_cms_client.py

@@ -0,0 +1,115 @@
+import random
+import numpy as np
+from sympy import primerange
+import pandas as pd
+from rich.progress import Progress
+from numba import njit
+from concurrent.futures import ThreadPoolExecutor
+
+from clip_protocol.utils.utils import generate_hash_functions
+
+@njit
+def bernoulli_vector(epsilon, m):
+    b = np.random.binomial(1, (np.exp(epsilon/2)) / ((np.exp(epsilon/2)) + 1), m)
+    return 2 * b - 1
+
+@njit
+def update_sketch_matrix(M, v, j, epsilon, k, m):
+    c_e = (np.exp(epsilon/2)+1) / ((np.exp(epsilon/2))-1)
+    x = k * ((c_e/2) * v + (1/2) * np.ones_like(v))
+    for i in range (m):
+        M[j,i] += x[i]
+
+class privateCMSClient:
+    def __init__(self, epsilon, k, m, df):
+        self.df = df
+        self.epsilon = epsilon
+        self.k = k
+        self.m = m
+        self.dataset = self.df['value'].tolist()
+        self.domain = self.df['value'].unique().tolist()
+        self.N = len(self.dataset)
+
+        # Creation of the sketch matrix
+        self.M = np.zeros((self.k, self.m))
+
+        # List to store the privatized matrices
+        self.client_matrix = []
+
+        # Definition of the hash family 3 by 3
+        primes = list(primerange(10**6, 10**7))
+        p = primes[random.randint(0, len(primes)-1)]
+        self.H, self.coefs = generate_hash_functions(self.k, p, 3, self.m)
+
+    def client(self, d):
+        j = random.randint(0, self.k-1)
+        v = np.full(self.m, -1)
+        selected_hash = self.H[j]
+        v[selected_hash(d)] = 1
+        b = bernoulli_vector(self.epsilon, self.m)
+        v_aux = v * b
+        return v_aux, j
+
+    def estimate_client(self,d):
+        sum_aux = 0
+        for i in range(self.k):
+            selected_hash = self.H[i]
+            sum_aux += self.M[i, selected_hash(d)]
+        f_estimated = (self.m/(self.m-1))*((sum_aux/self.k)-(self.N/self.m))
+        return f_estimated
+    
+    def execute_client(self):
+        privatized_data = []
+        def process(d):
+            return self.client(d)
+        
+        with Progress() as progress:
+            bar = progress.add_task("Processing client data", total=len(self.dataset))
+            with ThreadPoolExecutor() as executor:
+                for result in executor.map(process, self.dataset):
+                    privatized_data.append(result)
+                    progress.update(bar, advance=1)
+        self.client_matrix = privatized_data
+        return privatized_data
+    
+    def server_simulator(self,privatized_data):
+        with Progress() as progress:
+            bar = progress.add_task('Update sketch matrix', total=len(privatized_data))
+            
+            for v, j in privatized_data:
+                update_sketch_matrix(self.M, v, j, self.epsilon, self.k, self.m)
+                progress.update(bar, advance=1)
+
+            bar = progress.add_task('Estimate frequencies', total=len(self.domain))
+            F_estimated = {}
+            for x in self.domain:
+                F_estimated[x] = self.estimate_client(x)
+                progress.update(bar, advance=1)
+
+        return F_estimated, self.coefs
+
+def run_private_cms_client(k, m, e, df):
+    """
+    Runs the privatized Count-Min Sketch algorithm and displays the results.
+
+    Args:
+        k (int): Number of hash functions.
+        m (int): Size of the sketch matrix.
+        e (float): Privacy parameter.
+        df (DataFrame): Dataset to be processed.
+
+    Returns:
+        tuple: A tuple containing the hash functions, the results table, the error table, the privatized data, and the estimated frequency DataFrame.
+    """
+    # Initialize the private Count-Mean Sketch
+    PCMS = privateCMSClient(e, k, m, df)
+
+    # Client side: process the private data
+    privatized_data = PCMS.execute_client()
+
+    # Simulate the server side
+    f_estimated, coefs = PCMS.server_simulator(privatized_data)
+
+    df_estimated = pd.DataFrame(list(f_estimated.items()), columns=['Element', 'Frequency'])
+
+    return coefs, privatized_data, df_estimated

clip_protocol/count_mean/private_cms_server.py

@@ -0,0 +1,203 @@
+
+import numpy as np
+import pandas as pd
+import os
+from colorama import Fore, Style
+from rich.progress import Progress
+
+from clip_protocol.utils.utils import display_results
+
+class privateCMSServer:
+    """
+    This class represents the server side of the Private Count-Mean Sketch (PCMS).
+    It is responsible for updating the sketch matrix and providing frequency estimations.
+
+    Attributes:
+        df (pandas.DataFrame): The dataset containing the values.
+        epsilon (float): The privacy parameter epsilon.
+        k (int): The number of hash functions.
+        m (int): The size of the sketch.
+        dataset (list): The list of values in the dataset.
+        domain (list): The unique values in the dataset.
+        N (int): The size of the dataset.
+        H (list): The list of hash functions.
+        M (numpy.ndarray): The sketch matrix.
+    """
+    def __init__(self, epsilon, k, m, df, H):
+        """
+        Initializes the privateCMSServer class with the given parameters.
+
+        Args:
+            epsilon (float): The privacy parameter epsilon.
+            k (int): The number of hash functions.
+            m (int): The size of the sketch.
+            df (pandas.DataFrame): The dataset containing the values.
+            H (list): The list of hash functions.
+        """
+        self.df = df
+        self.epsilon = epsilon
+        self.k = k
+        self.m = m
+        self.dataset = self.df['value'].tolist()
+        self.domain = self.df['value'].unique().tolist()
+        self.N = len(self.dataset)
+        self.H = H
+
+        # Creation of the sketch matrix
+        self.M = np.zeros((self.k, self.m))
+    
+    def update_sketch_matrix(self,v,j):
+        """
+        Updates the sketch matrix based on the given privatized data.
+
+        Args:
+            v (numpy.ndarray): The privatized vector.
+            j (int): The index of the hash function used.
+        """
+        c_e = (np.exp(self.epsilon/2)+1) / ((np.exp(self.epsilon/2))-1)
+        x = self.k * ((c_e/2) * v + (1/2) * np.ones_like(v))
+        for i in range (self.m):
+            self.M[j,i] += x[i]
+
+    def execute_server(self,privatized_data):
+        """
+        Executes the server-side operations, including updating the sketch matrix
+        and estimating the frequencies.
+
+        Args:
+            privatized_data (list): The privatized data from the client.
+
+        Returns:
+            dict: A dictionary containing the estimated frequencies for each element.
+        """
+        with Progress() as progress:
+            task = progress.add_task('[cyan]Update sketch matrix', total=len(privatized_data))
+
+            for data in privatized_data:
+                self.update_sketch_matrix(data[0],data[1])
+                progress.update(task, advance=1)
+
+            F_estimated = {}
+            task = progress.add_task('[cyan]Obtaining histogram of estimated frequencies', total=len(self.domain))
+            for x in self.domain:
+                F_estimated[x] = self.estimate_server(x)
+                progress.update(task, advance=1)
+                
+        return F_estimated
+
+    def estimate_server(self,d):
+        """
+        Estimates the frequency of an element based on the current sketch matrix.
+
+        Args:
+            d (any): The element whose frequency is to be estimated.
+
+        Returns:
+            float: The estimated frequency of the element.
+        """
+        sum_aux = 0
+        for i in range(self.k):
+            selected_hash = self.H[i]
+            sum_aux += self.M[i, selected_hash(d)]
+        
+        f_estimated = (self.m/(self.m-1))*((sum_aux/self.k)-(self.N/self.m))
+        return f_estimated
+    
+    def query_server(self, query_element):
+        """
+        Queries the server for the estimated frequency of an element.
+
+        Args:
+            query_element (any): The element to query.
+
+        Returns:
+            float or str: The estimated frequency of the element, or a message if the element is not in the domain.
+        """
+        if query_element not in self.domain:
+            return "Element not in the domain"
+        estimation = self.estimate_server(query_element)
+        return estimation
+    
+def run_private_cms_server(k, m, e, df, H, privatized_data):
+    """
+    Runs the server-side operations for the Private Count-Mean Sketch, including
+    estimating frequencies and querying the server.
+
+    Args:
+        k (int): The number of hash functions.
+        m (int): The size of the sketch.
+        e (float): The privacy parameter epsilon.
+        df (pandas.DataFrame): The dataset containing the values.
+        H (list): The list of hash functions.
+        privatized_data (list): The privatized data from the client.
+    """
+    #Initialize the server Count-Mean Sketch
+    server = privateCMSServer(e, k, m, df, H)
+
+    # Save the privatized data
+    privatized_data_save = pd.DataFrame(privatized_data)
+    
+    # Execute the server
+    f_estimated = server.execute_server(privatized_data)
+
+    # Query the server
+    while True:
+        query = input("Enter a event to query the server or 'exit' to finish: ")
+        if query.lower() == 'exit':
+            break
+        estimation = server.query_server(query)
+        print(f"The estimated frequency of {query} is {estimation:.2f}")
+    
+    return privatized_data_save
+
+def run_private_cms_server_multiuser(k, m, private):
+    """
+    Runs the server-side operations for the Private Count-Mean Sketch,
+    storing a separate server instance for each user.
+
+    Args:
+        k (int): The number of hash functions.
+        m (int): The size of the sketch.
+        e (float): The privacy parameter epsilon.
+        df (pandas.DataFrame): The dataset containing the values.
+        H (list): The list of hash functions.
+        privatized (dict): A dictionary where keys are users and values contain privatized data.
+
+    Returns:
+        dict: A dictionary of servers where each user has its own server instance.
+    """
+
+    user_servers = {}
+
+    with Progress() as progress:
+        task = progress.add_task("[cyan]Initializing servers...", total=len(private))
+
+        for user, data in private.items():
+            progress.update(task, advance=1, description=f"[cyan]Processing user {user}...")
+
+            e = data["e"]
+            privatized_data = data["privatized_data"]
+            H = data["result"]
+
+            df = pd.DataFrame(privatized_data)
+
+            #Initialize the server Count-Mean Sketch
+            server = privateCMSServer(e, k, m, df, H)
+            
+            f_estimated = server.execute_server(privatized_data)
+            
+            user_servers[user] = server
+    
+    print(F"✅ {Fore.GREEN}All user servers initialized.{Style.RESET_ALL}")
+    
+    while True:
+        user_query = input("Enter a user to query or 'exit' to finish: ")
+        if user_query.lower() == 'exit':
+            break
+        if user_query not in user_servers:
+            raise ValueError(f"❌ {Fore.RED}User '{user_query}' not found.{Style.RESET_ALL}")
+        
+        event_query = input(f"Enter an event for user {user_query}: ")
+        estimation = user_servers[user_query].query_server(event_query)
+        print(f"The estimated frequency of '{event_query}' for user '{user_query}' is {estimation:.2f}")
+

clip_protocol/hadamard_count_mean/__init__.py

@@ -0,0 +1,17 @@
+"""
+This subpackage contains implementations of algorithms for calculating private count means
+using the Hadamard Count-Mean-Sketch (HCMS) approach. The Hadamard transform is used to
+efficiently encode and perturb data, ensuring differential privacy while maintaining data
+utility. This subpackage includes both client-side and server-side implementations for
+privacy-preserving data aggregation.
+
+Modules:
+- private_hcms_client.py: Contains the client-side logic for perturbing data using the Hadamard transform.
+- private_hcms_server.py: Implements the server-side logic for aggregating and analyzing perturbed data.
+
+Main Functions:
+- execute_client: Simulates the client side of the privatized Count-Min Sketch for all elements in the dataset.
+- server_simulator: Simulates the server side of the privatized Count-Min Sketch, processes the privatized data, and estimates frequencies.
+- update_sketch_matrix: Updates the sketch matrix based on the privatized data received from the client.
+- estimate_client: Estimates the frequency of an element based on the private CMS sketch matrix.
+"""

clip_protocol/hadamard_count_mean/private_hcms_client.py

@@ -0,0 +1,127 @@
+from sympy import primerange
+import random
+import numpy as np
+from rich.progress import Progress
+import pandas as pd
+from numba import njit
+from concurrent.futures import ThreadPoolExecutor
+
+from clip_protocol.utils.utils import generate_hash_functions
+
+@njit
+def hadamard_matrix(n):
+    if n == 1:
+        return np.array([[1]])
+    else:
+        h_half = hadamard_matrix(n // 2)
+        return np.block([[h_half, h_half], [h_half, -h_half]])
+
+@njit
+def update_sketch_matrix(epsilon, k, M, w, j, l):
+    c_e = (np.exp(epsilon/2)+1) / ((np.exp(epsilon/2))-1)
+    x = k * c_e * w
+    M[j,l] =  M[j,l] + x
+
+@njit
+def traspose_M(M, H):
+    return M @ np.transpose(H)
+
+class privateHCMSClient:
+    def __init__(self, epsilon, k, m, df):
+        self.df = df
+        self.epsilon = epsilon
+        self.k = k
+        self.m = m
+        self.dataset = self.df['value'].tolist()
+        self.domain = self.df['value'].unique().tolist()
+        self.H = hadamard_matrix(self.m)
+        self.N = len(self.dataset)
+
+        # Creation of the sketch matrix
+        self.M = np.zeros((self.k, self.m))
+
+        # List to store the privatized matrices
+        self.client_matrix = []
+
+        # Definition of the hash family 3 by 3
+        primes = list(primerange(10**6, 10**7))
+        p = primes[random.randint(0, len(primes)-1)]
+        self.hashes, self.coeffs = generate_hash_functions(self.k, p, 3, self.m)
+
+    def client(self,d):
+        j = random.randint(0, self.k - 1)
+        v = np.full(self.m, 0)
+        selected_hash = self.hashes[j]
+        v[selected_hash(d)] = 1
+        w = np.dot(self.H, v)
+        l = random.randint(0, self.m-1)
+
+        P_active = np.exp(self.epsilon) / (np.exp(self.epsilon) + 1)
+        if random.random() <= P_active:
+            b = 1
+        else:
+            b = -1
+        return b * w[l],j,l
+
+    def estimate_client(self, d):
+        return (self.m / (self.m-1)) * (1/self.k * np.sum([self.M[i,self.hashes[i](d)] for i in range(self.k)]) - self.N/self.m)
+    
+    def execute_client(self):
+        privatized_data = []
+        def process(d):
+            return self.client(d)
+        
+        with Progress() as progress:
+            task = progress.add_task('Processing client data', total=len(self.dataset))
+            with ThreadPoolExecutor() as executor:
+                for result in executor.map(process, self.dataset):
+                    privatized_data.append(result)
+                    progress.update(task, advance=1)
+        self.client_matrix = privatized_data
+        return privatized_data
+
+    def server_simulator(self, privatized_data):
+        with Progress() as progress:
+            task = progress.add_task('[cyan]Update sketch matrix', total=len(privatized_data))
+            for v, j, w in privatized_data:
+                update_sketch_matrix(self.epsilon, self.k, self.M, v, j, w)
+                progress.update(task, advance=1)
+
+            # Transpose the matrix
+            self.M = traspose_M(self.M, self.H)
+
+            # Estimate the frequencies
+            F_estimated = {}
+            task = progress.add_task('[cyan]Obtaining histogram of estimated frequencies', total=len(self.domain))
+            for x in self.domain:
+                F_estimated[x] = self.estimate_client(x)
+                progress.update(task, advance=1)
+        return F_estimated, self.coeffs
+    
+def run_private_hcms_client(k, m, e, df):
+    """
+    Runs the private Count-Min Sketch client, processes the data, and estimates frequencies on the server.
+
+    Args:
+        k (int): The number of hash functions.
+        m (int): The size of the sketch matrix.
+        e (float): The privacy parameter epsilon for differential privacy.
+        df (pandas.DataFrame): The dataset in DataFrame format.
+
+    Returns:
+        tuple: A tuple containing the hash functions, data table, error table, privatized data, and the estimated frequencies.
+    """
+    # Initialize the client 
+    client = privateHCMSClient(e, k, m, df)
+
+    # Client side: process the private data
+    privatized_data = client.execute_client()
+
+    # Simulate the server side
+    f_estimated, coeffs = client.server_simulator(privatized_data)
+    df_estimated = pd.DataFrame(list(f_estimated.items()), columns=['Element', 'Frequency'])
+
+    return coeffs, privatized_data, df_estimated
+
+
+