CtrlRound 0.1.0__py3-none-any.whl

CtrlRound/CtrlRound.py

@@ -0,0 +1,213 @@
+import pandas as pd
+from itertools import combinations
+import functools
+from time import perf_counter
+from .best_first_search import best_first_search
+from .distance import define_margin_distance, define_interior_distance
+
+def agg_by(df:pd.DataFrame, by, var, id):
+    #aggregate a grouped dataframe
+    if by is None or not by:
+        sum_value = df[var].sum()
+        contributing_rows = list(df[id])
+        df_agg = pd.DataFrame({
+            var: [sum_value],
+            id: [contributing_rows]
+        })
+    else:
+        df_agg = df.groupby(by).agg({
+      var: "sum", 
+      id: lambda x: x.tolist()
+      }).reset_index()
+    return df_agg
+
+def aggregate_and_list(df:pd.DataFrame, by, var=None, margins=None, id=None):
+    if by is not None and not isinstance(by,list):
+        by = [by]
+        
+    subsets=[]
+    if by is not None:
+        for i in range(0,len(by)):
+            comb = combinations(by,i)
+            subsets = subsets + [list(c) for c in comb]
+    else:
+        subsets=[[]]
+        
+    if margins is not None:
+        subsets = [sub for sub in subsets if sub in margins]
+        
+    df_out = pd.DataFrame()
+    for sub in subsets:
+        sub_agg = agg_by(df, by=sub, var=var, id=id)
+        df_out = pd.concat([df_out,sub_agg],ignore_index=True)
+    return df_out  
+
+
+def get_unique_col_name(df, base_name):
+  # Generate a unique column name
+  i = 1
+  new_name = base_name
+  while new_name in df.columns:
+      new_name = f"{base_name}_{i}"
+      i += 1   
+  return new_name
+
+def timer(func):
+    @functools.wraps(func)
+    def wrapper_timer(*args, **kwargs):
+        tic = perf_counter()
+        value = func(*args, **kwargs)
+        toc = perf_counter()
+        elapsed_time = toc - tic
+        print(f"Elapsed time: {elapsed_time:0.4f} seconds")
+        return value
+    return wrapper_timer
+
+@timer
+def ctrl_round(df_in, by, var, margins=None, rounding_base=1, fix_rounding_dist= 0, max_heap_size= 1000):
+  """
+  Aggregate a dataframe and perform controlled rounding of it's entries.
+  input:
+    df_in             : pandas dataframe
+    by                : list of column names on which to aggregate the input dataframe
+    margins           : list of list of column name indicating which grouping to aggregate. Can be empty in which case all grouping and subgrouping are aggregated. 
+    Controlling the rounding on a subset of margins will improve the run-time but will leave the other margins free to potentialy deviate far from their original values.
+    var               : column to be aggregated
+    rounding_base     : the rounding base. Has to be greater than 0.
+    fix_rounding_dist : if an entry is close to a rounded value by p% of the rounding base, round that entry to it's closest rounded value and remove the other rounded value from consideration for that entry. 
+    This reduces the serach space and run time at the cost of the quality of the solution.
+    max_heap_size     : the maximum size the heap. Has to be greater than 2. Default is 1000. 
+    A smaller heap will lead to faster run-time at the cost of the quality of the solution.
+    
+  output:
+    A dictionary with following keys:
+    input_table     : the original input data with columns listed in the "by" and "var" input parameters.
+    input_margins   : 
+    rounded_table   : the rounded solution of input data with columns listed in the "by" and "var" input parameters.
+    rounded_margins : 
+    objectives      : the objective functions value for the solution
+    opt_report      : a dictionary containing information about the optimisation process with folowing keys:
+      n_iterations  : the number of partial solution expanded
+      n_heap_purges : the number of times the heap was purged, keeping the best solutions so far
+      n_sol_purged  : the total number of partial solution that got purged and never further expanded
+    n_cells         : the number of entries in the input table
+    n_margins       : the number of margin values from the input table 
+    n_fixed_cells   : the number of cells where the rounding is fixed and not subject to the optimisation process
+
+  """
+  
+  # check input parameters
+  if rounding_base <= 0:
+    print("Error: rounding_base has to be greater than 0")
+    return
+    
+  if max_heap_size < 2:
+    print("Error: max_heap_size has to be greater than 1")
+    return
+    
+  if fix_rounding_dist < 0:
+    print("Error: fix_rounding_dist has to be greater than or equal to 0")
+    return
+    
+  if fix_rounding_dist >= 0.5:
+    print("Error: fix_rounding_dist has to be less than 0.5")
+    return
+    
+  n_cells       = 0
+  n_margins     = 0
+  n_fixed_cells = 0
+  
+  # aggregate "var" by "by" columns in case there are duplicates in the input to make sure we have a table with signle entries per cell
+  by_values               = df_in.groupby(by).sum(var).reset_index()
+  # get a unique name not already present in the dataframe to store cell identifier
+  cell_id_name            = get_unique_col_name(by_values,"cellId")
+  # create a unique identifer for each cell of the table
+  by_values[cell_id_name] = range(len(by_values))
+  cellId_lst              = list(by_values[cell_id_name])
+  n_cells                 = len(cellId_lst)
+  
+  # create a mapping of each cell identifer to each value from the table
+  var_values                = by_values[[cell_id_name,var]].copy()
+  initial_values            = {}
+  for index, row in var_values.iterrows():
+    initial_values[row[cell_id_name]]  = row[var]
+  
+  # create a mapping of each cell identifer to each possible rounded value 
+  possible_values = var_values
+  lower_residual  = possible_values[var] % rounding_base
+  lower           = get_unique_col_name(by_values,"lower")
+  upper           = get_unique_col_name(by_values,"upper")
+  residual        = get_unique_col_name(by_values,"residual")
+  
+  possible_values[lower]    = possible_values[var] - lower_residual
+  possible_values[upper]    = possible_values[lower] + rounding_base
+  possible_values[residual] = lower_residual
+  
+  
+  # check if the original value is not already rounded, in which case the upper value should be the same.  
+  possible_cell_values      = {cellId:[] for cellId in cellId_lst}
+  for index, row in possible_values[[cell_id_name, lower, upper, residual]].iterrows():
+    # if upper is the same as lower, generate only one possibility
+    if row[residual] <= fix_rounding_dist * rounding_base:
+      possible_cell_values[row[cell_id_name]]  = [row[lower]]
+      n_fixed_cells += 1
+    elif row[residual] > (1-fix_rounding_dist) * rounding_base:
+      possible_cell_values[row[cell_id_name]]  = [row[upper]]
+      n_fixed_cells += 1
+    else:
+      possible_cell_values[row[cell_id_name]]  = [row[lower], row[upper]]
+      
+  # get margins of the input table
+  df_margins                = aggregate_and_list(by_values, by, var, margins, cell_id_name)
+  cons_id_name              = get_unique_col_name(df_margins,"consId")
+  df_margins[cons_id_name]  = range(len(df_margins))
+  n_margins                 = len(df_margins)
+  
+  # create a mapping of each margin identifer to each aggregated value 
+  constraint_values           = {}
+  for index, row in df_margins[[cons_id_name, var]].iterrows():
+    constraint_values[row[cons_id_name]] = row[var]
+  
+  # create a mapping of each margin identifer to a list of each cell identifer adding up to it
+  constraints           = {}
+  for index, row in df_margins[[cons_id_name,cell_id_name]].iterrows():
+    constraints[row[cons_id_name]] = row[cell_id_name]
+  
+  # define out distances measures
+  calculate_margin_max_distance   = define_margin_distance(max, normalized=False)
+  calculate_margin_sum_distance   = define_margin_distance(sum)
+  calculate_interior_sum_distance = define_interior_distance(sum)
+  distanceFuncs                   = [calculate_margin_max_distance, calculate_margin_sum_distance, calculate_interior_sum_distance]
+  # obtain the best rounding
+  result, n_iterations, n_heap_purges, n_sol_purged = best_first_search(possible_cell_values, initial_values, constraints, constraint_values, distanceFuncs, n_solutions = 1, max_heap_size= max_heap_size )
+  solution    = result[0][-1]
+  objectives  = result[0][:-1]
+  # assign the rounded values into a dataframe ready for output
+  df_out      = by_values.copy()
+  
+  df_out[var] = by_values[cell_id_name].map(solution)
+  margins     = aggregate_and_list(df_out, by, var, margins, cell_id_name)
+  margins     = margins[[*by,var]]
+  df_out      = df_out.drop(cell_id_name,axis=1)
+  
+  by_values   = by_values.drop(cell_id_name,axis=1)
+  df_margins  = df_margins.drop(cell_id_name,axis=1)
+  df_margins  = df_margins.drop(cons_id_name,axis=1)
+  
+  # report information from the optimization process
+  opt_report                  = {}
+  opt_report["n_iterations"]  = n_iterations
+  opt_report["n_heap_purges"] = n_heap_purges
+  opt_report["n_sol_purged"]  = n_sol_purged
+    
+  output = {"input_table"     : by_values, \
+            "input_margins"   : df_margins, \
+            "rounded_table"   : df_out, \
+            "rounded_margins" : margins, \
+            "objectives"      : objectives, \
+            "opt_report"      : opt_report, \
+            "n_cells"         : n_cells, \
+            "n_margins"       : n_margins, \
+            "n_fixed_cells"   : n_fixed_cells}
+  return output
+

CtrlRound/__init__.py

@@ -0,0 +1,2 @@
+from .CtrlRound import ctrl_round
+from .generate_table import generate_random_table

CtrlRound/best_first_search.py

@@ -0,0 +1,75 @@
+import heapq
+
+def best_first_search(possible_cell_values, initial_values, constraints, constraint_values, distance_funcs, n_solutions=0, max_heap_size=1000, reset_heap_fraction=0.75):
+    """
+    Performs best first search
+    input:
+      possible_cell_values  : dictionary of all decision variables along of all possible values for each
+      initial_values        : dictionary of initial values for each decision variable
+      constraints           : dictionary of each contraints to a list of decison variables that aggregate to that constraint's value
+      constraint_values     : dictionary of each contraints to the value they shopudl aggregate to
+      n_solutions            : the number of solutions to output. The first solutions found.
+      distance_funcs         : list of functions that will be used to calculation a lsit of distances to associate with a current (partial) solution
+      max_heap_size         : the maximum size the heap can be. If reached, half the best solutions will be kept.
+      reset_heap_fraction   : When the heap reaches it's maximum size, it is trimmed to keep only the most promising solution. This parameter determines the size of the heap after being trimmed as a fraction of the maximum size. 
+      This parameter has to be between 0 and 1. The higher the value, the more often heap timming occurs. Each trim inceases run-time.
+    """
+    # a unique counter for each partial solution pushed in the heap
+    counter         = 0
+    n_heap_purges     = 0
+    n_sol_purged      = 0
+    
+    # number of distance functions passed
+    nfuncs = len(distance_funcs)
+    # the size of the heap after trimming
+    reset_heap_size = int(reset_heap_fraction * max_heap_size)
+    
+    # Priority queue for Best First Search
+    pq              = []
+    
+    #the first solution  is the one where no decision has been made yet
+    initial_partial_solution  = {}
+    param_list         = [initial_partial_solution,initial_values,constraints,constraint_values]
+    initial_distances = [f(*param_list) for f in distance_funcs]
+    initial_state     = (*initial_distances, counter, initial_partial_solution)
+    
+    heapq.heappush(pq, initial_state)
+    
+    Solutions = []
+    while pq:
+        current_total_distance, current_margin_distance, current_interior_distance, current_counter, current_partial_solution  = heapq.heappop(pq)
+        
+        if len(current_partial_solution) == len(initial_values):
+          Solutions.append((current_total_distance,current_margin_distance,current_interior_distance,current_partial_solution))
+          
+        # output the N first Solutions found
+        if n_solutions > 0 and len(Solutions) == n_solutions:
+          return Solutions, counter, n_heap_purges, n_sol_purged
+          
+        # Generate neighbors
+        for cell_id in possible_cell_values:
+            if cell_id not in current_partial_solution:
+              for value in possible_cell_values[cell_id]:
+                  new_partial_solution          = current_partial_solution.copy()
+                  new_partial_solution[cell_id] = value
+                  
+                  new_param_list                 = [new_partial_solution,initial_values,constraints,constraint_values]
+                  new_distances                 = [f(*new_param_list) for f in distance_funcs]
+                  
+                  # a unique counter is stored in the state so that the heap will never attempt at comparing partial soutions distionaries as this would result in an error
+                  # if both distances are the same as another element in the heap, at least the counter will be different and used to order the elements
+                  counter                       += 1
+                  new_state                     = (*new_distances, counter, new_partial_solution)
+                  heapq.heappush(pq,new_state)
+              break
+        
+        #if heap gets too large, cut it in half keeping only the best partial solutions
+        if len(pq) > max_heap_size:
+          pq.sort(key=lambda x: x[:nfuncs])
+          n_sol_purged += len(pq) - reset_heap_size
+          pq = pq[:reset_heap_size]
+          heapq.heapify(pq)
+          n_heap_purges += 1
+        
+        
+    return Solutions, counter, n_heap_purges, n_sol_purged

CtrlRound/distance.py

@@ -0,0 +1,38 @@
+#all distance functions must acces input parameters partial_solution, initial_values, constraints, constraint_values
+
+
+def define_margin_distance(func, normalized=True):  
+  #define the distance function used through the aggregation function used on the margins
+  def calculate_margin_distance(partial_solution, initial_values, constraints, constraint_values):
+      # merge the solution so far with initial values for the cells where no decision has been taken yet
+      current_values = initial_values.copy()
+      for cell,val in partial_solution.items():
+          current_values[cell]=val
+          
+      nCell = len(partial_solution)        
+      marginDiscrepancies = []
+      for cons in constraints:
+        target_value = constraint_values[cons]
+        current_value = sum(current_values[cell_id] for cell_id in constraints[cons])
+        marginDiscrepancies.append(abs(target_value - current_value)) 
+      # marginDiscrepancies = [abs(constraint_values[cons] - sum(current_values[cell_id] for cell_id in constraints[cons])) for cons in constraints]
+      if nCell >1 and normalized:
+        return func(marginDiscrepancies)/nCell
+      else:
+        return func(marginDiscrepancies)
+  return calculate_margin_distance
+
+def define_interior_distance(func, normalized=True): 
+  #define a distance function on the interior cells
+  def calculate_interior_distance(partial_solution, initial_values, constraints, constraint_values):
+      # calculate the deviation from the origianl values in the interior cells
+      # constraints and constraints values are not used but are input parameter to have a uniform interface across all distance functions
+      if len(partial_solution) == 0 :
+        return 0
+      nCell = len(partial_solution) 
+      if nCell >1 and normalized:
+        return func(abs(partial_solution[cell] - initial_values[cell]) for cell in partial_solution)/nCell
+      else:
+        return func(abs(partial_solution[cell] - initial_values[cell]) for cell in partial_solution)
+      
+  return calculate_interior_distance

CtrlRound/generate_table.py

@@ -0,0 +1,14 @@
+import pandas as pd
+import itertools
+import numpy as np
+
+
+def generate_random_table(n_dim,n_cat,scale=1):
+  #generate n_dim columns each with n_cat values
+  sets = [set(range(n_cat)) for _ in range(n_dim)]
+  cartesian_product = list(itertools.product(*sets))
+  df = pd.DataFrame(cartesian_product, columns=[*range(n_dim)])
+  #generate random values between 0 and scale
+  df["value"] = np.random.rand(len(df)) * scale
+  return df
+  

CtrlRound-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Christian Gagné
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

CtrlRound-0.1.0.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.1
+Name: CtrlRound
+Version: 0.1.0
+Summary: Performs controlled rounding of tabular data
+Home-page: https://github.com/Veozen/CtrlRound
+Author: Christian Gagné
+Author-email: christian.gagne@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy >=1.23.5
+Requires-Dist: pandas >=2.1.2
+
+## Description  
+Controlled rounding is a technique used in data publishing to ensure that data released for public use meets specific confidentiality requirements. 
+The process adjusts the data values by rounding them to a predetermined base in a such a way that the margins of the resulting table remain closer to their original values than they would should the table's entries have been simply rounded to their nearest base.  
+
+Here the solution to this problem is found by applying a best-first-search method where the decision is to round up or down each non-margin entry. Three distance functions are used to sort partial solutions. 
+- The max absolute difference between a margin's value and it's original value.  
+- The average absolute difference between a margin's value and it's original value. The average is taken over the rounded cells so far.  
+- The average absolute difference between a table cell's rounded value and it's original value. The average is taken over the rounded cells so far.  
+
+Ties on the first function are resolved by looking at the second and then third one. The first complete solution found is returned.  
+
+## Usage 
+
+**ctrl_round(df_in, by, var, margins, rounding_base, fix_rounding_dist, max_heap_size):**  
+Aggregates a dataframe and perform controlled rounding of it's entries.  
+
+**input:**  
+- **df_in**             : pandas DataFrame
+- **by**                : list of column names on which to aggregate the input DataFrame
+- **margins**           : list of lists of column names indicating which grouping to aggregate. Can be empty, in which case all grouping and subgrouping are aggregated. Controlling the rounding on a subset of margins will improve the run-time but will leave the other margins free to potentially deviate far from their original values.
+- **var**               : column to be aggregated
+- **rounding_base**     : the rounding base. Has to be greater than 0.
+- **fix_rounding_dist** : if an entry is close to a rounded value by p% of the rounding base, round that entry to its closest rounded value and remove the other rounded value from consideration for that entry. This reduces the search space and run time at the cost of the quality of the solution.
+- **max_heap_size**     : the maximum size of the heap. Has to be greater than 2. Default is 1000. A smaller heap will lead to faster run-time at the cost of the quality of the solution.
+
+**output:**  
+A dictionary with the following keys:
+- **input_table**     : the original input data with columns listed in the "by" and "var" input parameters.
+- **input_margins**   :
+- **rounded_table**   : the rounded solution of input data with columns listed in the "by" and "var" input parameters.
+- **rounded_margins** :
+- **objectives**      : the objective function's value for the solution
+- **opt_report**      : a dictionary containing information about the optimisation process with the following keys:
+  - **n_iterations**  : the number of partial solutions expanded
+  - **n_heap_purges** : the number of times the heap was purged, keeping the best solutions so far
+  - **n_sol_purged**  : the total number of partial solutions that got purged and never further expanded
+  - **n_cells**       : the number of entries in the input table
+  - **n_margins**     : the number of margin values from the input table
+  - **n_fixed_cells** : the number of cells where the rounding is fixed and not subject to the optimisation process
+
+
+**generate_random_table(n_dim,n_cat,scale):**  
+Creates a table filled with random values with the desired number of dimensions and number of categories per dimensions.  
+
+**input:**  
+- n_dim : number of dimensions
+- n_cat : number of categories per dimensions
+- scale : the scale of the numbers in the table: 0 < number < scale.
+
+**output:**  
+The generated the table as a pandas dataframe. With columns:  
+- 0,1,2 ... n_dim-1 : each column contains values from 0 to n_cat-1.     
+- value : contains a random value between 0 and scale.  
+
+
+## Example
+
+test = generate_random_table(3, 5, scale=2)  
+ctrl_round(test, by=[0,1,2], var="value", rounding_base=1, fix_rounding_dist=0.1, max_heap_size=100)  
+
+## License
+This project is licensed under the MIT License -
+see the [LICENSE](LICENSE) file for details.

CtrlRound-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+CtrlRound/CtrlRound.py,sha256=sBLQlZEePN1Gij00XBA8WFUzp9rp4z82rC6oZKaq-Z8,9521
+CtrlRound/__init__.py,sha256=sGMdYdUi7IZQckK23ycKa6s0rkrPVbZGExqPx_Kk_gw,84
+CtrlRound/best_first_search.py,sha256=bmbJ4v1aBEkyZlqTz0uK8PnCQsA5MEM2LyrwU7pPTnI,4359
+CtrlRound/distance.py,sha256=Il6YnVivFOyN6D45ll3zFT6evMujadap_-4HC15oPJo,2084
+CtrlRound/generate_table.py,sha256=zY52WpwtUjNtd_G-eVTmDYDdtY18QOCb5_h_y2kUNKs,444
+CtrlRound-0.1.0.dist-info/LICENSE,sha256=Dup0VF41C5uKQdZ_CoUEnfBWmOpbMi2O1S3IoONvAx4,1092
+CtrlRound-0.1.0.dist-info/METADATA,sha256=QM-ns1NR41bROp3bUT-wn3W9I4-ecxXsnVrbISmbDzE,4852
+CtrlRound-0.1.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+CtrlRound-0.1.0.dist-info/top_level.txt,sha256=5UX_Ixt5hSuYvYzbMB64vE1aYPjHw2K8Yhm1OUprZj4,10
+CtrlRound-0.1.0.dist-info/RECORD,,

CtrlRound-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.3)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

CtrlRound-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+CtrlRound