python-fedci 0.1.0__py3-none-any.whl

fedci/__init__.py

@@ -0,0 +1,4 @@
+from .server import Server, ProxyServer
+from .client import Client, ProxyClient
+from .utils import VariableType
+from .evaluation import get_symmetric_likelihood_tests

fedci/client.py

@@ -0,0 +1,405 @@
+from scipy.sparse.linalg._eigen.lobpcg.lobpcg import LinAlgError
+from .utils import VariableType, ClientResponseData, BetaUpdateData
+import polars as pl
+import numpy as np
+import scipy
+import rpyc
+
+from typing import Dict, List
+
+from .env import DEBUG, EXPAND_ORDINALS, RIDGE, LR, OVR
+
+import statsmodels.api as sm
+from statsmodels.genmod.generalized_linear_model import GLMResults
+from statsmodels.genmod.families import family
+
+class ComputationHelper():
+    @staticmethod
+    def get_regression_model(y, X, beta, glm_family):
+        model = sm.GLM(y, X, family=glm_family)
+        result = GLMResults(model, beta, normalized_cov_params=None, scale=None)
+        #result = GLMResults(model, beta, normalized_cov_params=None, scale=model.estimate_scale(result.predict()))
+        return result
+
+    @staticmethod
+    def run_model(y, X, model):
+        llf = model.llf
+        deviance = model.deviance
+
+        # calculate fisher information and score vector
+        eta = model.predict(which='linear')
+
+        # g' is inverse of link function
+        inverse_link = model.family.link.inverse
+        mu = inverse_link(eta)
+
+        # delta g' is derivative of inverse link function
+        derivative_inverse_link = model.family.link.inverse_deriv
+        dmu_deta = derivative_inverse_link(eta)
+        dmu_deta = np.clip(dmu_deta, 1e-8, 1-1e-8)
+
+        z = eta + LR*(y - mu)/dmu_deta
+
+        if type(model.family) == family.Gaussian:
+            var_y = np.var(y-mu)
+        elif type(model.family) == family.Binomial:
+            var_y = dmu_deta
+        else:
+            raise Exception(f'Cannot handle model family {model.family.__class__.__name__}')
+        W = np.diag((dmu_deta**2)/var_y)
+
+        xw = X.T @ W
+        xwx = xw @ X
+        xwz = xw @ z
+
+        return {'llf': llf, 'deviance': deviance, 'xwx': xwx, 'xwz': xwz}
+
+    @classmethod
+    def run_regression(cls, y, X, beta, glm_family):
+        model = cls.get_regression_model(y, X, beta, glm_family)
+        return cls.run_model(y, X, model)
+
+class ComputationUnit():
+    @staticmethod
+    def compute(data, y_label, X_labels, beta):
+        raise NotImplementedError()
+
+class ContinousComputationUnit(ComputationUnit):
+    @staticmethod
+    def compute(data, y_label, X_labels, beta):
+        assert len(beta) == 1, 'Continuos regression called with more than one beta'
+        beta = list(beta.values())[0]
+
+        X = data.to_pandas()[sorted(X_labels)]
+        X['__const'] = 1
+        X = X.to_numpy().astype(float)
+
+        y = data.to_pandas()[y_label]
+        y = y.to_numpy().astype(float)
+
+        return ComputationHelper.run_regression(
+            y=y,
+            X=X,
+            beta=beta,
+            glm_family=family.Gaussian()
+        )
+
+class BinaryComputationUnit(ComputationUnit):
+    @staticmethod
+    def compute(data, y_label, X_labels, beta):
+        assert len(beta) == 1, 'Binary regression called with more than one beta'
+        beta = list(beta.values())[0]
+
+        X = data.to_pandas()[sorted(X_labels)]
+        X['__const'] = 1
+        X = X.to_numpy().astype(float)
+
+        y = data.to_pandas()[y_label]
+        y = y.to_numpy().astype(float)
+
+        return ComputationHelper.run_regression(
+            y=y,
+            X=X,
+            beta=beta,
+            glm_family=family.Binomial()
+        )
+
+class CategoricalComputationUnit(ComputationUnit):
+    @staticmethod
+    def compute(data, y_label, X_labels, beta):
+        assert len(beta) == 1, 'Multinomial regression called with more than one beta'
+        beta = beta[y_label]
+
+        # Identify the dummy columns for the response
+        y_dummy_columns = [c for c in data.columns if c.startswith(f'{y_label}__cat__')]
+
+        # Design matrix
+        X = data.to_pandas()[sorted(X_labels)]
+        X['__const'] = 1
+        X = X.to_numpy().astype(float)
+
+        num_categories = len(y_dummy_columns)  # J
+        num_features = len(X_labels) + 1       # K
+
+        def softmax(eta):
+            exp_eta = np.exp(np.hstack([np.zeros((eta.shape[0], 1)), eta]))
+            return exp_eta / exp_eta.sum(axis=1, keepdims=True)
+
+        # Response matrix (N x (J-1))
+        Y = data.to_pandas()[y_dummy_columns[1:]].to_numpy()
+
+        # Reshape beta (K x (J-1))
+        beta = beta.reshape(num_features, -1, order='F')
+
+        # Compute eta and mu
+        eta = np.clip(X @ beta, -350, 350)             # N x (J-1)
+        mu = np.clip(softmax(eta), 1e-8, 1-1e-8)            # N x J
+        mu_reduced = mu[:, 1:]                         # N x (J-1)
+
+        # Initialize accumulators for XWX and XWz
+        XWX = np.zeros((num_features * (num_categories - 1), num_features * (num_categories - 1)))
+        XWz = np.zeros(num_features * (num_categories - 1))
+
+        # Construct W blocks and z
+        for i in range(Y.shape[0]):
+            yi = Y[i]  # (J-1)
+            pi = mu_reduced[i]
+            var_i = np.diag(pi) - np.outer(pi, pi)  # (J-1) x (J-1)
+
+            try:
+                var_i_inv = np.linalg.inv(var_i)
+            except np.linalg.LinAlgError:
+                var_i_inv = np.linalg.pinv(var_i)
+
+            z_i = eta[i] + var_i_inv @ (yi - pi)  # (J-1)
+
+            # Compute local contributions to XWX and XWz
+            Xi = np.kron(np.eye(num_categories - 1), X[i:i+1])  # (J-1) x (J-1)*K
+            Wi = var_i  # (J-1) x (J-1)
+            XWX += Xi.T @ Wi @ Xi
+            XWz += Xi.T @ Wi @ z_i
+
+        # Compute log-likelihood and deviance
+        Y_full = data.to_pandas()[y_dummy_columns].to_numpy()  # N x J
+        logprob = np.log(np.clip(mu, 1e-8, 1))
+        llf = np.sum(Y_full * logprob)
+        deviance = -2 * llf
+
+        results = {y_label: {'xwx': XWX, 'xwz': XWz}}
+
+        return {
+            'llf': llf,
+            'deviance': deviance,
+            'beta_update_data': results
+        }
+
+
+class CategoricalOVRComputationUnit(ComputationUnit):
+    @staticmethod
+    def compute(data, y_label, X_labels, betas):
+        X = data.to_pandas()[sorted(X_labels)]
+        X['__const'] = 1
+        X = X.to_numpy().astype(float)
+
+        models = {}
+        results = {}
+        for category in betas.keys():
+            y = data.to_pandas()[category]
+            y = y.to_numpy().astype(float)
+
+            models[category] = ComputationHelper.get_regression_model(
+                y=y,
+                X=X,
+                beta=betas[category],
+                glm_family=family.Binomial()
+            )
+            current_result = ComputationHelper.run_model(
+                y=y,
+                X=X,
+                model=models[category]
+            )
+            results[category] = {'xwx': current_result['xwx'], 'xwz': current_result['xwz']}
+
+        # calculate multinomial llf
+        etas = {c:np.clip(m.predict(which='linear'), -350, 350) for c,m in models.items()}
+        denom = 1 + sum(np.exp(eta) for eta in etas.values())
+        mus = {c:np.clip(np.exp(eta)/denom, 1e-8, 1-1e-8) for c,eta in etas.items()}
+
+        llf = 0
+        llf_saturated = 0
+        reference_category_indices = np.ones(len(data))
+        for category in betas.keys():
+            y = data.to_pandas()[category].to_numpy().astype(float)
+            mu = mus[category]
+            reference_category_indices = reference_category_indices * (y==0)
+            # LLF
+            llf += np.sum(np.log(np.take(mu, np.nonzero(y)[0])))
+            ## LLF SATURATED (for deviance)
+            #llf_saturated += np.sum(y * np.log(np.clip(y, 1e-10, None)))
+
+        # LLF
+        llf += np.sum(np.log(np.take(1/denom, reference_category_indices.nonzero()[0])))
+
+        # LLF SATURATED (for deviance)
+        #llf_saturated += np.sum(reference_category_indices * np.log(np.clip(reference_category_indices, 1e-10, None)))
+        deviance = 2 * (llf_saturated - llf)
+
+        return {
+            'llf': llf,
+            'deviance': deviance,
+            'beta_update_data': results
+        }
+
+class OrdinalComputationUnit(ComputationUnit):
+    @staticmethod
+    def compute(data, y_label, X_labels, betas):
+        X = data.to_pandas()[sorted(X_labels)]
+        X['__const'] = 1
+        X = X.to_numpy().astype(float)
+
+        models = {}
+        results = {}
+        for level in betas.keys():
+            level_int = int(level.split('__ord__')[-1])
+            y = data.to_pandas()[y_label]
+            y = (y.to_numpy() <= level_int).astype(float)
+
+            models[level] = ComputationHelper.get_regression_model(
+                y=y,
+                X=X,
+                beta=betas[level],
+                glm_family=family.Binomial()
+            )
+            current_result = ComputationHelper.run_model(
+                y=y,
+                X=X,
+                model=models[level]
+            )
+            results[level] = {'xwx': current_result['xwx'], 'xwz': current_result['xwz']}
+
+        mus = [(level, model.predict()) for level, model
+                      in sorted(models.items(), key=lambda lvl: int(lvl[0].split('__ord__')[-1]))]
+        # get diffs of mus of successive levels
+        mus_diff = [mus[0]]
+        mus_diff.extend([(mus[i][0], mus[i][1] - mus[i-1][1]) for i in range(1,len(mus))])
+        mus_diff.append(('ref',1-mus[-1][1]))
+
+        # fix negative probs
+        sign_fix = np.column_stack([e[1] for e in mus_diff])
+        problematic_indices = np.where(sign_fix < 0)[0]
+        if len(problematic_indices) > 0:
+            problem_probs = np.abs(sign_fix[problematic_indices])
+            row_sums = np.clip(np.sum(problem_probs, axis=1, keepdims=True), 1e-8, None)
+            normalized_probs = problem_probs / row_sums
+            sign_fix[problematic_indices] = normalized_probs
+            mus_diff = [(mus_diff[i][0], sign_fix[:,i]) for i in range(len(mus_diff))]
+        mus_diff = [(l, np.clip(p, 1e-8, None)) for l,p in mus_diff]
+
+        llf = 0
+        llf_saturated = 0
+        reference_level_indices = np.ones(len(data))
+        for i in range(len(mus_diff)-1):
+            level, mu_diff = mus_diff[i]
+            level_int = int(level.split('__ord__')[-1])
+            current_level_indices = data[y_label].to_numpy() == level_int
+            reference_level_indices = reference_level_indices * (1-current_level_indices)
+
+            llf += np.sum(np.log(np.take(mu_diff, current_level_indices.nonzero()[0])))
+        _, mu_diff = mus_diff[-1]
+        llf += np.sum(np.log(np.take(mu_diff, reference_level_indices.nonzero()[0])))
+        deviance = 2 * (llf_saturated - llf)
+
+        return {
+            'llf': llf,
+            'deviance': deviance,
+            'beta_update_data': results
+        }
+
+polars_dtype_map = {
+    pl.Float64: VariableType.CONTINUOS,
+    pl.Boolean: VariableType.BINARY,
+    pl.String: VariableType.CATEGORICAL,
+    pl.Int32: VariableType.ORDINAL,
+    pl.Int64: VariableType.ORDINAL
+}
+
+regression_computation_map = {
+    VariableType.CONTINUOS: ContinousComputationUnit,
+    VariableType.BINARY: BinaryComputationUnit,
+    VariableType.CATEGORICAL: CategoricalComputationUnit if OVR == 0 else CategoricalOVRComputationUnit,
+    VariableType.ORDINAL: OrdinalComputationUnit
+}
+
+class Client():
+    def __init__(self, data: pl.DataFrame, _network_fetch_function=lambda x: x):
+        self._network_fetch_function = _network_fetch_function
+        self.data: pl.DataFrame = data
+        self.schema: Dict[str, VariableType] = {column: polars_dtype_map[dtype] for column, dtype in dict(self.data.schema).items()}
+        self.categorical_expressions: Dict[str, List[str]] = {column: self.data.select(column).to_dummies(separator='__cat__').columns
+                                                              for column, dtype in self.schema.items() if dtype == VariableType.CATEGORICAL}
+        self.ordinal_expressions: Dict[str, List[str]] = {column: self.data.select(pl.col(column)).to_dummies(separator='__ord__').columns
+                                                          for column, dtype in self.schema.items() if dtype == VariableType.ORDINAL}
+
+        self.server_categorical_expressions: Dict[str, List[str]] = None
+        self.server_ordinal_expressions: Dict[str, List[str]] = None
+        self.expanded_data: pl.DataFrame = None
+
+    def get_data_schema(self):
+        return self.schema
+
+    def get_categorical_expressions(self):
+        return self.categorical_expressions
+    def get_ordinal_expressions(self):
+        return self.ordinal_expressions
+
+    def provide_expressions(
+        self,
+        categorical_expressions: Dict[str, List[str]],
+        ordinal_expressions: Dict[str, List[str]]
+    ):
+        self.server_categorical_expressions = categorical_expressions
+        self.server_ordinal_expressions = ordinal_expressions
+
+        # expand categoricals
+        all_possible_categorical_expressions = set([li for l in categorical_expressions.values() for li in l])
+        temp = self.data.select([column for column, dtype in self.schema.items() if dtype == VariableType.CATEGORICAL])
+        _data = self.data.to_dummies([column for column, dtype in self.schema.items() if dtype == VariableType.CATEGORICAL], separator='__cat__')
+        _data = _data.with_columns(temp)
+        missing_cols = list(all_possible_categorical_expressions - set(_data.columns))
+        _data = _data.with_columns(*[pl.lit(0.0).alias(c) for c in missing_cols])
+
+        if EXPAND_ORDINALS == 1:
+            # expand ordinals
+            all_possible_ordinal_expressions = set([li for l in ordinal_expressions.values() for li in l])
+            _data = _data.to_dummies([column for column, dtype in self.schema.items() if dtype == VariableType.ORDINAL], separator='__ord__')
+            missing_cols = list(all_possible_ordinal_expressions - set(_data.columns))
+            _data = _data.with_columns(*[pl.lit(0.0).alias(c) for c in missing_cols])
+
+            # keep original ordinal variables
+            _data = _data.with_columns(self.data.select([column for column, dtype in self.schema.items() if dtype == VariableType.ORDINAL]))
+
+        self.expanded_data = _data
+
+    def compute(self, y_label: str, X_labels, beta):
+        # NOTE: preemptive cast for netref arrays
+        beta = self._network_fetch_function(beta)
+        assert y_label in self.schema
+
+        result = regression_computation_map[self.schema[y_label]].compute(self.expanded_data, y_label, X_labels, beta)
+
+        if self.schema[y_label] in [VariableType.CONTINUOS, VariableType.BINARY]:
+            response: ClientResponseData = ClientResponseData(
+                llf=result['llf'],
+                deviance=result['deviance'],
+                beta_update_data={list(beta.keys())[0]: BetaUpdateData(xwx=result['xwx'], xwz=result['xwz'])}
+            )
+        else:
+            beta_update_data = {category: BetaUpdateData(xwx=data['xwx'], xwz=data['xwz']) for category, data in result['beta_update_data'].items()}
+            response: ClientResponseData = ClientResponseData(
+                llf=result['llf'],
+                deviance=result['deviance'],
+                beta_update_data=beta_update_data
+            )
+        return response
+
+class ProxyClient(rpyc.Service):
+    def __init__(self, data):
+        self.client = Client(data, _network_fetch_function=rpyc.classic.obtain)
+        self.server: rpyc.utils.server.ThreadedServer = None
+    def __del__(self):
+        self.close()
+    def start(self, port):
+        if self.server is not None:
+            self.close()
+        self.server = rpyc.utils.server.ThreadedServer(self, port=port, protocol_config={'allow_public_attrs': True, 'allow_pickle': True})
+        self.server.start()
+    def close(self):
+        if self.server is None:
+            return
+        self.server.close()
+        self.server = None
+    # expose all functions of client
+    def on_connect(self, conn):
+        for name in dir(self.client):
+            if callable(getattr(self.client, name)) and not name.startswith("_"):
+                setattr(self, f"exposed_{name}", getattr(self.client, name))

fedci/env.py

@@ -0,0 +1,6 @@
+import os
+DEBUG = 0 if (v:=os.getenv("DEBUG")) is None else int(v)
+EXPAND_ORDINALS = 1 if (v:=os.getenv("EXPAND_ORDINALS")) is None else int(v)
+LR = 1 if (v:=os.getenv("LR")) is None else float(v)
+RIDGE = 0 if (v:=os.getenv("RIDGE")) is None else float(v)
+OVR = 0 if (v:=os.getenv("OVR")) is None else int(v)

fedci/evaluation.py

@@ -0,0 +1,154 @@
+from .testing import Test
+from .env import DEBUG
+
+import scipy
+import numpy as np
+import polars as pl
+import polars.selectors as cs
+import os
+from typing import List
+
+class LikelihoodRatioTest():
+    def __init__(self, t0: Test, t1: Test) -> None:
+
+        assert t0.y_label == t1.y_label, 'Provided tests do not predict the same variable'
+        t0_req_labels = t0.get_required_labels() - {t0.y_label}
+        t1_req_labels = t1.get_required_labels() - {t0.y_label}
+        assert t0_req_labels.issubset(t1_req_labels), 'Provided tests are not nested'
+        assert len(t0_req_labels)+1 == len(t1_req_labels), 'Provided tests differ by more than one regressor variable'
+
+        self.y_label = t0.y_label
+        self.x_label = list(t1_req_labels - t0_req_labels)[0]
+        self.s_labels = sorted(list(t0_req_labels))
+
+        self.p_val = self._run_ci_test(t0, t1)
+
+    def _run_ci_test(self, t0: Test, t1: Test):
+        client_subset = t1.get_providing_clients()
+        t0_llf = t0.get_llf(client_subset)
+        t1_llf = t1.get_llf(client_subset)
+
+        t0_dof = t0.get_degrees_of_freedom()
+        t1_dof = t1.get_degrees_of_freedom()
+
+        p_val = scipy.stats.chi2.sf(2*(t1_llf - t0_llf), t1_dof-t0_dof)
+
+        if DEBUG >= 2:
+            print(f'*** Calculating p value for independence of {self.y_label} from {self.x_label} given {self.s_labels}')
+            print(f'{t1_dof-t0_dof} DOFs = {t1_dof} T1 DOFs - {t0_dof} T0 DOFs')
+            print(f'{2*(t1_llf - t0_llf):.4f} Test statistic = 2*({t1_llf:.4f} T1 LLF - {t0_llf:.4f} T0 LLF)')
+            print(f'p value = {p_val:.6f}')
+        return p_val
+
+    def __repr__(self):
+        return f"LikelihoodRatioTest - y: {self.y_label}, x: {self.x_label}, S: {self.s_labels}, p: {self.p_val:.4f}"
+
+    def __lt__(self, other):
+        if len(self.s_labels) < len(other.s_labels):
+            return True
+        elif len(self.s_labels) > len(other.s_labels):
+            return False
+
+        if self.y_label < other.y_label:
+            return True
+        elif self.y_label > other.y_label:
+            return False
+
+        if self.x_label < other.x_label:
+            return True
+        elif self.x_label > other.x_label:
+            return False
+
+        if tuple(sorted(self.s_labels)) < tuple(sorted(other.s_labels)):
+            return True
+        elif tuple(sorted(self.s_labels)) > tuple(sorted(other.s_labels)):
+            return False
+
+        return True
+
+class SymmetricLikelihoodRatioTest():
+    def __init__(self, lrt0: LikelihoodRatioTest, lrt1: LikelihoodRatioTest):
+
+        assert lrt0.y_label == lrt1.x_label and lrt1.y_label == lrt0.x_label and sorted(lrt0.s_labels) == sorted(lrt1.s_labels), 'Tests do not match'
+
+        self.lrt0: LikelihoodRatioTest = lrt0
+        self.lrt1: LikelihoodRatioTest = lrt1
+
+        self.v0, self.v1 = sorted([lrt0.y_label, lrt1.y_label])
+        self.conditioning_set = sorted(lrt0.s_labels)
+
+        self.p_val = min(2*min(self.lrt0.p_val, self.lrt1.p_val), max(self.lrt0.p_val, self.lrt1.p_val))
+        if DEBUG >= 2:
+            print(f'*** Combining p values for symmetry of tests between {self.v0} and {self.v1} given {self.conditioning_set}')
+            print(f'p value {self.lrt0.y_label}: {self.lrt0.p_val}')
+            print(f'p value {self.lrt1.y_label}: {self.lrt1.p_val}')
+            print(f'p value = {self.p_val:.4f}')
+
+
+    def __repr__(self):
+        return f"SymmetricLikelihoodRatioTest - v0: {self.v0}, v1: {self.v1}, conditioning set: {self.conditioning_set}, p: {self.p_val:.4f}\n\t- {self.lrt0}\n\t- {self.lrt1}"
+
+    def __lt__(self, other):
+        if len(self.conditioning_set) < len(other.conditioning_set):
+            return True
+        elif len(self.conditioning_set) > len(other.conditioning_set):
+            return False
+
+        if self.v0 < other.v0:
+            return True
+        elif self.v0 > other.v0:
+            return False
+
+        if self.v1 < other.v1:
+            return True
+        elif self.v1 > other.v1:
+            return False
+
+        if tuple(self.conditioning_set) < tuple(other.conditioning_set):
+            return True
+        elif tuple(self.conditioning_set) > tuple(other.conditioning_set):
+            return False
+
+        return False
+
+    def __eq__(self, other):
+        return self.v0 == other.v0 and self.v1 == other.v1 and self.conditioning_set == other.conditioning_set
+
+class EmptyLikelihoodRatioTest(SymmetricLikelihoodRatioTest):
+    def __init__(self, v0, v1, conditioning_set, p_val):
+        self.v0, self.v1 = sorted([v0, v1])
+        self.conditioning_set = conditioning_set
+        self.p_val = p_val
+
+    def __repr__(self):
+        return f"EmptyLikelihoodRatioTest - v0: {self.v0}, v1: {self.v1}, conditioning set: {self.conditioning_set}, p: {self.p_val:.4f}"
+
+def get_likelihood_tests(tests: List[Test]):
+    likelihood_tests = []
+    for test in tests:
+        curr_y = test.y_label
+        curr_X = test.get_required_labels() - {curr_y}
+        for x_var in curr_X:
+            curr_conditioning_set = curr_X - {x_var}
+            nested_test = [t for t in tests if ((t.get_required_labels() - {curr_y}) == curr_conditioning_set) and t.y_label == curr_y]
+            if len(nested_test) == 0:
+                print(f'No test for\n{test}\nin\n{tests}')
+                continue
+            assert len(nested_test) == 1, 'There is more than one nested test'
+            likelihood_tests.append(LikelihoodRatioTest(nested_test[0], test))
+    return likelihood_tests
+
+def get_symmetric_likelihood_tests(tests, test_targets=None):
+    symmetric_tests = []
+    asymmetric_tests = get_likelihood_tests(tests)
+    unique_tests = [t for t in asymmetric_tests if t.x_label < t.y_label]
+
+    for test in unique_tests:
+        if test_targets is not None and (test.x_label, test.y_label, tuple(sorted(test.s_labels))) not in test_targets:
+            continue
+        test_counterpart = [t for t in asymmetric_tests if (t.y_label == test.x_label) and (t.x_label == test.y_label) and (t.s_labels == test.s_labels)]
+        if len(test_counterpart) == 0:
+            continue
+        assert len(test_counterpart) == 1, 'There is more than one matching counterpart test'
+        symmetric_tests.append(SymmetricLikelihoodRatioTest(test, test_counterpart[0]))
+    return symmetric_tests

fedci/server.py

@@ -0,0 +1,105 @@
+from .client import Client
+from .testing import TestEngine
+from .env import DEBUG
+from typing import List, Dict
+import rpyc
+
+class Server():
+    def __init__(self, clients: Dict[str, Client], max_regressors=None, test_targets=None, max_iterations=25, _network_fetch_function=lambda x: x):
+        self._network_fetch_function = _network_fetch_function
+        self.clients = clients
+        self.client_schemas = {}
+        self.schema = {}
+
+        for client_id, client in self.clients.items():
+            client_schema = client.get_data_schema()
+            self.client_schemas[client_id] = client_schema
+            for column, dtype in client_schema.items():
+                if column not in self.schema:
+                    self.schema[column] = dtype
+                    continue
+                assert self.schema[column] == dtype, f'Schema mismatch between clients detected for variable {column}!'
+
+        self.category_expressions = {}
+        self.ordinal_expressions = {}
+        for client in self.clients.values():
+            for feature, expressions in client.get_categorical_expressions().items():
+                self.category_expressions[feature] = sorted(list(set(self.category_expressions.get(feature, [])).union(set(expressions))))
+            for feature, levels in client.get_ordinal_expressions().items():
+                self.ordinal_expressions[feature] = sorted(list(set(self.ordinal_expressions.get(feature, [])).union(set(levels))), key=lambda x: int(x.split('__ord__')[-1]))
+
+        for client in self.clients.values(): client.provide_expressions(self.category_expressions, self.ordinal_expressions)
+
+        self.test_engine: TestEngine = TestEngine(
+            schema=self.schema,
+            category_expressions=self.category_expressions,
+            ordinal_expressions=self.ordinal_expressions,
+            max_regressors=max_regressors,
+            max_iterations=max_iterations,
+            test_targets=test_targets
+        )
+
+    def run(self):
+        while not self.test_engine.is_finished():
+            required_labels = self.test_engine.get_currently_required_labels()
+            selected_clients = {client_id: client for client_id, client in self.clients.items() if set(required_labels).issubset(self.client_schemas[client_id].keys())}
+            if len(selected_clients) == 0:
+                self.test_engine.remove_current_test()
+                continue
+            assert len(selected_clients) > 0, f'No client is able to provide the data for {required_labels}'
+
+            y_label, X_labels, beta = self.test_engine.get_current_test_parameters()
+
+            results = {client_id: client.compute(y_label, X_labels, beta) for client_id, client in selected_clients.items()}
+            # NOTE: fetch ClientResponseData over network if necessary
+            results = {client_id: self._network_fetch_function(result) for client_id, result in results.items()}
+            self.test_engine.update_current_test(results)
+        if DEBUG >= 1:
+            print("*** All tests")
+            for test in self.test_engine.tests:
+                print(test)
+        return self.test_engine.tests
+
+    def get_tests(self):
+        return self.test_engine.tests
+
+class ProxyServerBuilder():
+    def __init__(self, cls):
+        self.clients = []
+        self.cls = cls
+        self.max_regressors = None
+        self.max_iterations = 25
+    def set_max_regressors(self, max_regressors):
+        self.max_regressors = max_regressors
+        return self
+    def set_max_iterations(self, max_iterations):
+        self.max_iterations = max_iterations
+        return self
+    def add_client(self, hostname, port):
+        if (hostname, port) in self.clients:
+            print('Client exists already')
+            return self
+        client = rpyc.connect(hostname, port, config={'allow_public_attrs': True, 'allow_pickle': True})
+        self.clients.append(client)
+        return self
+    def build(self):
+        return self.cls(self.clients, max_regressors=self.max_regressors, max_iterations=self.max_iterations)
+
+class ProxyServer():
+    @classmethod
+    def builder(cls, **kwargs):
+        return ProxyServerBuilder(cls, **kwargs)
+    def __init__(self, clients, max_regressors, max_iterations):
+        self.clients = {i: c.root for i, c in enumerate(clients)}
+        self.server = Server(
+            self.clients,
+            _network_fetch_function=rpyc.classic.obtain,
+            max_regressors=max_regressors,
+            max_iterations=max_iterations
+        )
+    def __getattr__(self, name):
+        return getattr(self.server, name)
+    def run(self):
+        return self.server.run()
+    def get_tests(self):
+        return self.server.get_tests()

fedci/testing.py

@@ -0,0 +1,275 @@
+
+import numpy as np
+from typing import List, Dict
+from itertools import chain, combinations
+
+from .env import EXPAND_ORDINALS, OVR, RIDGE
+from .utils import BetaUpdateData, ClientResponseData, VariableType
+
+class RegressionTest():
+    @classmethod
+    def create_and_overwrite_beta(cls, y_label, X_labels, beta):
+        c = cls(y_label, X_labels)
+        c.beta = beta
+        return c
+
+    def __init__(self, y_label: str, X_labels: List[str]):
+        self.y_label = y_label
+        self.X_labels = X_labels
+        self.beta = np.zeros(len(X_labels) + 1)
+
+    def update_beta(self, data: List[BetaUpdateData]):
+        xwx = sum([d.xwx for d in data])
+        xwz = sum([d.xwz for d in data])
+
+        if RIDGE > 0:
+            penalty_matrix = RIDGE * np.eye(len(xwx))
+            xwx += penalty_matrix
+
+        try:
+            xwx_inv = np.linalg.inv(xwx)
+        except np.linalg.LinAlgError:
+            xwx_inv = np.linalg.pinv(xwx)
+
+        if RIDGE > 0:
+            self.beta = (xwx_inv @ xwz) + RIDGE * xwx_inv @ self.beta
+        else:
+            self.beta = xwx_inv @ xwz
+
+    def __lt__(self, other):
+        if len(self.X_labels) < len(other.X_labels): return True
+        elif len(self.X_labels) > len(other.X_labels): return False
+
+        if self.y_label < other.y_label: return True
+        elif self.y_label > other.y_label: return False
+
+        if tuple(sorted(self.X_labels)) < tuple(sorted(other.X_labels)): return True
+        elif tuple(sorted(self.X_labels)) > tuple(sorted(other.X_labels)): return False
+
+        return True
+
+    def __repr__(self):
+        return f'RegressionTest {self.y_label} ~ {", ".join(self.X_labels + ["1"])} - beta: {self.beta}'
+
+class Test():
+    def __init__(self,
+        y_label,
+        X_labels: List[str],
+        y_labels: List[str] = None,
+        max_iterations=25,
+        y_type=None
+    ):
+        self.y_label = y_label
+        self.X_labels = X_labels
+        if y_labels is None: y_labels = [y_label]
+        self.y_labels = y_labels
+        self.tests: Dict[str, RegressionTest] = {_y_label: RegressionTest(_y_label, X_labels) for _y_label in y_labels}
+
+        if y_type == VariableType.CATEGORICAL and OVR == 0:
+            _beta = np.concatenate([t.beta for t in self.tests.values()])
+            self.tests = {y_label: RegressionTest.create_and_overwrite_beta(y_label, X_labels, _beta)}
+
+        self.llf = None
+        self.last_deviance = None
+        self.deviance = 0
+        self.iterations = 0
+        self.max_iterations = max_iterations
+
+    def is_finished(self):
+        return self.get_change_in_deviance() < 1e-3 or self.iterations >= self.max_iterations
+
+    def update_betas(self, data: Dict[str, ClientResponseData]):
+        self.llf = {client_id: client_response.llf for client_id, client_response in data.items()}
+        self.last_deviance = self.deviance
+        self.deviance = sum(client_response.deviance for client_response in data.values())
+
+        if self.is_finished():
+            return
+
+        beta_update_data = [client_response.beta_update_data for client_response in data.values()]
+        # Transform data from list of dicts to dict of lists => all data for one y_label grouped together
+        beta_update_data = {k: [dic[k] for dic in beta_update_data] for k in beta_update_data[0]}
+
+        for y_label, _data in beta_update_data.items():
+            self.tests[y_label].update_beta(_data)
+        self.iterations += 1
+
+    def get_degrees_of_freedom(self):
+        # len tests -> num_cats -1
+        # len X_labels + 1 -> x vars, intercept
+        return len(self.y_labels)*(len(self.X_labels) + 1)
+
+    def get_llf(self, client_subset=None):
+        if client_subset is not None:
+            return sum([llf for client_id, llf in self.llf.items() if client_id in client_subset])
+        return sum([llf for llf in self.llf.values()]) if self.llf is not None else 0
+
+    def get_providing_clients(self):
+        if self.llf is None:
+            return []
+        return set(self.llf.keys())
+
+    def get_beta(self):
+        return {t.y_label: t.beta for t in self.tests.values()}
+
+    def get_required_labels(self):
+        vars = {self.y_label}
+        for var in self.X_labels:
+            if '__cat__' in var:
+                vars.add(var.split('__cat__')[0])
+            elif '__ord__' in var:
+                vars.add(var.split('__ord__')[0])
+            else:
+                vars.add(var)
+        return vars
+
+    def get_change_in_deviance(self):
+        if self.last_deviance is None:
+            return 1
+        return abs(self.deviance - self.last_deviance)
+
+    def get_relative_change_in_deviance(self):
+        if self.last_deviance is None:
+            return 1
+        return abs(self.deviance - self.last_deviance) / (1e-5 + abs(self.deviance))
+
+    def __repr__(self):
+        test_string = "\n\t- "  + "\n\t- ".join([str(t) for t in sorted(self.tests.values())])
+        test_title = f'{self.y_label} ~ {",".join(list(set([l.split("__")[0] for l in self.X_labels])))},1'
+        return f'Test {test_title} - llf: {self.get_llf()}, deviance: {self.deviance}, {self.iterations}/{self.max_iterations} iterations{test_string}'
+
+    def __eq__(self, other):
+        req_labels = self.get_required_labels()
+        other_labels = other.get_required_labels()
+        return (
+            len(req_labels) == len(other_labels) and
+            self.y_label == other.y_label and
+            tuple(sorted(self.X_labels)) == tuple(sorted(other.X_labels))
+        )
+
+    def __lt__(self, other):
+        req_labels = self.get_required_labels()
+        other_labels = other.get_required_labels()
+        if len(req_labels) < len(other_labels):
+            return True
+        elif len(req_labels) > len(other_labels):
+            return False
+
+        if self.y_label < other.y_label:
+            return True
+        elif self.y_label > other.y_label:
+            return False
+
+        if tuple(sorted(self.X_labels)) < tuple(sorted(other.X_labels)):
+            return True
+        elif tuple(sorted(self.X_labels)) > tuple(sorted(other.X_labels)):
+            return False
+
+        return False
+
+
+class TestEngine():
+    def __init__(self,
+                 schema,
+                 category_expressions,
+                 ordinal_expressions,
+                 max_regressors=None,
+                 max_iterations=25,
+                 test_targets=None
+                 ):
+
+        self.tests = []
+        self.max_iterations = max_iterations
+        self.bad_tests = []
+
+        variables = set(schema.keys())
+        max_conditioning_set_size = min(len(variables)-1, max_regressors) if max_regressors is not None else len(variables)-1
+
+        all_test_targets = set(sum([(a,b) for a,b,_ in test_targets], ())) if test_targets is not None else None
+
+        for y_var in variables:
+            if all_test_targets is not None and y_var not in all_test_targets:
+                continue
+            set_of_possible_regressors = variables - {y_var}
+            powerset_of_regressors = chain.from_iterable(combinations(set_of_possible_regressors, r) for r in range(0, max_conditioning_set_size+1))
+
+            # expand categorical features in regressor sets
+            expanded_powerset_of_regressors = []
+            for variable_set in powerset_of_regressors:
+                for _var, expressions in category_expressions.items():
+                    if _var in variable_set:
+                        variable_set = (set(variable_set) - {_var}) | set(sorted(list(expressions)[1:])) # drop first cat
+                if EXPAND_ORDINALS:
+                    for _var, expressions in ordinal_expressions.items():
+                        if _var in variable_set:
+                            variable_set = (set(variable_set) - {_var}) | set(sorted(list(expressions)[1:])) # drop first cat
+                expanded_powerset_of_regressors.append(variable_set)
+            powerset_of_regressors = expanded_powerset_of_regressors
+
+            if schema[y_var] == VariableType.CONTINUOS:
+                self.tests.extend([Test(
+                    y_label=y_var,
+                    X_labels=sorted(list(x_vars)),
+                    max_iterations=max_iterations,
+                    y_type=schema[y_var]
+                ) for x_vars in powerset_of_regressors])
+            elif schema[y_var] == VariableType.BINARY:
+                self.tests.extend([Test(
+                    y_label=y_var,
+                    X_labels=sorted(list(x_vars)),
+                    max_iterations=max_iterations,
+                    y_type=schema[y_var]
+                ) for x_vars in powerset_of_regressors])
+            elif schema[y_var] == VariableType.CATEGORICAL:
+                assert y_var in category_expressions, f'Categorical variable {y_var} is not in expression mapping'
+                self.tests.extend([Test(
+                    y_label=y_var,
+                    X_labels=sorted(list(x_vars)),
+                    y_labels=category_expressions[y_var][:-1],
+                    max_iterations=max_iterations,
+                    y_type=schema[y_var]
+                ) for x_vars in powerset_of_regressors])
+            elif schema[y_var] == VariableType.ORDINAL:
+                assert y_var in ordinal_expressions, f'Ordinal variable {y_var} is not in expression mapping'
+                self.tests.extend([Test(
+                    y_label=y_var,
+                    X_labels=sorted(list(x_vars)),
+                    y_labels=ordinal_expressions[y_var][:-1],
+                    max_iterations=max_iterations,
+                    y_type=schema[y_var]
+                ) for x_vars in powerset_of_regressors])
+            else:
+                raise Exception(f'Unknown variable type {schema[y_var]} encountered!')
+
+        self.tests: List[Test] = sorted(self.tests)
+        self.current_test_index = 0
+
+    def is_finished(self):
+        return self.current_test_index >= len(self.tests)
+
+    def get_currently_required_labels(self):
+        if self.is_finished():
+            return None
+        current_test = self.tests[self.current_test_index]
+        return current_test.get_required_labels()
+
+    def get_current_test_parameters(self):
+        if self.is_finished():
+            return None, None, None
+        current_test = self.tests[self.current_test_index]
+        return current_test.y_label, current_test.X_labels, current_test.get_beta()
+
+    def update_current_test(self, client_responses: Dict[str, ClientResponseData]):
+        if self.is_finished():
+            return
+        current_test = self.tests[self.current_test_index]
+        current_test.update_betas(client_responses)
+        if current_test.is_finished():
+            self.current_test_index += 1
+
+    def remove_current_test(self):
+        if self.is_finished():
+            return
+        current_test = self.tests[self.current_test_index]
+        self.bad_tests.append(current_test)
+        self.tests = self.tests[:self.current_test_index] + self.tests[self.current_test_index+1:]

fedci/utils.py

@@ -0,0 +1,26 @@
+import enum
+from dataclasses import dataclass
+from typing import Dict
+
+class VariableType(enum.Enum):
+    CONTINUOS = 0
+    BINARY = 1
+    CATEGORICAL = 2
+    ORDINAL = 3
+    def __eq__(self, other):
+        if self.__class__ is other.__class__:
+            return self.value == other.value
+        return NotImplemented
+    def __hash__(self):
+        return hash(self.value)
+
+@dataclass
+class BetaUpdateData:
+    xwx: object
+    xwz: object
+
+@dataclass
+class ClientResponseData:
+    llf: float
+    deviance: float
+    beta_update_data: Dict[str, BetaUpdateData]

python_fedci-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Maximilian Hahn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_fedci-0.1.0.dist-info/METADATA

@@ -0,0 +1,112 @@
+Metadata-Version: 2.2
+Name: python-fedci
+Version: 0.1.0
+Summary: A small package for federated independence tests
+Author-email: Maximilian Hahn <max.hahn@gmx.de>
+License: MIT License
+        
+        Copyright (c) 2025 Maximilian Hahn
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/maxhahn/fedci
+Project-URL: Repository, https://github.com/maxhahn/fedci
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.22.0
+Requires-Dist: pyarrow>=19.0.0
+Requires-Dist: rpyc>=6.0.1
+Requires-Dist: scipy>=1.15.1
+Requires-Dist: statsmodels>=0.14.4
+
+# About this Project
+
+This projects aims to create an accessible platform and user interface to obtain causal knowledge from distributed datasets.
+Different parties can share key insights about their data without compromising their privacy.
+Results come in the form of causal graphs (PAGs).
+
+There are two main algorithm this paper supports:
+* rIOD
+* FedGLM
+
+## General
+
+This application is made up of two main components.
+There is a streamlit UI and a litestar server.
+
+Via the streamlit UI, one can connect to an existing server instance and run distributed/federated algorithms with peers connected to the same server.
+
+It is designed to be easily self-hostable, and is fully contained within a docker container.
+
+**Beware**: As of now, this client-server architecture communicates via http (_not_ https!).
+As such a malicious agent may spoof your identity or steal data that is transmitted over the network.
+
+When hosting this application, you may use a reverse-proxy and your own SSL certificates to enable https.
+
+## rIOD
+
+This project implements the IOD algorithm created by [Tillman and Spirtes](http://proceedings.mlr.press/v15/tillman11a.html).
+
+When running rIOD, (conditional) independence tests are performed and the resulting p-values are transmitted to the server.
+On the server-side, these p-values are aggregated to give insights about the independences in the distributed dataset.
+
+rIOD only supports numerical (float) features.
+As per the IOD algorithm, not all participating parties have to have identical features in their dataset.
+
+Without code modifications, this application will only ever transmit p-values and PAG adjacency matrices over the network.
+The original dataset is kept completely private and is never transmitted over the network.
+(This can be easily checked inside the source code)
+
+## FedGLM
+
+This project implemens an algorithm (titled FedGLM for now) which utilizes federated learning to create linear models, which are then used for likelihood-ratio tests in order to obtain independence information about the dataset.
+
+FedGLM supports numerical (float), categorical (string), and ordinal (int) features.
+Similar to IOD, here as well, not all participating parties have to have the exact same feature set.
+
+This algorithm requires the transmission of the expression levels of ordinal and categorical variables.
+Additionaly, when the algorithm is running, linear model coefficients are transmitted, as well as matrices that do not contain recreatable information about the data (see algorithm 2 in [Cellamare et al.](https://www.mdpi.com/1999-4893/15/7/243))
+As such, data privacy is preserved.
+
+## Setup
+
+First, install docker and docker-compose.
+
+Use the following commands to build and run the application:
+
+* Run client and server on same machine:
+`docker-compose up`
+
+* Run client or server only:
+`docker-compuse up client`
+`docker-compose up server`
+
+* The client can be accessed via `localhost:8081` on the host machine.
+When hosting the client on a machine within the same network, use it's ip address and ensure proper connectivity between the two machines.
+
+* The first step within the client is to connect to a server.
+A server hosted by us can be used with the following URL: `heiderlab.com:8080`
+
+## Configuration
+
+No major changes of this setup should be required.
+All configurations can be changed in `docker-compose.yml`.
+
+The only reasonable change is the port mapping, if your host machine requires specific ports to be exposed.

python_fedci-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+fedci/__init__.py,sha256=v0ZdJg0rQaCGcfrKuFuUV0s22DDHpN0iRJaCcMo00aY,167
+fedci/client.py,sha256=yYPT6I2Ek4C5GKKvKIwV4XQ_kfghi_pNFqX29FhCuhA,15939
+fedci/env.py,sha256=BqLlyUxC8of89P96bH_H5_2vzlVoUupwAPA7po7IvHs,309
+fedci/evaluation.py,sha256=vHqIdPfo7JI-ux0PL6byl2ltJ4nwN57OPAxVmvIWVGk,6293
+fedci/server.py,sha256=eXhkwEwnPDGmTvfupxTrobx_d2uAPHqtAXc7H6-vBmE,4711
+fedci/testing.py,sha256=x5lVmwD8ZK2QZ-jhncfQNcBXk5ZovM2Lp_XMc4A_D7M,10988
+fedci/utils.py,sha256=GQ8xeU7SRqUgnoJdynMnDCT8_qn_6j6XnWsoar4LiPg,564
+python_fedci-0.1.0.dist-info/LICENSE,sha256=zm3uXqpOuah2iueuXaCJZxXk1xZNzJzfo_csVwG56ZU,1072
+python_fedci-0.1.0.dist-info/METADATA,sha256=TSl-tWX8TEbXBMgIieitpEiSaNHNnX1fCADCHOfipPQ,5253
+python_fedci-0.1.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+python_fedci-0.1.0.dist-info/top_level.txt,sha256=cqbR8aI0m4KvYOXgv94Lg1XmKxIBBAFKyDmJD1zefDQ,6
+python_fedci-0.1.0.dist-info/RECORD,,

python_fedci-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

python_fedci-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fedci