rulelearn-ripper-interactive 0.2.3__py3-none-any.whl

rulelearn_ri/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("rulelearn-ripper-interactive")
+except PackageNotFoundError:
+    pass

rulelearn_ri/algorithms/r2n/base.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+
+from torch import nn
+
+
+class RuleNetwork(nn.Module, ABC):
+    """
+    Encapsulates a neural network that corresponds syntactically to some ruleset.
+    """
+
+    @abstractmethod
+    def extract_ruleset(self):
+        """ Extract a symbolic representation of the learned ruleset. The semantics of the ruleset must be provided
+        externally.
+
+        :return: A 2D list representing a ruleset in Disjunctive Normal Form. An inner list takes values from {-1, 0, 1}
+         representing a single rule, indicating the inclusion of the negated condition, exclusion,
+         and inclusion of the condition, respectively.
+         E.g. [[1, 0, 0, -1], [0, 1, 0, 0]] represents the ruleset
+         "(condition0 ^ not condition3) v condition1" for a rule network with 4 input conditions.
+        """
+        raise NotImplementedError
+
+
+class PredicateLearningLayer(nn.Module, ABC):
+    @abstractmethod
+    def get_predicates(self, *args, **kwargs):
+        raise NotImplementedError

rulelearn_ri/algorithms/r2n/functions.py

@@ -0,0 +1,25 @@
+import torch
+
+
+class DifferentiableHeaviside(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, input):
+        ctx.save_for_backward(input)
+        return torch.heaviside(input, torch.ones_like(input))
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        input, = ctx.saved_tensors
+        return grad_output * torch.sigmoid(input) * (1 - torch.sigmoid(input))
+
+
+class DifferentiableClamp(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, input):
+        ctx.save_for_backward(input)
+        return torch.clamp(input, max=1)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        input, = ctx.saved_tensors
+        return grad_output

rulelearn_ri/algorithms/r2n/mask.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+
+import torch
+from torch import nn
+
+from rulelearn_ri.algorithms.r2n.functions import DifferentiableHeaviside
+
+
+class WeightMask(nn.Module, ABC):
+    """
+    Abstract class for the mask that is applied to the weight matrix of the NN. 
+    """
+
+    def __init__(self, dim_out, dim_in):
+        super(WeightMask, self).__init__()
+        self.dim_out = dim_out
+        self.dim_in = dim_in
+        self.loc = torch.nn.Parameter(torch.randn(self.dim_out, self.dim_in), requires_grad=True)
+
+    @abstractmethod
+    def forward(self):
+        pass
+
+
+class SimpleWeightMask(WeightMask):
+    """
+    Binary weight mask that is used in the AND/OR layer. Note that depening of the Boolean value cooling, we either implement a cooled sigmoid or stricly binary differentiable heavyside function. 
+    """
+
+    def __init__(self, dim_out, dim_in, temp, cooling):
+        super(SimpleWeightMask, self).__init__(dim_out, dim_in)
+        self.cooling = cooling
+        self.temp = temp
+
+    def forward(self):
+        if self.cooling is True:
+            return torch.sigmoid((1 / self.temp) * self.loc)
+        else:
+            return DifferentiableHeaviside.apply(self.loc)
+
+    def set_temp(self, temp):
+        self.temp = temp
+
+    def get_temp(self):
+        return self.temp
+
+
+class DeterministicWeightMask(WeightMask):
+    """
+    Simple deterministic weight mask. 
+    """
+
+    def __init__(self, mask):
+        super(DeterministicWeightMask, self).__init__(mask.size()[0], mask.size()[1])
+        self.mask = mask
+
+    def forward(self):
+        return self.mask

rulelearn_ri/algorithms/r2n/preprocessing_layers.py

@@ -0,0 +1,192 @@
+import inspect
+import itertools
+import operator
+from copy import copy
+from functools import reduce
+
+import numpy as np
+import torch
+from torch import nn
+
+import rulelearn_ri.algorithms.r2n.mask as wm
+from rulelearn_ri.algorithms.r2n.base import PredicateLearningLayer
+from rulelearn_ri.trxf.core import Feature, Predicate, Relation
+
+
+class NegationLayer(nn.Module):
+    """
+    Augment the input layer with its negation to explicitly provide negated conditions to the downstream rule net.
+    """
+
+    def __init__(self):
+        super(NegationLayer, self).__init__()
+
+    def forward(self, x):
+        return torch.cat((x, torch.ones_like(x) - x), 1)
+
+
+def _normalize_weights(bias, weights):
+    max_coef = np.max(np.abs(weights), axis=1)
+    normbias = np.divide(bias, max_coef)
+    max_coef = np.tile(max_coef, (weights.shape[1], 1)).T
+    normweights = np.divide(weights, max_coef)
+    return normbias, normweights
+
+
+class LinearFeatureLayer(PredicateLearningLayer):
+    r"""
+    This layer takes as input a set of input featues and creates a linear combinations thereof to constuct a predicate in the form \sum w x - b > 0
+    where w and b are the weights and biases of the neural network layer.
+    """
+
+    def __init__(self, dim_in, dim_out, init_temp=1):
+        super(LinearFeatureLayer, self).__init__()
+        torch.manual_seed(0)
+        self.layer = nn.Linear(dim_in, dim_out)
+        self.mask = wm.DeterministicWeightMask(torch.ones(dim_out, dim_in))
+        self.temp = init_temp
+        self.dim_out = dim_out
+
+    def forward(self, x):
+        z = self.mask()
+        regularized_weights = z * self.layer.weight
+        self.layer.weight.data = regularized_weights
+        return torch.sigmoid((1 / self.temp) * self.layer(x))
+
+    def get_penalty(self):
+        # return self.mask().mean()
+        return self.layer.weight.norm(1)
+
+    def update_temp(self, temp):
+        self.temp = temp
+
+    def get_temp(self):
+        return self.temp
+
+    def get_predicates(self, column_names, normalizer, thr, name_format='generic'):
+        weight = copy(self.layer.weight.detach().numpy())
+        bias = copy(self.layer.bias.detach().numpy())
+        assert len(column_names) == weight.shape[1], 'column_names: {} but weight dim_in: {}'.format(column_names,
+                                                                                                     weight.shape[1])
+        norm_bias, norm_weights = _normalize_weights(bias, weight)
+        weight[abs(norm_weights) < thr] = 0
+        bias[abs(norm_bias) < thr] = 0
+        if normalizer:
+            weight = normalizer.transform(weight)
+
+        features = []
+        for w, b in zip(weight, bias):
+            terms = [x for x in zip(w, column_names) if abs(x[0]) > 0]
+            feature = ' + '.join(list(map(lambda term: str(term[0]) + '*' + str(term[1]), terms)))
+            feature += ' + ' + str(b)
+            features.append(feature.replace('+ -', '- '))
+
+        predicates = list(map(lambda expression: Predicate(Feature(expression), Relation.GE, 0), features))
+        return predicates
+
+
+class BasisExpansionLayer(nn.Module):
+    def __init__(self, basis_functions, application_list=None):
+        super().__init__()
+        self.functions = basis_functions + [lambda x: x]
+        self.application_list = application_list
+
+    def forward(self, x):
+        result = []
+        batch_size = x.size()[0]
+        for i in range(batch_size):
+            batch_result = []
+            for f in self.functions:
+                arity = len(inspect.getfullargspec(f).args)
+                args = [x[i]] * arity
+                cartesian = torch.cartesian_prod(*args).detach().numpy()
+                if arity == 1:
+                    cartesian = np.expand_dims(cartesian, axis=1)
+                batch_result += [f(*args) for args in cartesian]
+            result.append(batch_result)
+
+        return torch.FloatTensor(result)
+
+
+class ExpandedLinearFeatureLayer(PredicateLearningLayer):
+    def __init__(self, basis_functions, dim_in, dim_out):
+        super().__init__()
+        torch.manual_seed(0)
+
+        self.functions = basis_functions
+        self.dim_in = dim_in
+        self.dim_out = dim_out
+        self.expansion_layer = BasisExpansionLayer(self.functions)
+        arities = [len(inspect.getfullargspec(f).args) for f in self.functions] + [1]
+        dim_mid = sum([dim_in ** n for n in arities])
+        self.linear_feature_layer = LinearFeatureLayer(dim_mid, self.dim_out)
+
+    def forward(self, x):
+        return nn.Sequential(self.expansion_layer, self.linear_feature_layer)(x)
+
+    def get_predicates(self, column_names, normalizer, thr, name_format='generic'):
+        norm_obj = copy(normalizer)
+        if normalizer is not None:
+            scale = normalizer.scale_
+            expanded_scale = _expand_normalizer(scale, self.functions)
+            norm_obj.scale_ = expanded_scale
+            norm_obj.n_features_in_ = len(expanded_scale)
+        expanded_col_names = _expand_columns(column_names, self.functions, name_format)
+        return self.linear_feature_layer.get_predicates(expanded_col_names, norm_obj, thr, name_format)
+
+    def get_penalty(self):
+        return self.linear_feature_layer.get_penalty()
+
+    def update_temp(self, temp):
+        self.linear_feature_layer.update_temp(temp)
+
+
+def _expand_columns(column_names, functions, name_format):
+    names = []
+    for i, f in enumerate(functions):
+        arity = len(inspect.getfullargspec(f).args)
+        repeat = [column_names] * arity
+        cols = itertools.product(*repeat)
+        names += ['f{}_'.format(i) + '_'.join(tup) for tup in cols] if name_format == 'generic' \
+            else ['*'.join(tup) for tup in cols]
+
+    return names + column_names
+
+
+def _expand_normalizer(scale, functions):
+    expanded_scale = []
+    for i, f in enumerate(functions):
+        arity = len(inspect.getfullargspec(f).args)
+        repeat = [scale] * arity
+        scales = itertools.product(*repeat)
+        expanded_scale += [_prod(tup) for tup in scales]
+
+    return np.concatenate((np.array(expanded_scale), scale))
+
+
+def _prod(iterable):
+    return reduce(operator.mul, iterable, 1)
+
+
+class PredicateLayer(nn.Module):
+    """
+    This predicate layer combines categorical and numerical input features and, depending of the NegLayer flag, negates the categorical features.
+    """
+
+    def __init__(self, learning_layer, negation):
+        super(PredicateLayer, self).__init__()
+        self.LearningLayer = learning_layer
+        self.negation = negation
+
+    def forward(self, x):
+        x_cat_in = x[0]
+        x_cat_out = x_cat_in
+        x_num_in = x[1]
+        x_num_out = x_num_in
+        if x_num_in.nelement() != 0:
+            x_num_out = self.LearningLayer(x_num_in)
+        if x_cat_in.nelement() != 0 and self.negation:
+            neglayer = NegationLayer()
+            x_cat_out = neglayer(x_cat_out)
+        result = torch.cat((x_num_out, x_cat_out), -1)
+        return result

rulelearn_ri/algorithms/r2n/r2n_algo.py

@@ -0,0 +1,287 @@
+import math
+
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from torch import nn
+from torch.utils.data import DataLoader, Dataset
+
+import rulelearn_ri.algorithms.r2n.preprocessing_layers as pp
+import rulelearn_ri.algorithms.r2n.simple_rulenet as rn
+import rulelearn_ri.algorithms.r2n.utilities as util
+from rulelearn_ri.algorithms.r2n.training import train as train_r2n
+from rulelearn_ri.trxf.core.conjunction import Conjunction
+from rulelearn_ri.trxf.core.dnf_ruleset import DnfRuleSet
+from rulelearn_ri.trxf.core.feature import Feature
+from rulelearn_ri.trxf.core.predicate import Predicate, Relation
+
+
+class R2Nalgo:
+    """
+    Wrapper for the R2N algorithm which is described in the following paper: arxiv.org/abs/2201.06515. 
+    """
+
+    def __init__(self,
+                 cat_columns: list = None,
+                 num_columns: list = None,
+                 batch_size: int = 100,
+                 learning_rate: float = 1e-3,
+                 coef: float = 1e-3,
+                 n_rules: int = 25,
+                 n_conditions: int = 10,
+                 n_seeds: int = 2,
+                 init_temp: float = 1,
+                 min_temp: float = 10 ** -4,
+                 decay_rate: float = 0.999,
+                 negation: bool = False,
+                 normalize_num: bool = False,
+                 basis_functions: list = None,
+                 name_format='generic'
+                 ):
+        """
+        batch_size : int
+            The batch size used during the training of the NN
+        learning rate : float
+            Learning rate of the neural network
+        coef: float
+            Sparsity promoting coefficient. Larger is sparser.
+        n_rules: int
+            Numbers of rules in the and layer
+        n_conditions: int
+            Numbers of output nodes of the multivariate predicate layer 
+        n_seeds: int
+            Number of times the optimizer and network weights are reinitialized (number of runs)
+        max_epochs: int
+            Number of epochs ran per seed 
+        init_temp: float
+            Initial temperature of the cooling schedule
+        min_temp: float
+            Minimal temperature of the cooling schedule
+        decay_rate: float
+            Decay rate of the cooling schedule
+        negation: bool
+            Whether negations of categorical features are considered or not
+        normalize_num: bool
+            Scale the input features before training. Only works for polynomial basis functions.
+        basis_functions: list of lambdas
+            List of basis expansion functions to be applied to the original input
+        name_format: bool
+            An option to specify a more informative feature names for some basis expansions (currently only quadratic).
+            E.g., x1*x0 instead of f1_x1_x0 as the feature name.
+         """
+        super(R2Nalgo, self).__init__()
+        self.batch_size = batch_size
+        self.learning_rate = learning_rate
+        self.coef = coef
+        self.n_rules = n_rules
+        self.n_conditions = n_conditions
+        self.n_seeds = n_seeds
+        self.decay_rate = decay_rate
+        self.init_temp = init_temp
+        self.min_temp = min_temp
+        self.max_epochs = math.ceil(
+            1.2 * (math.log(self.min_temp) - math.log(self.init_temp)) / math.log(self.decay_rate))
+        self.negation = negation
+        self.normalize_num = normalize_num
+        self.basis_functions = basis_functions
+        self.name_format = name_format
+
+        self.encoder_object = None
+        self.norm_object = None
+        self.n_cat_features, self.n_num_features = None, None
+        self.num_column_names, self.cat_column_names =  num_columns, cat_columns
+        self.model = None
+        self.opt_rule = None
+        self.predicate_learning_layer = None
+
+    def fit(self, train, train_labels):
+        """
+        The fit function for R2N algorithm. All float and int dtypes are considered numerical features. 
+        All others are treated as categorical.
+        """
+
+        # Partition the input features through their dtype if cat and num features are not provided. 
+        train.columns = train.columns.str.replace(' ', '')
+        if self.cat_column_names == None: 
+            self.cat_column_names = list(train.select_dtypes(exclude=['float']).columns.str.replace(' ', ''))         
+        if self.num_column_names == None: 
+            self.num_column_names = list(train.select_dtypes(include=['float']).columns.str.replace(' ', ''))   
+        X_cat, X_num, train_dataloader = self._preprocess(train, train_labels)
+
+        # Training loop 
+        loss_fn = nn.MSELoss()
+        best_loss = math.inf
+        best_acc = -1
+        for _ in np.arange(0, self.n_seeds):
+
+            self.model, mlpnet, rulenet = self._construct_r2n()
+            optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate, amsgrad=True)
+            temp = self.init_temp
+            for t in range(self.max_epochs):
+                temp, loss, success_rate_train = train_r2n(train_dataloader, self.model, loss_fn, optimizer, temp,
+                                                           self.min_temp, self.decay_rate, self.coef, mv_reg=self.coef,
+                                                           multivariate=(self.n_num_features != 0))
+                if t % 10 == 0:
+                    print("Epoch:", t, "Success rate:", '%.1f' % success_rate_train, "Loss:", '%.5f' % loss,
+                          "Temperature:", '%.1e' % temp)
+                if temp < self.min_temp and loss < best_loss:
+                    if self.n_num_features > 0:
+                        self.predicate_learning_layer = mlpnet
+                    self.opt_rule = rulenet.extract_ruleset()
+                    best_loss = loss
+                    best_acc = success_rate_train
+            print('---------')
+
+        print("Max performance of the network:", best_acc)
+
+        # Post-processing
+        binarized_input = _binarize_input(X_num, X_cat, self.predicate_learning_layer, self.negation)
+        self.opt_rule = util.simplify_ruleset(util.filter_trivial_predicates(self.opt_rule, binarized_input))
+
+    def predict(self, features: pd.DataFrame) -> pd.DataFrame:
+        """
+        :param features: A pandas dataframe containing feature values
+        :return predictions: A pandas dataframe containing a single column called '__predictions__'
+        """
+        features.columns = features.columns.str.replace(' ', '')
+        _, X_cat_t, X_num_t = self._data_preprocessing(features, None, training=False)
+        pred_evaluation = _binarize_input(X_num_t, X_cat_t, self.predicate_learning_layer, self.negation)
+        eval_sol = util.batch_apply_ruleset(self.opt_rule, pred_evaluation.T)
+        predictions = pd.DataFrame(data=eval_sol, columns=['__predictions__'])
+
+        return predictions
+
+    def export_rules_to_trxf_dnf_ruleset(self, thr=0):
+        """
+        Args:
+            thr: Threshold to remove negligible coefficients. Any coefficient that has a smaller ratio to the dominating
+            coefficient than thr will be eliminated in the extracted ruleset.
+
+
+        Returns:
+            trxf.DnfRuleSet
+        """
+        learned_predicates = []
+        predetermined_predicates = []
+        if self.n_num_features > 0:
+            learned_predicates = self.predicate_learning_layer.get_predicates(
+                self.num_column_names, self.norm_object, thr, self.name_format)
+        if self.n_cat_features > 0:
+            a = list(zip(self.cat_column_names, self.encoder_object.categories_))
+            for item in a:
+                col_name = item[0]
+                categories = item[1]
+                predetermined_predicates += [Predicate(Feature(col_name), Relation.EQ, v) for v in categories]
+        predicates = learned_predicates + predetermined_predicates
+
+        return DnfRuleSet([Conjunction([pred for pred, w in zip(predicates, rule) if w > 0]) for rule in self.opt_rule],
+                          then_part='Churn')
+
+    def _preprocess(self, train, train_labels):
+        y, X_cat, X_num = self._data_preprocessing(train, train_labels, training=True)
+        self.n_num_features = X_num.shape[1] if X_num is not None else 0
+        self.n_cat_features = X_cat.shape[1] if X_cat is not None else 0
+        dataset = _NumCatDataset(y, X_num, X_cat)
+        train_dataloader = DataLoader(dataset, self.batch_size)
+        return X_cat, X_num, train_dataloader
+
+    def _construct_r2n(self):
+        pred_learning = None
+        dim_num = 0
+        if self.n_num_features > 0:
+            if self.basis_functions is not None:
+                pred_learning = pp.ExpandedLinearFeatureLayer(self.basis_functions, self.n_num_features, self.n_conditions)
+            else:
+                pred_learning = pp.LinearFeatureLayer(self.n_num_features, self.n_conditions)
+            dim_num = pred_learning.dim_out
+        pred_layer = pp.PredicateLayer(pred_learning, self.negation)
+        dim_cat = 2 * self.n_cat_features if self.negation else self.n_cat_features
+        rulenet = rn.SimpleRuleNet(dim_num + dim_cat, self.n_rules)
+        model = nn.Sequential(pred_layer, rulenet)
+        return model, pred_learning, rulenet
+    
+    def _data_preprocessing(self, train, train_labels, training):
+        """
+        Imports a pandas data frame and selects the numerical ('float') and categorical (NOT 'float') input features and
+        returns a pytorch dataframe and the number of categorical and numerical features.
+        """
+        X_cat, X_num = None, None
+        table_X_num, table_X_cat = train[self.num_column_names], train[self.cat_column_names] 
+
+        if len(self.num_column_names)>0:
+            if self.normalize_num:
+                if training:
+                    self.norm_object = StandardScaler(with_mean=False)
+                    table_X_num = self.norm_object.fit_transform(np.array(table_X_num))
+                else:
+                    assert self.norm_object is not None
+                    table_X_num = self.norm_object.transform(np.array(table_X_num))
+            X_num = np.array(table_X_num, dtype=float)
+
+        if len(self.cat_column_names)>0:
+            if training:
+                print(table_X_num.shape[1], "numerical features and ", table_X_cat.shape[1], "categorical features")
+                print('---------')
+                self.encoder_object, X_cat = _categorical_encoding(table_X_cat.astype(str))
+                print(X_cat.shape[1], "categorical dummies")
+                print('---------')
+            else:
+                assert self.encoder_object is not None
+                X_cat = self.encoder_object.transform(table_X_cat.astype(str)).toarray()
+
+        if training:
+            y = np.array(train_labels, dtype=int)
+        else:
+            y = None
+
+        return y, X_cat, X_num
+
+
+
+# Preprocessing function R2N ####
+
+def _categorical_encoding(X_cat):
+    """
+    Encodes all categorical variables with OneHot encoding
+    """
+    ohe = OneHotEncoder(handle_unknown='ignore')
+    ohe.fit_transform(X_cat)
+    X = ohe.transform(X_cat).toarray()
+    return ohe, X
+
+
+class _NumCatDataset(Dataset):
+    def __init__(self, y, X_num=None, X_cat=None):
+        self.X_num = X_num
+        self.X_cat = X_cat
+        self.num_tensor = torch.tensor(X_num, dtype=torch.float64) if X_num is not None else None
+        self.cat_tensor = torch.tensor(X_cat, dtype=torch.float64) if X_cat is not None else None
+        self.y_train = torch.tensor(y)
+
+    def __len__(self):
+        return len(self.y_train)
+
+    def __getitem__(self, idx):
+        xcat = self.cat_tensor[idx] if self.X_cat is not None else torch.empty(0)
+        xnum = self.num_tensor[idx] if self.X_num is not None else torch.empty(0)
+        return (xcat, xnum), self.y_train[idx]
+
+
+def _binarize_input(X_num=None, X_cat=None, predicate_learning_layer=None, negation=False):
+    """
+    Evaluates the predicates and concatenates if necessary the numerical and categorical ones.
+    """
+    pred_evaluation = None
+    if X_num is not None:
+        pred_evaluation = (predicate_learning_layer(torch.tensor(X_num).float()) > 0.5).T.detach().numpy()
+    if X_cat is not None:
+        if X_num is not None:
+            pred_evaluation = np.concatenate((pred_evaluation, X_cat.T), axis=0)
+        else:
+            pred_evaluation = X_cat.T
+        if negation is not False:
+            pred_evaluation = np.concatenate((pred_evaluation, np.logical_not(X_cat.T)), axis=0)
+    if pred_evaluation is None:
+        raise ValueError('The input X_num and X_cat should not be both None')
+    return pred_evaluation

rulelearn_ri/algorithms/r2n/simple_rulenet.py

@@ -0,0 +1,75 @@
+import torch
+from torch import nn
+
+import rulelearn_ri.algorithms.r2n.mask as wm
+from rulelearn_ri.algorithms.r2n.base import RuleNetwork
+
+
+class SimpleRuleNet(RuleNetwork):
+    """
+    Class that assembles the rule network part of r2n. It initializes and configures the conjunction and disjunction layer. 
+    """
+    def __init__(self, n_conditions, n_rules, cooling=True):
+        super(SimpleRuleNet, self).__init__()
+        self.n_rules = n_rules
+        self.n_conditions = n_conditions
+        self.temp = 1
+        self.cooling = cooling
+        self.weightmask_1 = wm.SimpleWeightMask(self.n_rules, self.n_conditions, self.temp, self.cooling)
+        self.weightmask_2 = wm.SimpleWeightMask(1, self.n_rules, self.temp, self.cooling)
+        self.network = nn.Sequential(
+            ConjunctionLayer(self.weightmask_1),
+            DisjunctionLayer(self.weightmask_2),
+        )
+
+    def forward(self, x):
+        return self.network(x)
+
+    def extract_ruleset(self):
+        ruleset = []
+        w_conj = torch.heaviside(self.network[0].mask() - torch.tensor(0.5),
+                                 torch.ones_like(self.network[0].mask()))
+        w_disj = torch.heaviside(self.network[1].mask() - torch.tensor(0.5),
+                                 torch.ones_like(self.network[1].mask()))
+        for d_idx, d_weight in enumerate(torch.flatten(w_disj).tolist()):
+            if d_weight > 0:
+                rule = w_conj.tolist()[d_idx]
+                if sum(rule) > 0:
+                    ruleset.append(rule)
+        return ruleset
+
+    def get_penalty(self):
+        and_masks = self.network[0].mask()
+        or_masks = self.network[1].mask()
+
+        return (or_masks.sum() + and_masks.sum()) / self.n_rules
+
+    def update_temp(self, temp):
+        self.weightmask_1.set_temp(temp)
+        self.weightmask_2.set_temp(temp)
+
+
+class ConjunctionLayer(nn.Module):
+    """
+    Conjuction layer (And layer)
+    """
+    def __init__(self, mask):
+        super(ConjunctionLayer, self).__init__()
+        self.mask = mask
+
+    def forward(self, x):
+        neg_x = torch.ones_like(x) - x
+        prod = torch.matmul(neg_x, torch.transpose(self.mask().float(), 0, 1))
+        return torch.ones_like(prod) - torch.clamp(prod, max=1)
+
+
+class DisjunctionLayer(nn.Module):
+    """
+    Disjunction layer (OR layer)
+    """
+    def __init__(self, mask):
+        super(DisjunctionLayer, self).__init__()
+        self.mask = mask
+
+    def forward(self, x):
+        return torch.clamp(torch.matmul(self.mask(), torch.transpose(x, 0, 1)), max=1)