linearrf 1.0.0__py3-none-any.whl

linearrf-1.0.0.dist-info/METADATA

@@ -0,0 +1,17 @@
+Metadata-Version: 2.1
+Name: linearrf
+Version: 1.0.0
+Summary: A python libary to build Random Forests with Linear Models at the leaves.
+Author-email: Marian Biermann <marianbiermann@gmx.de>
+Project-URL: homepage, https://github.com/marianbiermann/lrf
+Keywords: ml,rf,linear model,tree,dart,model tree,linear tree
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: numpy>=1.20.3
+Provides-Extra: dev
+Requires-Dist: sklearn; extra == "dev"
+

linearrf-1.0.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+lrf/__init__.py,sha256=uiaviCEyab13xWdm0xvVwliqyK5Tk0_KyOvMIvXdKwE,44
+lrf/_base_lrf.py,sha256=xgVQItslZd1c8lHosr-oX8C3jigGyFELpWAl_HO3EVE,14446
+lrf/_bfgs.py,sha256=xMgi3vV3JuYLpAF8VIjmoCtT4vmMmrc-1T2rpCWyOlc,6341
+lrf/_criterion.py,sha256=iKBzfroMT7AuN6urZxFDuMlvH46FZZcG7y36pEpGr24,3989
+lrf/_linear_models.py,sha256=b-hwsKJc3sH7vqtf_VVTj7Vfp0IC85WUlaNdbCc7L5U,8512
+lrf/_node.py,sha256=_4EmSxUZwWPtu0CJhC5iGJ0MOFNyHQlQByLwYF_r4tA,448
+lrf/_preprocessor.py,sha256=KibBIUhXOoVCvX3v3ZBsaZ4rjq5do7oA0Zjm26Qur2w,4103
+lrf/lrf.py,sha256=EZujkK7tCf9sMEjBT2lI5JjyAvXAhiImcYSf5LVEy2E,10027
+linearrf-1.0.0.dist-info/LICENSE.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linearrf-1.0.0.dist-info/METADATA,sha256=BIflfzWZuSBx1DFgVz9CpB2JnJetwJz6_iZlJpE6JEA,631
+linearrf-1.0.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+linearrf-1.0.0.dist-info/top_level.txt,sha256=Er_kmcN7GxzkBUkt7tBX7sAOd4tNlNDLkEPEX61pVok,4
+linearrf-1.0.0.dist-info/RECORD,,

linearrf-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.3.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

linearrf-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lrf

lrf/__init__.py

@@ -0,0 +1 @@
+from .lrf import LRFClassifier, LRFRegressor

lrf/_base_lrf.py

@@ -0,0 +1,360 @@
+import copy
+import datetime
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from multiprocessing import cpu_count
+from typing import List, Union
+
+import numpy as np
+
+from lrf._linear_models import Regressor, Classifier
+from lrf._node import Node
+
+
+class _LinearRandomForest:
+    def __init__(self, linear_model: Union[Regressor, Classifier] = None, n_estimators: int = 100, max_depth: int = 5,
+                 criterion: str = None, n_splits: int = 15, split_samples_to_features_ratio: float = 4.5,
+                 leaf_samples_to_features_ratio: float = 2.0, min_abs_improvement: float = 5*10**(-4),
+                 warm_start: bool = True, n_jobs: int = -1, random_state: int = None, verbose: bool = False,
+                 classification: bool = False):
+        self.linear_model = linear_model
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.criterion = criterion
+        self.n_splits = n_splits
+        self.split_samples_to_features_ratio = split_samples_to_features_ratio
+        self.leaf_samples_to_features_ratio = leaf_samples_to_features_ratio
+        self.min_abs_improvement = min_abs_improvement
+        self.warm_start = warm_start
+        self.n_jobs = n_jobs
+        self.random_state = random_state
+        self.verbose = verbose
+        self.classification = classification
+
+    def _init_more_attributes(self, y):
+        if self.classification:
+            self.classes_ = None
+
+        self.forest = None
+        self.min_samples_split = None
+        self.min_samples_leaf = None
+
+        if self.max_depth is None:
+            self.max_depth = 10 ** 32
+
+        if self.n_jobs == -1 or self.n_jobs == 0:
+            self.n_jobs = cpu_count()
+        else:
+            self.n_jobs = min(self.n_jobs, cpu_count())
+
+        if self.split_samples_to_features_ratio < self.leaf_samples_to_features_ratio * 2:
+            self.split_samples_to_features_ratio = self.leaf_samples_to_features_ratio * 2
+
+        self.min_samples_split = None
+        self.min_samples_leaf = None
+
+        self.total_data_points = y.shape[0]
+
+    def fit(self, x: np.ndarray, y: np.ndarray):
+
+        assert y.ndim == 1
+        assert not np.all(y == y[0])
+
+        self._init_more_attributes(y=y)
+
+        if self.classification:
+            self._check_targets_classification(y)
+
+        random_state_list = np.random.default_rng(self.random_state).integers(2**63, size=self.n_estimators)
+
+        self.min_samples_split = self.split_samples_to_features_ratio * x.shape[1]
+        self.min_samples_leaf = self.leaf_samples_to_features_ratio * x.shape[1]
+
+        forest = []
+
+        # add intercept here and not inside linear model for performance reasons
+        x = np.insert(x, 0, 1, axis=1)
+
+        # parallel process combinations of chunks of the data
+        with ProcessPoolExecutor(max_workers=self.n_jobs) as executor:
+            if self.verbose:
+                print('\nStart growing trees...')
+                finished_tasks = 0
+                start_time = time.time()
+
+            results = [executor.submit(self._grow_tree, x=x, y=y, random_state=i) for i in random_state_list]
+
+            # collect the results and print the progress
+            for r in as_completed(results):
+                # collecting results
+                grown_tree = r.result()
+                forest.append(grown_tree)
+
+                # printing progress
+                if self.verbose:
+                    finished_tasks += 1
+                    self._print_progress(frac=finished_tasks/self.n_estimators, start_time=start_time)
+
+        self.forest = forest
+        
+        if self.verbose:
+            elapsed_seconds = round(time.time() - start_time)
+            print('Finished planting the forest in {}     '.format(str(datetime.timedelta(seconds=elapsed_seconds))))
+
+    @staticmethod
+    def _print_progress(frac: float, start_time: float):
+        """
+        Prints the progress of the parallel multiprocessing
+        Args:
+            frac (int): Fraction of tasks which are already finished
+
+        """
+        elapsed_seconds = round(time.time() - start_time)
+        remaining_seconds = round(elapsed_seconds / frac - elapsed_seconds)
+        print('LRF - Progress: {}%, [{}<{}]'.format(
+            round(100 * frac, 2),
+            str(datetime.timedelta(seconds=elapsed_seconds)),
+            str(datetime.timedelta(seconds=remaining_seconds))
+        ), end='\r')
+
+    def _grow_tree(self, x: np.ndarray, y: np.ndarray, random_state: int):
+
+        rng = np.random.default_rng(random_state)
+
+        idx = rng.choice(np.arange(x.shape[0]), x.shape[0])
+        x = x[idx]
+        y = y[idx]
+
+        tree = self._root_node(x=x, y=y)
+
+        # split
+        tree = self._split(node=tree, x=x, y=y, depth=0, rng=rng)
+
+        return tree
+
+    def _root_node(self, x: np.ndarray, y: np.ndarray):
+        # initial linear model
+        root_model = copy.deepcopy(self.linear_model)
+
+        if isinstance(root_model, (Regressor, Classifier)):
+            root_model.fit(x, y, None)
+        else:
+            root_model.fit(x, y)
+
+        if self.criterion == 'cross_entropy':
+            y_pred = root_model.predict_proba(x)
+        elif (self.criterion == 'neg_roc_auc') or (self.criterion == 'neg_pr_auc'):
+            y_pred = root_model.predict_proba(x)[:, 1]
+        else:
+            y_pred = root_model.predict(x)
+
+        metric = self._calculate_metric(y_true=y, y_pred=y_pred)
+
+        # create node object
+        tree = Node(depth=0, metric=metric, model=root_model)
+
+        return tree
+
+    def _split(self, node: Node, x: np.ndarray, y: np.ndarray, depth: int, rng: np.random.Generator):
+        if (depth == self.max_depth) or np.all(np.all(x == x[0, :], axis=1)) or (x.shape[0] < self.min_samples_split):
+            return node
+        else:
+            split = self._find_best_split(x=x, y=y, last_metric=node.metric, old_coefs=node.model.coef_, rng=rng)
+
+            if split.get('threshold') is not None:
+                node.threshold = split['threshold']
+                node.split_col_idx = split['column']
+
+                left_node = Node(depth=depth + 1, model=split['model_left'], metric=split['metric_left'])
+                left_node = self._split(node=left_node, x=split['x_left'], y=split['y_left'],
+                                        depth=depth + 1, rng=rng)
+
+                right_node = Node(depth=depth + 1, model=split['model_right'], metric=split['metric_right'])
+                right_node = self._split(node=right_node, x=split['x_right'], y=split['y_right'],
+                                         depth=depth + 1, rng=rng)
+
+                node.left_node = left_node
+                node.right_node = right_node
+                node.model = None
+
+            return node
+
+    def _find_best_split(self, x: np.ndarray, y: np.ndarray,
+                         last_metric: float, old_coefs: np.ndarray, rng: np.random.Generator):
+        split = {}
+
+        random_col_ids = rng.choice(np.arange(1, (x.shape[1])), int(round(np.sqrt(x.shape[1] - 1))), replace=False)
+
+        for col in random_col_ids:
+            split_candidates = self._split_values(x[:, col], rng=rng)
+
+            for thresh in split_candidates:
+                left_idx = x[:, col] <= thresh
+                left_idx, right_idx = left_idx.nonzero()[0], (~left_idx).nonzero()[0]
+
+                if x[:, col].max() == thresh:
+                    continue
+
+                x_left, y_left = x.take(left_idx, axis=0), y.take(left_idx, axis=0)
+                x_right, y_right = x.take(right_idx, axis=0), y.take(right_idx, axis=0)
+
+                if np.all(y_left == y_left[0]) or np.all(y_right == y_right[0]):
+                    continue
+
+                observations_left, observations_right = y_left.shape[0], y_right.shape[0]
+
+                if (
+                        observations_left < self.min_samples_leaf
+                ) or (
+                        observations_right < self.min_samples_leaf
+                ):
+                    continue
+
+                # initialize models
+                model_left, model_right = copy.deepcopy(self.linear_model), copy.deepcopy(self.linear_model)
+
+                # fit models
+                if self.warm_start and isinstance(model_left, (Regressor, Classifier)) and isinstance(
+                        model_right, (Regressor, Classifier)):
+                    model_left.fit(x_left, y_left, initial_coefs=old_coefs)
+                    model_right.fit(x_right, y_right, initial_coefs=old_coefs)
+                else:
+                    model_left.fit(x_left, y_left, None)
+                    model_right.fit(x_right, y_right, None)
+
+                # get prediction for these nodes
+                if self.criterion == 'cross_entropy':
+                    y_pred_left = model_left.predict_proba(x_left)
+                    y_pred_right = model_right.predict_proba(x_right)
+                elif (self.criterion == 'neg_roc_auc') or (self.criterion == 'neg_pr_auc'):
+                    y_pred_left = model_left.predict_proba(x_left)[:, 1]
+                    y_pred_right = model_right.predict_proba(x_right)[:, 1]
+                else:
+                    y_pred_left = model_left.predict(x_left)
+                    y_pred_right = model_right.predict(x_right)
+
+                # get metrics for these nodes
+                metric_left = self._calculate_metric(y_true=y_left, y_pred=y_pred_left)
+                metric_right = self._calculate_metric(y_true=y_right, y_pred=y_pred_right)
+
+                new_metric = ((metric_left * observations_left + metric_right * observations_right)
+                              / (observations_left + observations_right))
+                better_split = new_metric < (last_metric - self.min_abs_improvement)
+
+                if better_split:
+                    last_metric = new_metric
+
+                    split = {'column': col,
+                             'threshold': thresh,
+                             'model_left': model_left,
+                             'model_right': model_right,
+                             'x_right': x_right,
+                             'y_right': y_right,
+                             'x_left': x_left,
+                             'y_left': y_left,
+                             'metric_left': metric_left,
+                             'metric_right': metric_right}
+
+        return split
+
+    def _split_values(self, values: np.ndarray, rng: np.random.Generator) -> List:
+        unique_values = np.unique(values)
+        if unique_values.shape[0] <= self.n_splits:
+            split_values = unique_values.tolist()
+        else:
+            perc_splits = np.ceil(2 * self.n_splits / 3)
+            perc_splits = np.unique(np.percentile(values, np.arange(50 / perc_splits, 100, 100 / perc_splits),
+                                                  method='closest_observation'))
+
+            n_smart_splits = self.n_splits - perc_splits.shape[0]
+
+            diff = np.diff(unique_values, prepend=unique_values[0])
+            std = diff[1:].std()
+            length = unique_values.shape[0]
+
+            mask = np.array([False] * length)
+            k = np.arange(0, 15, 0.02)
+            for j in k:
+                mask += diff > (0.5 - j / 200) * length ** (1 / (2 + j)) * std
+                if np.count_nonzero(mask) >= n_smart_splits:
+                    break
+
+            mask = mask.nonzero()[0]
+            smart_splits = unique_values.take(mask, axis=0)
+            if smart_splits.shape[0] > n_smart_splits:
+                smart_splits = rng.choice(smart_splits, n_smart_splits, replace=False)
+
+            split_values = perc_splits.tolist() + smart_splits.tolist()
+
+        max_value = unique_values[-1]
+        split_values = [val for val in split_values if val != max_value]
+        return split_values
+
+    def predict(self, x: np.ndarray):
+        NotImplementedError()
+
+    def _calculate_metric(self, y_true: np.ndarray, y_pred: np.ndarray) -> float:
+        NotImplementedError()
+
+    def export_text(self, tree: int = None, column_names: List[str] = None, ndigits: int = 5):
+        txt = ''
+        if tree is None:
+            for i, node in enumerate(self.forest):
+                txt += 'Tree {}:\n'.format(i)
+                txt += self._node_to_text(node=node, column_names=column_names, ndigits=ndigits)
+                txt += '\n' + '\n'
+        else:
+            txt += 'Tree {}:\n'.format(tree)
+            node = self.forest[tree]
+            txt += self._node_to_text(node=node, column_names=column_names, ndigits=ndigits)
+
+        return txt
+
+    def _node_to_text(self, node: Node, column_names: List[str] = None, ndigits: int = 3):
+
+        txt = ''.join(['|   ']*node.depth)
+        txt += '|---'
+
+        if node.model is None:
+            if column_names is None:
+                col = 'col_{}'.format(node.split_col_idx - 1)
+            else:
+                col = column_names[node.split_col_idx - 1]
+
+            txt += ' '.join([col, '<', str(round(node.threshold, ndigits))])
+            txt += '\n'
+
+            txt += self._node_to_text(node=node.left_node, column_names=column_names, ndigits=ndigits)
+
+            txt += ''.join(['|   '] * node.depth)
+            txt += '|---'
+            txt += ' '.join([col, '>=', str(round(node.threshold, ndigits))])
+            txt += '\n'
+
+            txt += self._node_to_text(node=node.right_node, column_names=column_names, ndigits=ndigits)
+
+        else:
+            intercept = node.model.coef_[0]
+            weights = node.model.coef_[1:]
+            weights = ['+' + str(round(w, ndigits)) if w > 0 else str(round(w, ndigits)) for w in weights]
+            if column_names is None:
+                cols = ['col_{}'.format(i) for i in range(len(weights))]
+            else:
+                cols = column_names
+
+            weights_and_cols = ' '.join(['*'.join(p) for p in list(zip(weights, cols))])
+
+            txt += ' '.join(['model: y =', str(round(intercept, ndigits)), weights_and_cols])
+
+            txt += '\n'
+
+        return txt
+
+    def set_params(self, **parameters):
+        for parameter, value in parameters.items():
+            setattr(self, parameter, value)
+        return self
+
+    def _check_targets_classification(self, y: np.ndarray):
+        self.classes_ = np.unique(y)
+        assert issubclass(self.classes_.dtype.type, np.integer), 'Please convert targets to integer values'

lrf/_bfgs.py

@@ -0,0 +1,176 @@
+import numpy as np
+
+from lrf._criterion import cross_entropy
+
+
+class BFGS:
+    def __init__(self, n_iter: int = 100, tol: float = 10**(-4), intercept: bool = True):
+        self.n_iter = n_iter
+        self.tol = tol
+        self.intercept = intercept
+
+    def classification(self, x: np.ndarray, y_true: np.ndarray, coef_: np.ndarray, C: float = 1.0):
+        y_true = y_true[:, np.newaxis]
+
+        coef_ = coef_[:, np.newaxis]
+        new_grad = self._grad_cross_entropy_logistic(y_true=y_true, x=x, coef_=coef_, C=C,
+                                                     y_pred=self._sigmoid(x@coef_))
+
+        H_inv = np.eye(coef_.shape[0]) / 0.2
+
+        alpha = 1
+        for _ in range(self.n_iter):
+
+            grad = new_grad
+
+            direction = -H_inv @ grad
+
+            alpha, new_grad = self._line_search(x=x, y=y_true, coef_=coef_, direction=direction,
+                                                grad=grad, C=C, alpha=alpha)
+
+            if alpha is None:
+                break
+
+            s = alpha * direction
+
+            change_mask = coef_ != 0
+            change = np.abs(s[change_mask] / coef_[change_mask]) if np.count_nonzero(change_mask) > 0 else 1
+
+            coef_ += s
+
+            if (np.max(change) <= self.tol) or np.all(new_grad == 0):
+                break
+            else:
+                grad_diff = new_grad - grad
+
+                st_grad_diff = s.T @ grad_diff
+
+                A = ((st_grad_diff + grad_diff.T @ H_inv @ grad_diff) * (s @ s.T)) / (st_grad_diff**2)
+
+                B = (H_inv @ grad_diff @ s.T + s @ grad_diff.T @ H_inv) / st_grad_diff
+
+                H_inv += A - B
+
+        return coef_.flatten()
+
+    @staticmethod
+    def _sigmoid(y: np.ndarray):
+        """
+        Sigmoid function to map input to values between 0 and 1 on the characteristic s-shaped curve (sigmoid curve).
+        This is the probability for the positive class.
+
+        Args:
+            y: np.ndarray
+                Input values, which will be mapped to values between 0 and 1.
+
+        Returns:
+            np.ndarray: Returns the probability for the positive class.
+        """
+        return np.exp(-np.logaddexp(0, -y))
+
+    def _grad_cross_entropy_logistic(self, y_true: np.ndarray, x: np.ndarray, y_pred: np.ndarray,
+                                     coef_: np.ndarray, C: float):
+
+        weights = coef_.copy()
+        if self.intercept:
+            weights[0] = 0
+
+        norm = np.linalg.norm(weights)
+        if norm != 0.0:
+            penalty = np.einsum('ij->', weights) / (C * norm)
+        else:
+            penalty = 0
+
+        grad = x.T @ (y_pred - y_true) + penalty
+        norm = np.linalg.norm(grad)
+        if norm == 0:
+            grad = np.zeros(grad.shape)
+        else:
+            grad /= norm
+        return grad
+
+    def _armijo(self, y: np.ndarray, y_pred: np.ndarray, coef_: np.ndarray, alpha: float, C: float,
+                c1: float, grad_dir: float, cross_entropy_value: float):
+
+        penalty = self.get_penalty(coef=coef_, C=C)
+
+        left_armijo = cross_entropy(y_true=y, y_pred=y_pred, penalty=penalty)
+        right_armijo = cross_entropy_value + c1 * alpha * grad_dir
+
+        armijo = left_armijo <= right_armijo
+
+        return armijo
+
+    def _wolfe(self, x: np.ndarray, y: np.ndarray, coef_: np.ndarray, alpha: float, C: float,
+               direction: np.ndarray, c1: float, c2: float, grad_dir: float, cross_entropy_value: float,
+               x_coef: np.ndarray, x_direction: np.ndarray):
+
+        y_pred = self._sigmoid(x_coef + alpha*x_direction)
+
+        armijo = self._armijo(y=y, coef_=coef_, alpha=alpha, c1=c1, grad_dir=grad_dir,
+                              cross_entropy_value=cross_entropy_value, C=C, y_pred=y_pred)
+
+        if armijo:
+            grad = self._grad_cross_entropy_logistic(y_true=y, x=x, coef_=coef_ + alpha * direction, C=C, y_pred=y_pred)
+            left_curvature = (direction.T @ grad).item()
+            right_curvature = c2 * grad_dir
+
+            # since wolfe conditions are armijo and weak/strong curvature, the curvature directly implies weak or
+            # strong wolfe since armijo is given to be True at this point
+            weak_wolfe = left_curvature >= right_curvature
+            strong_wolfe = np.abs(left_curvature) <= np.abs(right_curvature)
+        else:
+            weak_wolfe, strong_wolfe = False, False
+            grad = None
+
+        return weak_wolfe, strong_wolfe, grad
+
+    def _line_search(self, x: np.ndarray, y: np.ndarray, coef_: np.ndarray,
+                     direction: np.ndarray, grad: np.ndarray, C: float,
+                     c1: float = 10 ** (-4), c2: float = 0.9,
+                     alpha_upper: float = 2.0, alpha_lower: float = 10**-10, alpha: float = 1.0,
+                     n_iter: int = 10):
+
+        grad_dir = (direction.T @ grad).item()
+        x_coef = x @ coef_
+        x_direction = x @ direction
+
+        penalty = self.get_penalty(coef=coef_, C=C)
+
+        cross_entropy_value = cross_entropy(y_true=y, y_pred=self._sigmoid(x_coef), penalty=penalty)
+
+        weak_wolfe_value, grad_value = 0, 0
+        for _ in range(n_iter):
+            weak_wolfe, strong_wolfe, grad = self._wolfe(x=x, y=y, coef_=coef_, alpha=alpha, direction=direction, c1=c1,
+                                                         c2=c2, grad_dir=grad_dir,
+                                                         cross_entropy_value=cross_entropy_value,
+                                                         C=C, x_coef=x_coef, x_direction=x_direction)
+
+            if strong_wolfe:
+                break
+            else:
+                if weak_wolfe and alpha > weak_wolfe_value:
+                    weak_wolfe_value = alpha
+                    grad_value = grad
+
+                    alpha_lower = alpha
+                else:
+                    alpha_upper = alpha
+
+                alpha = (alpha_lower + alpha_upper) / 2
+        else:
+            if weak_wolfe_value != 0:
+                alpha = weak_wolfe_value
+                grad = grad_value
+            else:
+                alpha, grad = None, None
+
+        return alpha, grad
+
+    def get_penalty(self, coef: np.ndarray, C: float):
+        if self.intercept:
+            penalty = np.linalg.norm(coef[1:]) / C
+        else:
+            penalty = np.linalg.norm(coef) / C
+
+        return penalty

lrf/_criterion.py

@@ -0,0 +1,147 @@
+import numpy as np
+
+
+def mse(y_true: np.ndarray, y_pred: np.ndarray):
+    return ((y_true - y_pred)**2).mean()
+
+
+def rmse(y_true: np.ndarray, y_pred: np.ndarray):
+    return np.sqrt(((y_true - y_pred)**2).mean())
+
+
+def mae(y_true: np.ndarray, y_pred: np.ndarray):
+    return (np.abs(y_true - y_pred)).mean()
+
+
+def mape(y_true: np.ndarray, y_pred: np.ndarray):
+    return np.abs((y_true - y_pred)/y_true).mean()
+
+
+def neg_explained_variance(y_true: np.ndarray, y_pred: np.ndarray):
+    return np.var(y_true - y_pred)/np.var(y_true) - 1
+
+
+def neg_r2(y_true: np.ndarray, y_pred: np.ndarray):
+    return -np.einsum('i->', (y_true - y_pred)**2)/np.einsum('i->', (y_true - y_true.mean())**2)
+
+
+def wape(y_true: np.ndarray, y_pred: np.ndarray):
+    return np.einsum('i->', np.abs(y_true - y_pred))/np.einsum('i->', np.abs(y_true))
+
+
+def _confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray, thresh: float = 0.5):
+    mask_ones = y_pred >= thresh
+
+    tmp = y_true[mask_ones] == 1
+    tp = np.count_nonzero(tmp)
+    fp = tmp.shape[0] - tp
+
+    tmp = y_true[~mask_ones] == 1
+    fn = np.count_nonzero(tmp)
+    tn = tmp.shape[0] - fn
+
+    return tn, fp, fn, tp
+
+
+def _thresholds(y_pred: np.ndarray):
+    if y_pred.shape[0] < 10_000:
+        thresholds = np.unique(y_pred).tolist()
+    else:
+        step = 1 / 6180
+        thresholds = np.arange(max(np.min(y_pred) - 2 * step, 0), min(np.max(y_pred) + 2 * step, 1), step).tolist()
+
+    if np.min(y_pred) > 0:
+        thresholds = [0] + thresholds
+
+    if np.max(y_pred) < 1:
+        thresholds = thresholds + [1]
+
+    return thresholds
+
+
+def hamming(y_true: np.ndarray, y_pred: np.ndarray):
+    return (y_true != y_pred).mean()
+
+
+def cross_entropy(y_true: np.ndarray, y_pred: np.ndarray, penalty: float = 0):
+    if y_true.ndim == 1:
+        y_true = y_true[:, np.newaxis]
+
+    y_pred = np.clip(y_pred, 1e-12, 1 - 1e-12)
+    return -(y_true*np.log(y_pred)+(1-y_true)*np.log(1-y_pred)).sum()/y_pred.shape[0] + penalty
+
+
+def neg_mcc(y_true: np.ndarray, y_pred: np.ndarray):
+    samples = y_true.shape[0]
+    tn, fp, fn, tp = _confusion_matrix(y_true=y_true, y_pred=y_pred, thresh=0.5)
+    cm = np.array([[tn, fp], [fn, tp]]).reshape(2, 2)
+
+    c = np.einsum('ii', cm)
+    t = np.einsum('ij->j', cm)
+    p = np.einsum('ij->i', cm)
+
+    dividend = (c*samples - t @ p)
+    divisor = (np.sqrt(samples**2 - p @ p) * np.sqrt(samples**2 - t @ t))
+
+    if divisor == 0:
+        mcc = 0.0
+    else:
+        mcc = dividend / divisor
+
+    return -mcc
+
+
+def neg_roc_auc(y_true: np.ndarray, y_pred: np.ndarray):
+    thresholds = _thresholds(y_pred=y_pred)
+
+    positives = y_true.sum()
+    negatives = y_true.shape[0] - positives
+
+    fpr, tpr = [], []
+
+    for thresh in thresholds:
+        _, fp, _, tp = _confusion_matrix(y_true=y_true, y_pred=y_pred, thresh=thresh)
+
+        tpr.append(tp / positives)
+        fpr.append(fp / negatives)
+
+    # integration is from right to left, therefore this is already the negative ROC AUC
+    neg_auc = np.trapz(tpr, fpr)
+
+    return neg_auc
+
+
+def neg_pr_auc(y_true: np.ndarray, y_pred: np.ndarray):
+    thresholds = _thresholds(y_pred=y_pred)
+
+    precision, recall = [], []
+
+    for thresh in thresholds:
+        _, fp, fn, tp = _confusion_matrix(y_true=y_true, y_pred=y_pred, thresh=thresh)
+
+        p = tp / (tp + fp) if (tp + fp) > 0 else 0
+        r = tp / (tp + fn) if (tp + fn) > 0 else 0
+
+        precision.append(p)
+        recall.append(r)
+
+    # integration is from right to left, therefore this is already the negative PR AUC
+    neg_auc = np.trapz(precision, recall)
+
+    return neg_auc
+
+
+def neg_auk(y_true: np.ndarray, y_pred: np.ndarray):
+    NotImplementedError()
+
+
+def neg_g_mean(y_true: np.ndarray, y_pred: np.ndarray):
+    tn, fp, fn, tp = _confusion_matrix(y_true=y_true, y_pred=y_pred)
+
+    return -(tp * tn / ((tp + fn) * (tn + fp)))
+
+
+def neg_f1(y_true: np.ndarray, y_pred: np.ndarray):
+    _, fp, fn, tp = _confusion_matrix(y_true=y_true, y_pred=y_pred)
+
+    return -(tp/(tp + 0.5*(fp + fn)))

lrf/_linear_models.py

@@ -0,0 +1,255 @@
+import numpy as np
+
+from lrf._bfgs import BFGS
+from lrf._preprocessor import Preprocessor
+
+
+class Regressor:
+    def __init__(self, alpha: float = 2.0, preprocessing: str = None, fit_intercept: bool = True,
+                 intercept_in_input: bool = False):
+        """
+        Linear least-squares with l2 regularization, also knows as Ridge Regression or Tikhonov regularization. This
+        implementation supports several preprocessing methods, namely centering, normalizing and standardizing the
+        data.
+
+        Args:
+            alpha : float, default=2.0
+                Regularization strength, larger values imply stronger regularization.
+                Must be a positive float. If alpha=0, there is no regularization and this implementation is equal to
+                a linear regression using least-squares.
+
+            preprocessing: str, default=None
+                Specifies the method for data preprocessing. Can be either 'center', 'normalize', 'standardize' or
+                None (default)
+
+            fit_intercept: bool, default=True
+                Whether to calculate the intercept.
+
+            intercept_in_input: bool, default=False
+                Whether there is an intercept column at index 0 in the data.
+        """
+
+        self.alpha = alpha
+        self.preprocessing = preprocessing
+        self.fit_intercept = fit_intercept
+        self.intercept_in_input = intercept_in_input
+
+        if self.preprocessing is not None:
+            self.preprocessor = Preprocessor(method=self.preprocessing)
+        
+        self.coef_ = None
+
+    def fit(self, x: np.ndarray, y: np.ndarray, initial_coefs: np.ndarray = None):
+        """
+        Fit linear regression model.
+
+        Args:
+            x: np.ndarray
+                Training data, containing the feature values.
+
+            y: np.ndarray
+                Target values.
+
+        Returns:
+            self:
+                Returns an instance of self.
+        """
+
+        assert self.alpha >= 0
+
+        x = self._preprocessing(x, fit=True, intercept=self.intercept_in_input)
+
+        if self.fit_intercept and not self.intercept_in_input:
+            # insert a 1 as the intercept
+            x = np.insert(x, 0, 1, axis=1)
+
+        # ridge regression, by l2 regularization
+        A = self.alpha * np.identity(x.shape[1])
+        # we do not want to regularize the intercept
+        A[0, 0] = 0
+
+        # self.coef_ = np.dot(np.dot(np.linalg.pinv(np.dot(x.T, x) + A), x.T), y)
+        self.coef_ = np.linalg.lstsq(np.dot(x.T, x) + A, np.dot(x.T, y), rcond=None)[0]
+
+        return self
+
+    def predict(self, x: np.ndarray):
+        """
+        Make a prediction using the linear regression model.
+
+        Args:
+            x: Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted values.
+        """
+
+        assert self.coef_ is not None, 'This linear model is not fitted yet. Call "fit" before using this model to ' \
+                                       'make predictions'
+
+        x = self._preprocessing(x, fit=False, intercept=self.intercept_in_input)
+
+        if self.fit_intercept and not self.intercept_in_input:
+            # insert a 1 as the intercept
+            x = np.insert(x, 0, 1, axis=1)
+
+        return np.dot(x, self.coef_)
+
+    def _preprocessing(self, x: np.ndarray, fit: bool, intercept: bool):
+        """
+       Fit the preprocessor or transform the data according to a given method.
+
+        Args:
+            x: np.ndarray
+                Samples to be processed or on which the processor should be fitted.
+
+            fit: bool
+                Whether to call the 'fit' method of the preprocessor or to call the 'transform' method.
+
+            intercept: bool
+                Whether there is an intercept column at index 0 of x.
+
+        Returns:
+            np.ndarray: Returns the data, which is transformed if fit=False.
+        """
+
+        if self.preprocessing is not None:
+            if fit:
+                self.preprocessor.fit(x)
+
+            x = self.preprocessor.transform(x, intercept)
+
+        return x
+
+
+class Classifier:
+    def __init__(self, n_iter: int = 100, tol: float = 10**(-6), C: float = 1,
+                 preprocessing: str = None, fit_intercept: bool = True, intercept_in_input: bool = False):
+        self.n_iter = n_iter
+        self.tol = tol
+        self.C = C
+        self.preprocessing = preprocessing
+        self.fit_intercept = fit_intercept
+        self.intercept_in_input = intercept_in_input
+
+        if self.preprocessing is not None:
+            self.preprocessor = Preprocessor(method=self.preprocessing)
+
+        self.coef_ = None
+
+    def fit(self, x: np.ndarray, y: np.ndarray, initial_coefs: np.ndarray = None):
+        """
+        Fit linear classification model.
+
+        Args:
+            x: np.ndarray
+                Training data, containing the feature values.
+
+            y: np.ndarray
+                Target classes.
+
+        Returns:
+            self:
+                Returns an instance of self.
+        """
+
+        if np.unique(y).shape[0] > 2:
+            raise ValueError('You can only use the internal linear classification model for binary classification.'
+                             'For multi-class classification provide a suitable model to the "linear_model" parameter.')
+
+        x = self._preprocessing(x, fit=True, intercept=self.intercept_in_input)
+
+        if self.fit_intercept and not self.intercept_in_input:
+            # insert a 1 as the intercept
+            x = np.insert(x, 0, 1, axis=1)
+
+        self._logistic_regression(x=x, y=y, initial_coefs=initial_coefs)
+
+        return self
+
+    def predict_proba(self, x: np.ndarray):
+        """
+        Make a prediction of the probability of each class using the linear classification model.
+
+        Args:
+            x: np.ndarray
+                Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted probability of each class.
+        """
+
+        x = self._preprocessing(x, fit=False, intercept=self.intercept_in_input)
+
+        if self.fit_intercept and not self.intercept_in_input:
+            # insert a 1 as the intercept
+            x = np.insert(x, 0, 1, axis=1)
+
+        one_proba = self._sigmoid(np.dot(x, self.coef_))
+        y_pred_proba = np.ones((x.shape[0], 2))
+
+        y_pred_proba[:, 0] -= one_proba
+        y_pred_proba[:, 1] = one_proba
+
+        return y_pred_proba
+
+    def predict(self, x: np.ndarray):
+        """
+        Predict the classes using the linear classification model.
+
+        Args:
+            x: np.ndarray
+                Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted classes.
+        """
+
+        return np.argmax(self.predict_proba(x), axis=1)
+
+    @staticmethod
+    def _sigmoid(y: np.ndarray):
+        """
+        Sigmoid function to map input to values between 0 and 1 on the characteristic s-shaped curve (sigmoid curve).
+        This is the probability for the positive class.
+
+        Args:
+            y: np.ndarray
+                Input values, which will be mapped to values between 0 and 1.
+
+        Returns:
+            np.ndarray: Returns the probability for the positive class.
+        """
+        return np.exp(-np.logaddexp(0, -y))
+
+    def _preprocessing(self, x: np.ndarray, fit: bool, intercept: bool):
+        """
+       Fit the preprocessor or transform the data according to a given method.
+
+       Args:
+           x: np.ndarray
+               Samples to be processed or on which the processor should be fitted.
+
+           fit: bool
+               Whether to call the 'fit' method of the preprocessor or to call the 'transform' method.
+
+       Returns:
+           np.ndarray: Returns the data, which is transformed if fit=False.
+       """
+
+        if self.preprocessing is not None:
+            if fit:
+                self.preprocessor.fit(x)
+
+            x = self.preprocessor.transform(x, intercept)
+
+        return x
+
+    def _logistic_regression(self, x: np.ndarray, y: np.ndarray, initial_coefs: np.ndarray):
+        intercept = self.fit_intercept or self.intercept_in_input
+
+        bfgs = BFGS(intercept=intercept)
+        if initial_coefs is None:
+            self.coef_ = bfgs.classification(x=x, y_true=y, coef_=np.zeros((x.shape[1], )), C=self.C)
+        else:
+            self.coef_ = bfgs.classification(x=x, y_true=y, coef_=initial_coefs, C=self.C)

lrf/_node.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from typing import Union
+
+from lrf._linear_models import Regressor, Classifier
+
+
+@dataclass
+class Node:
+    """
+    Consolidates all attributes needed at a node, regardless of whether the node is a leaf or not.
+    """
+    depth: int = None
+    split_col_idx: int = None
+    threshold: float = None
+    metric: float = None
+    left_node = None
+    right_node = None
+    model: Union[Regressor, Classifier] = None

lrf/_preprocessor.py

@@ -0,0 +1,152 @@
+import numpy as np
+
+
+class Preprocessor:
+    def __init__(self, method: str):
+        """
+        Preprocessor for data transformation by a given method.
+
+        Args:
+            method: str
+                Specifies the transformation method, which could be 'center', 'normalize' or 'standardize'.
+        """
+
+        assert method in ['center', 'normalize', 'standardize'],\
+            'The scaling method should be "center", "normalize" or "standardize".'
+
+        self.method = method
+
+        self.mean = None
+        self.min = None
+        self.max = None
+        self.std = None
+
+    def fit(self, x: np.ndarray):
+        """
+        Compute the values which are needed for the given preprocessing method.
+
+        Args:
+            x: np.ndarray
+                The data used to compute the values which are needed for the given preprocessing method.
+
+        Returns:
+            self: Returns an instance of self.
+        """
+
+        if self.method == 'center':
+            self.mean = x.mean(axis=0)
+        elif self.method == 'normalize':
+            self.min = x.min(axis=0)
+            self.max = x.max(axis=0)
+
+            # if the difference between min and max is 0, this would raise an error. In this case, do not preprocess
+            # the data.
+            diff = self.max - self.min
+            diff_zero = diff == 0
+            self.min[diff_zero] = 0
+            self.max[diff_zero] = 1
+        elif self.method == 'standardize':
+            self.mean = x.mean(axis=0)
+            self.std = x.std(axis=0)
+
+            # if the standard deviation is 0, this would raise an error. In this case, do not preprocess the data.
+            zero_std = self.std == 0
+            self.mean[zero_std] = 0
+            self.std[zero_std] = 1
+
+        return self
+
+    def transform(self, x: np.ndarray, intercept: bool):
+        """
+        Transform the data according to the given method.
+
+        Args:
+            x: np.ndarray
+                The data which will be transformed.
+
+            intercept: bool
+                Whether there is an intercept column at index 0 of x.
+
+        Returns:
+            np.ndarray: Returns the transformed data.
+        """
+        if self.method == 'center':
+            return self._center(x, intercept)
+        elif self.method == 'normalize':
+            return self._normalize(x, intercept)
+        elif self.method == 'standardize':
+            return self._standardize(x, intercept)
+
+    def _center(self, x: np.ndarray, intercept: bool):
+        """
+        Center the data.
+
+        Args:
+            x: np.ndarray
+                The data which will be centered.
+
+            intercept: bool
+                Whether there is an intercept column at index 0 of x.
+
+        Returns:
+            np.ndarray: Returns the centered data.
+        """
+        x -= self.mean
+
+        if intercept:
+            if x.ndim == 2:
+                x[:, 0] = 1
+            else:
+                x[0] = 1
+
+        return x
+
+    def _normalize(self, x: np.ndarray, intercept: bool):
+        """
+        Normalize the data.
+
+        Args:
+            x: np.ndarray
+                The data which will be normalized.
+
+            intercept: bool
+                Whether there is an intercept column at index 0 of x.
+
+        Returns:
+            np.ndarray: Returns the normalized data.
+        """
+
+        x = (x - self.min)/(self.max - self.min)
+
+        if intercept:
+            if x.ndim == 2:
+                x[:, 0] = 1
+            else:
+                x[0] = 1
+
+        return x
+
+    def _standardize(self, x: np.ndarray, intercept: bool):
+        """
+        Standardize the data.
+
+        Args:
+            x: np.ndarray
+                The data which will be standardized.
+
+            intercept: bool
+                Whether there is an intercept column at index 0 of x.
+
+        Returns:
+            np.ndarray: Returns the standardized data.
+        """
+
+        x = (x - self.mean) / self.std
+
+        if intercept:
+            if x.ndim == 2:
+                x[:, 0] = 1
+            else:
+                x[0] = 1
+
+        return x

lrf/lrf.py

@@ -0,0 +1,221 @@
+from typing import List, Any
+
+import numpy as np
+
+from lrf._base_lrf import _LinearRandomForest
+from lrf._linear_models import Regressor, Classifier
+from lrf._criterion import (mse, rmse, mae, mape, wape, neg_explained_variance, neg_r2,
+                            hamming, cross_entropy, neg_mcc, neg_roc_auc, neg_pr_auc)
+from lrf._node import Node
+
+
+class LRFRegressor(_LinearRandomForest):
+    def __init__(self, linear_model: Any = None, alpha: float = 2.0, preprocessing: str = None,
+                 n_estimators: int = 100, max_depth: int = 5, n_splits: int = 15,
+                 split_samples_to_features_ratio: float = 4.5, min_abs_improvement: float = 1 * 10 ** (-4),
+                 leaf_samples_to_features_ratio: float = 2.0, criterion: str = 'mse',
+                 n_jobs: int = -1, random_state: int = None, verbose: bool = False):
+
+        self.alpha = alpha
+        self.preprocessing = preprocessing
+        self._estimator_type = 'regressor'
+
+        if linear_model is None:
+            linear_model = Regressor(alpha=self.alpha, preprocessing=self.preprocessing, intercept_in_input=True)
+        else:
+            assert hasattr(linear_model, 'fit')
+            assert hasattr(linear_model, 'predict')
+
+        super().__init__(linear_model=linear_model,
+                         n_estimators=n_estimators, max_depth=max_depth, criterion=criterion,
+                         n_splits=n_splits, split_samples_to_features_ratio=split_samples_to_features_ratio,
+                         leaf_samples_to_features_ratio=leaf_samples_to_features_ratio,
+                         min_abs_improvement=min_abs_improvement,
+                         n_jobs=n_jobs, random_state=random_state, verbose=verbose)
+
+    def predict(self, x: np.ndarray):
+        # add intercept here and not inside linear model for performance reasons
+        x = np.insert(x, 0, 1, axis=1)
+
+        # add columns with row index for sorting after multiprocessing
+        x = np.insert(x, 0, np.arange(x.shape[0]), axis=1)
+
+        results = [self._predict_tree(node=node, x=x, results=[]) for node in self.forest]
+
+        results = [np.vstack(i) for i in results]
+        results = np.array([i[np.argsort(i[:, 0])][:, 1:] for i in results])
+        results = results.mean(axis=0).flatten()
+        return results
+
+    def _predict_tree(self, node: Node, x: np.ndarray, results: List):
+        if node.model is None:
+            if x.shape[0] > 0:
+                left_indices = x[:, node.split_col_idx + 1] < node.threshold
+                right_indices = ~left_indices
+
+                results = self._predict_tree(node.left_node, x[left_indices], results)
+                results = self._predict_tree(node.right_node, x[right_indices], results)
+        else:
+            if x.shape[0] > 0:
+                node_results = node.model.predict(x[:, 1:])
+                node_results = np.insert(node_results[:, np.newaxis], 0, x[:, 0], axis=1)
+
+                results.extend(node_results)
+
+        return results
+
+    def _calculate_metric(self, y_true: np.ndarray, y_pred: np.ndarray):
+        if self.criterion == 'mse':
+            val = mse(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'rmse':
+            val = rmse(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'mae':
+            val = mae(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'mape':
+            val = mape(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'wape':
+            val = wape(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'neg_explained_variance':
+            val = neg_explained_variance(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'neg_r2':
+            val = neg_r2(y_true=y_true, y_pred=y_pred)
+        else:
+            print(' Metric "{}" is not implemented, MSE is used instead.'.format(self.criterion))
+            val = mse(y_true=y_true, y_pred=y_pred)
+
+        return val
+
+    def get_params(self, deep: bool = True):
+        return {'linear_model': self.linear_model, 'alpha': self.alpha, 'preprocessing': self.preprocessing,
+                'n_estimators': self.n_estimators, 'max_depth': self.max_depth, 'n_splits': self.n_splits,
+                'split_samples_to_features_ratio': self.split_samples_to_features_ratio,
+                'leaf_samples_to_features_ratio': self.leaf_samples_to_features_ratio, 'criterion': self.criterion,
+                'n_jobs': self.n_jobs, 'random_state': self.random_state, 'verbose': self.verbose}
+
+    def score(self, X: np.ndarray, y: np.ndarray):
+        y_pred = self.predict(X)
+        return -neg_r2(y_true=y, y_pred=y_pred)
+
+
+class LRFClassifier(_LinearRandomForest):
+    def __init__(self, linear_model: Any = None, C: float = 1.0, n_estimators: int = 100, max_depth: int = 5,
+                 n_splits: int = 15,  split_samples_to_features_ratio: float = 4.5,
+                 leaf_samples_to_features_ratio: float = 2.0, criterion: str = 'neg_mcc',
+                 min_abs_improvement: float = 5*10**(-4), n_jobs: int = -1, random_state: int = None,
+                 verbose: bool = False, preprocessing: str = 'standardize', warm_start: bool = True):
+
+        self.C = C
+        self.preprocessing = preprocessing
+        self._estimator_type = 'classifier'
+
+        if linear_model is None:
+            linear_model = Classifier(C=self.C, preprocessing=self.preprocessing, intercept_in_input=True)
+        else:
+            assert hasattr(linear_model, 'fit')
+            assert hasattr(linear_model, 'predict')
+            assert hasattr(linear_model, 'predict_proba')
+
+        super().__init__(linear_model=linear_model,
+                         n_estimators=n_estimators, max_depth=max_depth, criterion=criterion,
+                         n_splits=n_splits, min_abs_improvement=min_abs_improvement,
+                         split_samples_to_features_ratio=split_samples_to_features_ratio, random_state=random_state,
+                         leaf_samples_to_features_ratio=leaf_samples_to_features_ratio, n_jobs=n_jobs, verbose=verbose,
+                         classification=True, warm_start=warm_start)
+
+    def predict(self, x: np.ndarray):
+        """
+        Predict the classes using the linear random forest.
+
+        Args:
+            x: Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted classes.
+        """
+
+        return np.argmax(self.predict_proba(x=x), axis=1)
+
+    def predict_proba(self, x: np.ndarray):
+        """
+        Make a prediction of the probability of each class using the linear random forest.
+
+        Args:
+            x: Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted probability of each class.
+        """
+
+        # add intercept here and not inside linear model for performance reasons
+        x = np.insert(x, 0, 1, axis=1)
+
+        # add columns with row index for sorting after multiprocessing
+        x = np.insert(x, 0, np.arange(x.shape[0]), axis=1)
+
+        results = [self._predict_proba_tree(node=node, x=x, results=[]) for node in self.forest]
+
+        c = results[0][0].shape[0]
+
+        results = [np.concatenate(i).reshape((-1, c)) for i in results]
+        results = np.array([i[np.argsort(i[:, 0])][:, 1:] for i in results])
+        results = results.mean(axis=0)
+        return results
+
+    def _predict_proba_tree(self, node: Node, x: np.ndarray, results: List):
+        """
+        Make a prediction of the probability of each class using one given tree of the linear random forest.
+
+        Args:
+            x: Samples of the features to derive the prediction from.
+
+        Returns:
+            np.ndarray: Returns the predicted probability of each class.
+        """
+
+        if node.model is None:
+            if x.shape[0] > 0:
+                left_indices = x[:, node.split_col_idx + 1] < node.threshold
+                right_indices = ~left_indices
+
+                results = self._predict_proba_tree(node.left_node, x[left_indices], results)
+                results = self._predict_proba_tree(node.right_node, x[right_indices], results)
+        else:
+            if x.shape[0] > 0:
+                node_results = node.model.predict_proba(x[:, 1:])
+                n, m = node_results.shape
+                res = np.empty((n, m + 1))
+                res[:, 0] = x[:, 0]
+                res[:, 1:] = node_results
+
+                results.extend(res)
+
+        return results
+
+    def _calculate_metric(self, y_true: np.ndarray, y_pred: np.ndarray):
+        if self.criterion == 'neg_mcc':
+            val = neg_mcc(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'neg_pr_auc':
+            val = neg_pr_auc(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'hamming':
+            val = hamming(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'cross_entropy':
+            val = cross_entropy(y_true=y_true, y_pred=y_pred)
+        elif self.criterion == 'neg_roc_auc':
+            val = neg_roc_auc(y_true=y_true, y_pred=y_pred)
+        else:
+            print(' Metric "{}" is not implemented, the negative Matthews Correlation Coefficient is used '
+                  'instead.'.format(self.criterion))
+            val = neg_mcc(y_true=y_true, y_pred=y_pred)
+
+        return val
+
+    def get_params(self, deep=True):
+        return {'linear_model': self.linear_model, 'C': self.C, 'preprocessing': self.preprocessing,
+                'n_estimators': self.n_estimators, 'max_depth': self.max_depth, 'n_splits': self.n_splits,
+                'split_samples_to_features_ratio': self.split_samples_to_features_ratio,
+                'leaf_samples_to_features_ratio': self.leaf_samples_to_features_ratio, 'criterion': self.criterion,
+                'n_jobs': self.n_jobs, 'random_state': self.random_state, 'verbose': self.verbose}
+
+    def score(self, X: np.ndarray, y: np.ndarray):
+        y_pred = self.predict(X)
+        return np.count_nonzero(y == y_pred)/y.shape[0]