PyDLLib 1.0__py3-none-any.whl

DLL/MachineLearning/SupervisedLearning/Trees/_RegressionTree.py

@@ -0,0 +1,148 @@
+import torch
+
+from ._DecisionTree import Node
+from ....Exceptions import NotFittedError
+
+
+class RegressionTree:
+    """
+    RegressionTree implements a regression algorithm splitting the data along features minimizing the variance.
+
+    Args:
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 25. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+        ccp_alpha (non-negative float, optional): Determines how easily subtrees are pruned in cost-complexity pruning. The larger the value, more subtrees are pruned. Defaults to 0.0.
+    """
+    def __init__(self, max_depth=25, min_samples_split=2, ccp_alpha=0.0):
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(ccp_alpha, int | float) or ccp_alpha < 0:
+            raise ValueError("ccp_alpha must be non-negative.")
+        
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.ccp_alpha = ccp_alpha
+        self.root = None
+    
+    def fit(self, X, y):
+        """
+        Fits the RegressionTree model to the input data by generating a tree, which splits the data appropriately.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The target values corresponding to each sample.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix or the target matrix is not a PyTorch tensor.
+            ValueError: If the input matrix or the target matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The target must be 1 dimensional with the same number of samples as the input data")
+        self.n_features = X.shape[1]
+        self.root = self._grow_tree(X, y, 0)
+        if self.ccp_alpha > 0:
+            self.root, _, _ = self._prune(self.root, X, y)
+
+    def _grow_tree(self, x, y, depth):
+        n_samples, n_features = x.size()
+        classes = torch.unique(y)
+
+        if depth >= self.max_depth or len(classes) == 1 or n_samples < self.min_samples_split:
+            return Node(value=torch.mean(y))
+
+        feature_indicies = torch.randperm(n_features)[:self.n_features]
+        split_threshold, split_index = self._best_split(x, y, feature_indicies)
+        # if no split gains more information
+        if split_threshold is None:
+            return Node(value=torch.mean(y))
+        
+        left_indicies, right_indicies = self._split(x[:, split_index], split_threshold)
+        left = self._grow_tree(x[left_indicies], y[left_indicies], depth + 1)
+        right = self._grow_tree(x[right_indicies], y[right_indicies], depth + 1)
+        return Node(left, right, split_threshold, split_index)
+    
+    def _best_split(self, x, y, feature_indicies):
+        max_variance_reduction = -1
+        split_index = None
+        split_threshold = None
+
+        for index in feature_indicies:
+            feature_values = x[:, index]
+            possible_thresholds = torch.unique(feature_values)
+            for threshold in possible_thresholds:
+                variance_reduction = self._variance_reduction(y, feature_values, threshold)
+                if variance_reduction > max_variance_reduction:
+                    max_variance_reduction = variance_reduction
+                    split_index = index
+                    split_threshold = threshold
+        if max_variance_reduction <= 0: return None, None
+        return split_threshold, split_index
+
+    def _split(self, feature_values, threshold):
+        left_indicies = torch.argwhere(feature_values <= threshold).flatten()
+        right_indicies = torch.argwhere(feature_values > threshold).flatten()
+        return left_indicies, right_indicies
+
+    def _variance_reduction(self, y, feature_values, threshold):
+        left_indicies, right_indicies = self._split(feature_values, threshold)
+        if len(left_indicies) <= 1 or len(right_indicies) <= 1:
+            return 0
+        n = len(y)
+        return torch.var(y) - len(left_indicies) / n * torch.var(y[left_indicies]) - len(right_indicies) / n * torch.var(y[right_indicies])
+
+    def predict(self, X):
+        """
+        Applies the fitted RegressionTree model to the input data, predicting the correct values.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be regressed.
+        Returns:
+            target values (torch.Tensor of shape (n_samples,)): The predicted values corresponding to each sample.
+        Raises:
+            NotFittedError: If the RegressionTree model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if self.root is None:
+            raise NotFittedError("RegressionTree.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        return torch.tensor([self._predict_single(datapoint, self.root) for datapoint in X])
+
+    def _predict_single(self, x, current_node):
+        if current_node.is_leaf():
+            return current_node.value
+        
+        if x[current_node.feature_index] <= current_node.threshold:
+            return self._predict_single(x, current_node.left)
+        
+        return self._predict_single(x, current_node.right)
+    
+    def _prune(self, node, X, y):
+        if node.is_leaf():
+            return node, torch.sum((y - node.value) ** 2).item(), 1
+        
+        left_indicies, right_indicies = self._split(X[:, node.feature_index], node.threshold)
+        node.left, left_cost, left_leaf_nodes = self._prune(node.left, X[left_indicies], y[left_indicies])
+        node.right, right_cost, right_leaf_nodes = self._prune(node.right, X[right_indicies], y[right_indicies])
+        subtree_cost = left_cost + right_cost
+        total_subleaves = left_leaf_nodes + right_leaf_nodes
+
+        leaf_value = torch.mean(y)
+        new_leaf_nodes = 1
+        cost_of_node_replacement = torch.sum((y - leaf_value) ** 2).item()
+
+        subtree_cost_with_n_nodes_penalty = subtree_cost + self.ccp_alpha * total_subleaves
+        
+        if cost_of_node_replacement + self.ccp_alpha * new_leaf_nodes < subtree_cost_with_n_nodes_penalty:
+            return Node(value=leaf_value), cost_of_node_replacement, 1
+        return node, subtree_cost, total_subleaves

DLL/MachineLearning/SupervisedLearning/Trees/_XGBoostClassifier.py

@@ -0,0 +1,239 @@
+import torch
+
+from ._XGBoostTree import _XGBoostTree
+from ....DeepLearning.Layers.Activations import Sigmoid, SoftMax
+from ....DeepLearning.Losses import BCE, CCE, Exponential
+from ....Data.Preprocessing import OneHotEncoder
+from ....Exceptions import NotFittedError
+from ....Data.Metrics import calculate_metrics, prob_to_pred
+
+
+class XGBoostingClassifier:
+    """
+    XGBoostingClassifier implements a classification algorithm fitting many consecutive trees to gradients and hessians of the predictions.
+
+    Args:
+        n_trees (int, optional): The number of trees used for predicting. Defaults to 10. Must be a positive integer.
+        learning_rate (float, optional): The number multiplied to each additional trees residuals. Must be a real number in range (0, 1). Defaults to 0.5.
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 25. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+        reg_lambda (float | int, optional): The regularisation parameter used in fitting the trees. The larger the parameter, the smaller the trees. Must be a positive real number. Defaults to 1.
+        gamma (float | int, optional): The minimum gain to make a split. Must be a non-negative real number. Defaults to 0.
+        loss (string, optional): The loss function used in calculations of the residuals. Must be one of "log_loss" or "exponential". Defaults to "log_loss". "exponential" can only be used for binary classification.
+    Attributes:
+        n_features (int): The number of features. Available after fitting.
+        n_classes (int): The number of classes. 2 for binary classification. Available after fitting.
+    """
+    def __init__(self, n_trees=10, learning_rate=0.5, max_depth=25, min_samples_split=2, reg_lambda=1, gamma=0, loss="log_loss"):
+        if not isinstance(n_trees, int) or n_trees < 1:
+            raise ValueError("n_trees must be a positive integer.")
+        if not isinstance(learning_rate, float) or learning_rate <= 0 or learning_rate >= 1:
+            raise ValueError("learning_rate must be a float in range (0, 1).")
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(reg_lambda, int | float) or reg_lambda <= 0:
+            raise ValueError("reg_lambda must be a positive real number.")
+        if not isinstance(gamma, int | float) or gamma < 0:
+            raise ValueError("gamma must be a non-negative real number.")
+        if loss not in ["log_loss", "exponential"]:
+            raise ValueError('loss must be one of ["log_loss", "exponential"]')
+
+        self.n_trees = n_trees
+        self.learning_rate = learning_rate
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.reg_lambda = reg_lambda
+        self.gamma = gamma
+        self.trees = None
+        self.loss_ = loss
+    
+    def _get_activation_and_loss(self, classes):
+        self.n_classes = len(classes)
+        if self.loss_ == "log_loss":
+            if self.n_classes == 2:
+                self.loss = BCE(reduction="sum")
+                self.activation = Sigmoid()
+            else:
+                self.loss = CCE(reduction="sum")
+                self.activation = SoftMax()
+        elif self.loss_ == "exponential":
+            if self.n_classes != 2:
+                raise ValueError("The exponential loss is only applicable in binary classification. Use log_loss for multiclass classification instead.")
+            self.loss = Exponential(reduction="sum")
+            self.activation = Sigmoid()
+
+    def fit(self, X, y, metrics=["loss"]):
+        """
+        Fits the XGBoostingClassifier model to the input data by fitting trees to the errors made by previous trees.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The labels corresponding to each sample. Every element must be in [0, ..., n_classes - 1].
+            metrics (dict[str, torch.Tensor]): Contains the metrics that will be calculated between fitting each tree and returned. Only available for binary classification.
+        Returns:
+            metrics if binary classification else None
+        Raises:
+            TypeError: If the input matrix or the label vector is not a PyTorch tensor or if the problem is binary and metrics is not a list or a tuple.
+            ValueError: If the input matrix or the label vector is not the correct shape or the label vector contains wrong values.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the label matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The labels must be 1 dimensional with the same number of samples as the input data")
+        vals = torch.unique(y).numpy()
+        if set(vals) != {*range(len(vals))}:
+            raise ValueError("y must only contain the values in [0, ..., n_classes - 1].")
+        
+        self._get_activation_and_loss(torch.unique(y))
+        y = y.to(X.dtype)
+        self.n_features = X.shape[1]
+
+        if self.n_classes == 2:
+            if not isinstance(metrics, list | tuple):
+                raise ValueError("metrics must be a list or tuple containing the shorthand names of each wanted metric.")
+
+            return self._binary_fit(X, y, metrics=metrics)
+        else:
+            self._multi_fit(X, y)
+    
+    def _binary_hessian_diag(self, prob, true_output):
+        loss_gradient = self.loss.gradient(prob, true_output)
+        loss_hessian = self.loss.hessian(prob, true_output)
+        sigmoid_value = prob
+        sigmoid_gradient = self.activation.backward(loss_gradient)
+        
+        first_term = sigmoid_gradient * (1 - sigmoid_value) * loss_gradient
+        second_term = -sigmoid_value * sigmoid_gradient * loss_gradient
+        third_term = sigmoid_value * (1 - sigmoid_value) * loss_hessian  # is the same as self.activation.backward(loss_hessian)
+        return first_term + second_term + third_term
+    
+    def _multi_hessian_diag(self, prob, true_output):
+        loss_gradient = self.loss.gradient(prob, true_output)
+        loss_hessian = self.loss.hessian(prob, true_output)
+        softmax_value = prob
+        softmax_gradient = self.activation.backward(loss_gradient)
+
+        softmax_hessian = softmax_value * (1 - softmax_value) * (1 - 2 * softmax_value)
+        
+        # From Mathematica: D[f[g[x]], {x, 2}] = Derivative[1][g][x]^2 (f^\[Prime]\[Prime])[g[x]] + Derivative[1][f][g[x]] (g^\[Prime]\[Prime])[x]
+        return softmax_gradient ** 2 * loss_hessian + loss_gradient * softmax_hessian
+
+    def _binary_fit(self, X, y, metrics=["loss"]):
+        positive_ratio = y.mean()
+        self.initial_log_odds = torch.log(positive_ratio / (1 - positive_ratio))
+        pred = torch.full(y.shape, self.initial_log_odds)
+        trees = []
+        history = {metric: torch.zeros(self.n_trees) for metric in metrics}
+
+        for i in range(self.n_trees):
+            prob = self.activation.forward(pred)
+            gradient = self.activation.backward(self.loss.gradient(prob, y))
+            hessian = self._binary_hessian_diag(prob, y)
+
+            tree = _XGBoostTree(max_depth=self.max_depth, min_samples_split=self.min_samples_split, reg_lambda=self.reg_lambda, gamma=self.gamma)
+            tree.fit(X, gradient, hessian)
+            prediction = tree.predict(X)
+            pred += self.learning_rate * prediction
+
+            trees.append(tree)
+
+            values = calculate_metrics(data=(self.activation.forward(pred), y), metrics=metrics, loss=self.loss.loss)
+            for metric, value in values.items():
+                history[metric][i] = value
+        
+        self.trees = trees
+        return history
+    
+    def _multi_fit(self, X, y):
+        encoder = OneHotEncoder()
+        y = encoder.fit_encode(y)
+        
+        self.initial_log_odds = 0.0
+        pred = torch.full(y.shape, self.initial_log_odds)
+        trees = []
+
+        for class_index in range(self.n_classes):
+            class_trees = []
+            for _ in range(self.n_trees):
+                prob = self.activation.forward(pred)
+                gradient = self.activation.backward(self.loss.gradient(prob, y))[:, class_index]
+                hessian = self._multi_hessian_diag(prob, y)[:, class_index]
+
+                tree = _XGBoostTree(max_depth=self.max_depth, min_samples_split=self.min_samples_split, reg_lambda=self.reg_lambda, gamma=self.gamma)
+                tree.fit(X, gradient, hessian)
+                prediction = tree.predict(X)
+                pred[:, class_index] += self.learning_rate * prediction
+
+                class_trees.append(tree)
+            trees.append(class_trees)
+        
+        self.trees = trees
+    
+    def predict_proba(self, X):
+        """
+        Applies the fitted XGBoostingClassifier model to the input data, predicting the probabilities of each class.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            probabilities (torch.Tensor of shape (n_samples, n_classes) or for binary classification (n_samples,)): The predicted probabilities corresponding to each sample.
+        Raises:
+            NotFittedError: If the XGBoostingClassifier model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "initial_log_odds"):
+            raise NotFittedError("XGBoostingClassifier.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        if self.n_classes > 2:
+            return self._multi_predict_proba(X)
+        
+        pred = torch.full((X.shape[0],), self.initial_log_odds)
+        
+        for tree in self.trees:
+            prediction = tree.predict(X)
+            pred += self.learning_rate * prediction
+
+        return self.activation.forward(pred)
+
+    def predict(self, X):
+        """
+        Applies the fitted XGBoostingClassifier model to the input data, predicting the correct classes.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The predicted labels corresponding to each sample.
+        Raises:
+            NotFittedError: If the XGBoostingClassifier model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "initial_log_odds"):
+            raise NotFittedError("XGBoostingClassifier.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        
+        prob = self.predict_proba(X)
+        return prob_to_pred(prob)
+    
+    def _multi_predict_proba(self, X):
+        pred = torch.full((X.shape[0], self.n_classes), self.initial_log_odds)
+
+        for i in range(self.n_classes):
+            class_trees = self.trees[i]
+            for tree in class_trees:
+                pred[:, i] += self.learning_rate * tree.predict(X)
+
+        return self.activation.forward(pred)
+

DLL/MachineLearning/SupervisedLearning/Trees/_XGBoostRegressor.py

@@ -0,0 +1,131 @@
+import torch
+
+from ._XGBoostTree import _XGBoostTree
+from ....DeepLearning.Losses import MSE, MAE, Huber
+from ....Exceptions import NotFittedError
+from ....Data.Metrics import calculate_metrics
+
+
+class XGBoostingRegressor:
+    """
+    XGBoostingRegressor implements a regression algorithm fitting many consecutive trees to gradients and hessians of the loss function.
+
+    Args:
+        n_trees (int, optional): The number of trees used for predicting. Defaults to 50. Must be a positive integer.
+        learning_rate (float, optional): The number multiplied to each additional trees residuals. Must be a real number in range (0, 1). Defaults to 0.5.
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 3. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+        reg_lambda (float | int, optional): The regularisation parameter used in fitting the trees. The larger the parameter, the smaller the trees. Must be a positive real number. Defaults to 1.
+        gamma (float | int, optional): The minimum gain to make a split. Must be a non-negative real number. Defaults to 0.
+        loss (string, optional): The loss function used in calculations of the gradients and hessians. Must be one of "squared", "absolute" or "huber". Defaults to "squared".
+        huber_delta (float | int, optional): The delta parameter for the possibly used huber loss. If loss is not "huber", this parameter is ignored.
+    Attributes:
+        n_features (int): The number of features. Available after fitting.
+    """
+    def __init__(self, n_trees=50, learning_rate=0.5, max_depth=3, min_samples_split=2, reg_lambda=1, gamma=0, loss="squared", huber_delta=1):
+        if not isinstance(n_trees, int) or n_trees < 1:
+            raise ValueError("n_trees must be a positive integer.")
+        if not isinstance(learning_rate, float) or learning_rate <= 0 or learning_rate >= 1:
+            raise ValueError("learning_rate must be a float in range (0, 1).")
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(reg_lambda, int | float) or reg_lambda <= 0:
+            raise ValueError("reg_lambda must be a positive real number.")
+        if not isinstance(gamma, int | float) or gamma < 0:
+            raise ValueError("gamma must be a non-negative real number.")
+        if loss not in ["squared", "absolute", "huber"]:
+            raise ValueError('loss must be one in ["squared", "absolute", "huber"].')
+        if not isinstance(huber_delta, int | float) or huber_delta <= 0:
+            raise ValueError("huber_delta must be a positive real number.")
+
+        self.n_trees = n_trees
+        self.learning_rate = learning_rate
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.reg_lambda = reg_lambda
+        self.gamma = gamma
+        self.trees = None
+        self.loss = self._choose_loss(loss, huber_delta)
+
+    def _choose_loss(self, loss, huber_delta):
+        if loss == "squared":
+            return MSE(reduction="sum")
+        elif loss == "absolute":
+            return MAE(reduction="sum")
+        elif loss == "huber":
+            return Huber(reduction="sum", delta=huber_delta)
+
+    def fit(self, X, y, metrics=["loss"]):
+        """
+        Fits the XGBoostingRegressor model to the input data by fitting trees to the errors made by previous trees.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The target values corresponding to each sample.
+            metrics (dict[str, torch.Tensor]): Contains the metrics that will be calculated between fitting each tree and returned.
+        Returns:
+            metrics (dict[str, torch.Tensor]): The calculated metrics.
+        Raises:
+            TypeError: If the input matrix or the target vector is not a PyTorch tensor.
+            ValueError: If the input matrix or the target vector is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The target must be 1 dimensional with the same number of samples as the input data")
+        if not isinstance(metrics, list | tuple):
+            raise ValueError("metrics must be a list or tuple containing the shorthand names of each wanted metric.")
+        
+        self.n_features = X.shape[1]
+        self.initial_pred = y.mean()
+        pred = torch.full(y.shape, self.initial_pred)
+        trees = []
+        history = {metric: torch.zeros(self.n_trees) for metric in metrics}
+
+        for i in range(self.n_trees):
+            gradient = self.loss.gradient(pred, y)
+            hessian = self.loss.hessian(pred, y)
+
+            tree = _XGBoostTree(max_depth=self.max_depth, min_samples_split=self.min_samples_split, reg_lambda=self.reg_lambda, gamma=self.gamma)
+            tree.fit(X, gradient, hessian)
+            prediction = tree.predict(X)
+
+            pred += self.learning_rate * prediction
+            trees.append(tree)
+
+            values = calculate_metrics(data=(pred, y), metrics=metrics, loss=self.loss.loss)
+            for metric, value in values.items():
+                history[metric][i] = value
+
+        self.trees = trees
+        return history
+
+    def predict(self, X):
+        """
+        Applies the fitted XGBoostingRegressor model to the input data, predicting the target values.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            targets (torch.Tensor of shape (n_samples,)): The predicted target values corresponding to each sample.
+        Raises:
+            NotFittedError: If the XGBoostingRegressor model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "initial_pred"):
+            raise NotFittedError("XGBoostingRegressor.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        pred = torch.full((X.shape[0],), self.initial_pred)
+        
+        for tree in self.trees:
+            pred += self.learning_rate * tree.predict(X)
+        return pred

DLL/MachineLearning/SupervisedLearning/Trees/_XGBoostTree.py

@@ -0,0 +1,105 @@
+import torch
+
+from ._DecisionTree import Node
+from ....Exceptions import NotFittedError
+
+
+class _XGBoostTree:
+    def __init__(self, max_depth=25, min_samples_split=2, reg_lambda=1, gamma=1):
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(reg_lambda, int | float) or reg_lambda <= 0:
+            raise ValueError("reg_lambda must be a positive real number.")
+        if not isinstance(gamma, int | float) or gamma < 0:
+            raise ValueError("gamma must be a non-negative real number.")
+        
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.reg_lambda  = reg_lambda
+        self.gamma = gamma
+        self.root = None
+    
+    def _value(self, gradient, hessian):
+        return -gradient.sum() / (hessian.sum() + self.reg_lambda)
+    
+    def fit(self, X, gradient, hessian):
+        if not isinstance(X, torch.Tensor) or not isinstance(gradient, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if gradient.ndim != 1 or gradient.shape[0] != X.shape[0]:
+            raise ValueError("The gradient must be 1 dimensional with the same number of samples as the input data")
+        if hessian.ndim != 1 or hessian.shape[0] != X.shape[0]:
+            raise ValueError("The hessian must be 1 dimensional with the same number of samples as the input data")
+        
+        self.n_features = X.shape[1]
+        self.root = self._grow_tree(X, gradient, hessian, 0)
+
+    def _grow_tree(self, X, gradient, hessian, depth):
+        n_samples, n_features = X.size()
+        vals = torch.unique(gradient)
+
+        if depth >= self.max_depth or len(vals) == 1 or n_samples < self.min_samples_split:
+            return Node(value=self._value(gradient, hessian))
+
+        feature_indicies = torch.randperm(n_features)[:self.n_features]
+        split_threshold, split_index = self._best_split(X, gradient, hessian, feature_indicies)
+        # if no split gains more information
+        if split_threshold is None:
+            return Node(value=self._value(gradient, hessian))
+        
+        left_indicies, right_indicies = self._split(X[:, split_index], split_threshold)
+        left = self._grow_tree(X[left_indicies], gradient[left_indicies], hessian[left_indicies], depth + 1)
+        right = self._grow_tree(X[right_indicies], gradient[right_indicies], hessian[right_indicies], depth + 1)
+        return Node(left, right, split_threshold, split_index)
+    
+    def _best_split(self, X, gradient, hessian, feature_indicies):
+        max_gain = -1
+        split_index = None
+        split_threshold = None
+
+        for index in feature_indicies:
+            feature_values = X[:, index]
+            possible_thresholds = torch.unique(feature_values)
+            for threshold in possible_thresholds:
+                gain = self._gain(gradient, hessian, feature_values, threshold)
+                if gain > max_gain:
+                    max_gain = gain
+                    split_index = index
+                    split_threshold = threshold
+        if max_gain <= 0: return None, None
+        return split_threshold, split_index
+
+    def _split(self, feature_values, threshold):
+        left_indicies = torch.argwhere(feature_values <= threshold).flatten()
+        right_indicies = torch.argwhere(feature_values > threshold).flatten()
+        return left_indicies, right_indicies
+
+    def _gain(self, gradient, hessian, feature_values, threshold):
+        left_indicies, right_indicies = self._split(feature_values, threshold)
+        if len(left_indicies) <= 1 or len(right_indicies) <= 1:
+            return 0
+        return self._gain_node(gradient, hessian) - self._gain_node(gradient[left_indicies], hessian[left_indicies]) - self._gain_node(gradient[right_indicies], hessian[right_indicies]) - self.gamma
+
+    def _gain_node(self, gradient, hessian):
+        return -(gradient.sum() ** 2) / (hessian.sum() + self.reg_lambda)
+
+    def predict(self, X):
+        if self.root is None:
+            raise NotFittedError("_XGBoostTree.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        return torch.tensor([self._predict_single(datapoint, self.root) for datapoint in X])
+
+    def _predict_single(self, x, current_node):
+        if current_node.is_leaf():
+            return current_node.value
+        
+        if x[current_node.feature_index] <= current_node.threshold:
+            return self._predict_single(x, current_node.left)
+        
+        return self._predict_single(x, current_node.right)

DLL/MachineLearning/SupervisedLearning/Trees/__init__.py

@@ -0,0 +1,12 @@
+from ._DecisionTree import DecisionTree
+from ._RandomForest import RandomForestClassifier
+from ._RegressionForest import RandomForestRegressor
+from ._RegressionTree import RegressionTree
+from ._GradientBoostingRegressor import GradientBoostingRegressor
+from ._GradientBoostingClassifier import GradientBoostingClassifier
+from ._AdaBoostClassifier import AdaBoostClassifier
+from ._AdaBoostRegressor import AdaBoostRegressor
+from ._XGBoostRegressor import XGBoostingRegressor
+from ._XGBoostClassifier import XGBoostingClassifier
+from ._LGBMRegressor import LGBMRegressor
+from ._LGBMClassifier import LGBMClassifier