PyDLLib 1.0__py3-none-any.whl

DLL/MachineLearning/SupervisedLearning/Trees/_LGBMRegressor.py

@@ -0,0 +1,168 @@
+import torch
+
+from ._LGBMTree import _LGBMTree, _ExclusiveFeatureBundling
+from ....DeepLearning.Losses import MSE, MAE, Huber
+from ....Exceptions import NotFittedError
+from ....Data.Metrics import calculate_metrics
+
+
+class LGBMRegressor:
+    """
+    LGBMRegressor implements a regression algorithm fitting many consecutive trees to gradients and hessians of the loss function. The algorithm is based on `this paper <https://proceedings.neurips.cc/paper_files/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf>`_.
+
+    Args:
+        n_trees (int, optional): The number of trees used for predicting. Defaults to 50. Must be a positive integer.
+        learning_rate (float, optional): The number multiplied to each additional trees residuals. Must be a real number in range (0, 1). Defaults to 0.5.
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 3. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+        n_bins (int, optional): The number of bins used to find the optimal split of data. Must be greater than 1. Defaults to 30.
+        reg_lambda (float | int, optional): The regularisation parameter used in fitting the trees. The larger the parameter, the smaller the trees. Must be a positive real number. Defaults to 1.
+        gamma (float | int, optional): The minimum gain to make a split. Must be a non-negative real number. Defaults to 0.
+        large_error_proportion (float, optional): The proportion of the whole data with the largest error, which is always used to train the next weak learner. Defaults to 0.3.
+        small_error_proportion (float, optional): The proportion of data randomly selected from the remaining (1 - large_error_proportion) percent of data to train the next weak learner. Defaults to 0.2.
+        loss (string, optional): The loss function used in calculations of the gradients and hessians. Must be one of "squared", "absolute" or "huber". Defaults to "squared".
+        huber_delta (float | int, optional): The delta parameter for the possibly used huber loss. If loss is not "huber", this parameter is ignored.
+        max_conflict_rate (float, optional): The proportion of samples, which are allowed to be nonzero without featuers being bundled. Is ignored if use_efb=False. Defaults to 0.0.
+        use_efb (bool, optional): Determines if the exclusive feature bundling algorithm is used. Defaults to True.
+    Attributes:
+        n_features (int): The number of features. Available after fitting.
+    """
+    def __init__(self, n_trees=50, learning_rate=0.5, max_depth=3, min_samples_split=2, n_bins=30, reg_lambda=1, gamma=0, large_error_proportion=0.3, small_error_proportion=0.2, loss="squared", huber_delta=1, max_conflict_rate=0.0, use_efb=True):
+        if not isinstance(n_trees, int) or n_trees < 1:
+            raise ValueError("n_trees must be a positive integer.")
+        if not isinstance(learning_rate, float) or learning_rate <= 0 or learning_rate >= 1:
+            raise ValueError("learning_rate must be a float in range (0, 1).")
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(n_bins, int) or n_bins <= 1:
+            raise ValueError("n_bins must be an integer greater than 1.")
+        if not isinstance(reg_lambda, int | float) or reg_lambda <= 0:
+            raise ValueError("reg_lambda must be a positive real number.")
+        if not isinstance(gamma, int | float) or gamma < 0:
+            raise ValueError("gamma must be a non-negative real number.")
+        if not isinstance(large_error_proportion, float) or large_error_proportion <= 0 or large_error_proportion >= 1:
+            raise ValueError("large_error_proportion must be a float in range (0, 1).")
+        if not isinstance(small_error_proportion, float) or small_error_proportion <= 0 or small_error_proportion >= 1:
+            raise ValueError("small_error_proportion must be a float in range (0, 1).")
+        if large_error_proportion + small_error_proportion > 1:
+            raise ValueError("large_error_proportion and small_error_proportion must sum to a value less than 1.")
+        if loss not in ["squared", "absolute", "huber"]:
+            raise ValueError('loss must be one in ["squared", "absolute", "huber"].')
+        if not isinstance(huber_delta, int | float) or huber_delta <= 0:
+            raise ValueError("huber_delta must be a positive real number.")
+        if not isinstance(max_conflict_rate, float) or max_conflict_rate < 0 or max_conflict_rate >= 1:
+            raise ValueError("max_conflict_rate must be a float in range [0, 1).")
+        if not isinstance(use_efb, bool):
+            raise TypeError("use_efb must be a boolean.")
+
+        self.n_trees = n_trees
+        self.learning_rate = learning_rate
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.n_bins = n_bins
+        self.reg_lambda = reg_lambda
+        self.gamma = gamma
+        self.trees = None
+        self.large_error_proportion = large_error_proportion
+        self.small_error_proportion = small_error_proportion
+        self.loss = self._choose_loss(loss, huber_delta)
+        self.use_efb = use_efb
+        if self.use_efb: self.efb = _ExclusiveFeatureBundling(max_conflict_rate=max_conflict_rate, n_bins=n_bins)
+
+    def _choose_loss(self, loss, huber_delta):
+        if loss == "squared":
+            return MSE(reduction="sum")
+        elif loss == "absolute":
+            return MAE(reduction="sum")
+        elif loss == "huber":
+            return Huber(reduction="sum", delta=huber_delta)
+
+    def fit(self, X, y, metrics=["loss"]):
+        """
+        Fits the LGBMRegressor model to the input data by fitting trees to the errors made by previous trees.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The target values corresponding to each sample.
+            metrics (dict[str, torch.Tensor]): Contains the metrics that will be calculated between fitting each tree and returned.
+        Returns:
+            metrics (dict[str, torch.Tensor]): The calculated metrics.
+        Raises:
+            TypeError: If the input matrix or the target vector is not a PyTorch tensor.
+            ValueError: If the input matrix or the target vector is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The target must be 1 dimensional with the same number of samples as the input data")
+        if not isinstance(metrics, list | tuple):
+            raise ValueError("metrics must be a list or tuple containing the shorthand names of each wanted metric.")
+        self.n_features = X.shape[1]
+
+        if self.use_efb: X = self.efb.fit_transform(X)
+
+        self.initial_pred = y.mean()
+        pred = torch.full(y.shape, self.initial_pred)
+        trees = []
+        history = {metric: torch.zeros(self.n_trees) for metric in metrics}
+
+        top_n = int(len(y) * self.large_error_proportion)
+        rand_n = int(len(y) * self.small_error_proportion)
+        fact = (1 - self.large_error_proportion) / self.small_error_proportion
+
+        for i in range(self.n_trees):
+            gradient = self.loss.gradient(pred, y).to(X.dtype)
+            hessian = self.loss.hessian(pred, y).to(X.dtype)
+
+            # GOSS (gradient-based one-sided sampling)
+            indicies = torch.argsort(gradient, descending=True)
+            large_error_indicies = indicies[:top_n]
+            small_error_indicies = indicies[torch.randperm(len(indicies) - len(large_error_indicies))[:rand_n] + top_n]
+            train_indicies = torch.cat([large_error_indicies, small_error_indicies])
+            hessian[small_error_indicies] *= fact
+
+            tree = _LGBMTree(max_depth=self.max_depth, min_samples_split=self.min_samples_split, n_bins=self.n_bins, reg_lambda=self.reg_lambda, gamma=self.gamma)
+            tree.fit(X[train_indicies], gradient[train_indicies], hessian[train_indicies])
+            prediction = tree.predict(X)
+            pred += self.learning_rate * prediction
+
+            trees.append(tree)
+
+            values = calculate_metrics(data=(pred, y), metrics=metrics, loss=self.loss.loss)
+            for metric, value in values.items():
+                history[metric][i] = value
+
+        self.trees = trees
+        return history
+
+    def predict(self, X):
+        """
+        Applies the fitted LGBMRegressor model to the input data, predicting the target values.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            targets (torch.Tensor of shape (n_samples,)): The predicted target values corresponding to each sample.
+        Raises:
+            NotFittedError: If the LGBMRegressor model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "initial_pred"):
+            raise NotFittedError("LGBMRegressor.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        if self.use_efb: X = self.efb.transform(X)
+
+        pred = torch.full((X.shape[0],), self.initial_pred)
+        
+        for tree in self.trees:
+            pred += self.learning_rate * tree.predict(X)
+        return pred

DLL/MachineLearning/SupervisedLearning/Trees/_LGBMTree.py

@@ -0,0 +1,199 @@
+import torch
+import multiprocessing
+
+from ._DecisionTree import Node
+from ....Exceptions import NotFittedError
+from ....Data.Preprocessing import OneHotEncoder
+
+
+class _LGBMTree:
+    def __init__(self, max_depth=25, min_samples_split=2, n_bins=30, reg_lambda=1, gamma=1):
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        if not isinstance(reg_lambda, int | float) or reg_lambda <= 0:
+            raise ValueError("reg_lambda must be a positive real number.")
+        if not isinstance(n_bins, int) or n_bins <= 1:
+            raise ValueError("n_bins must be an integer greater than 1.")
+        if not isinstance(gamma, int | float) or gamma < 0:
+            raise ValueError("gamma must be a non-negative real number.")
+        
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.reg_lambda  = reg_lambda
+        self.gamma = gamma
+        self.n_bins = n_bins
+        self.root = None
+    
+    def _value(self, gradient, hessian):
+        return -gradient.sum() / (hessian.sum() + self.reg_lambda)
+    
+    def fit(self, X, gradient, hessian):
+        if not isinstance(X, torch.Tensor) or not isinstance(gradient, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if gradient.ndim != 1 or gradient.shape[0] != X.shape[0]:
+            raise ValueError("The gradient must be 1 dimensional with the same number of samples as the input data")
+        if hessian.ndim != 1 or hessian.shape[0] != X.shape[0]:
+            raise ValueError("The hessian must be 1 dimensional with the same number of samples as the input data")
+        
+        self.n_features = X.shape[1]
+        self.root = self._grow_tree(X, gradient, hessian, 0)
+
+    def _grow_tree(self, X, gradient, hessian, depth):
+        n_samples, n_features = X.size()
+        vals = torch.unique(gradient)
+
+        if depth >= self.max_depth or len(vals) == 1 or n_samples < self.min_samples_split:
+            return Node(value=self._value(gradient, hessian))
+
+        feature_indicies = torch.randperm(n_features)[:self.n_features]
+        split_threshold, split_index = self._best_split(X, gradient, hessian, feature_indicies)
+        # if no split gains more information
+        if split_threshold is None:
+            return Node(value=self._value(gradient, hessian))
+        
+        left_indicies, right_indicies = self._split(X[:, split_index], split_threshold)
+        left = self._grow_tree(X[left_indicies], gradient[left_indicies], hessian[left_indicies], depth + 1)
+        right = self._grow_tree(X[right_indicies], gradient[right_indicies], hessian[right_indicies], depth + 1)
+        return Node(left, right, split_threshold, split_index)
+    
+    def _best_split(self, X, gradient, hessian, feature_indicies):
+        max_gain = -1
+        split_index = None
+        split_threshold = None
+
+        parent_gain = self._gain_node(gradient, hessian)
+
+        for index in feature_indicies:
+            feature_values = X[:, index]
+            possible_thresholds = torch.unique(feature_values)
+            if len(possible_thresholds) > self.n_bins: _, possible_thresholds = torch.histogram(feature_values, bins=self.n_bins, weight=gradient)
+            for threshold in possible_thresholds:
+                gain = parent_gain + self._children_gain(gradient, hessian, feature_values, threshold)
+                if gain > max_gain:
+                    max_gain = gain
+                    split_index = index
+                    split_threshold = threshold
+        if max_gain <= 0: return None, None
+        return split_threshold, split_index
+    
+    def _split(self, feature_values, threshold):
+        left_indicies = torch.argwhere(feature_values <= threshold).flatten()
+        right_indicies = torch.argwhere(feature_values > threshold).flatten()
+        return left_indicies, right_indicies
+
+    def _children_gain(self, gradient, hessian, feature_values, threshold):
+        left_indicies, right_indicies = self._split(feature_values, threshold)
+        if len(left_indicies) <= 1 or len(right_indicies) <= 1:
+            return 0
+        return -self._gain_node(gradient[left_indicies], hessian[left_indicies]) - self._gain_node(gradient[right_indicies], hessian[right_indicies]) - self.gamma
+
+    def _gain_node(self, gradient, hessian):
+        return -(gradient.sum() ** 2) / (hessian.sum() + self.reg_lambda)
+
+    def predict(self, X):
+        if self.root is None:
+            raise NotFittedError("_XGBoostTree.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        return torch.tensor([self._predict_single(datapoint, self.root) for datapoint in X])
+
+    def _predict_single(self, x, current_node):
+        if current_node.is_leaf():
+            return current_node.value
+        
+        if x[current_node.feature_index] <= current_node.threshold:
+            return self._predict_single(x, current_node.left)
+        
+        return self._predict_single(x, current_node.right)
+
+
+# _merge_features and _greedy_bundling functions based on https://github.com/meanxai/machine_learning/blob/main/12.LGBM/5.efb_onehot.py
+class _ExclusiveFeatureBundling:
+    def __init__(self, max_conflict_rate=0.0, n_bins=30):
+        self.max_conflict_rate = max_conflict_rate
+        self.n_bins = n_bins
+        self.bundled_features = []
+        self.mapping = {}
+        self.non_sparse_features = None
+
+    def fit(self, X):
+        self.mins = torch.min(X, dim=0).values
+        self.maxs = torch.max(X, dim=0).values
+        X = self._discretize(X)
+
+        self.bundles = self._greedy_bundling(X)
+        result = self._merge_bundles(X, self.bundles)
+        return result
+
+    def _merge_bundles(self, X, bundles):
+        result = torch.zeros(len(X), len(bundles))
+        for i, bundle in enumerate(bundles):
+            combined_features = torch.stack([X[:, i] for i in bundle])
+            new_feature, _ = self._merge_features(combined_features)
+            result[:, i] = new_feature - 1
+        return result
+    
+    def _merge_features(self, combined_features):
+        bin_ranges = [0]
+        total_bin = 0
+        for feature in combined_features:
+            total_bin += torch.max(feature)
+            bin_ranges.append(total_bin)
+
+        new_feature = combined_features[0]
+        for i in range(len(combined_features)):
+            for j in range(1, len(combined_features)):
+                if combined_features[j][i] != 0:
+                    new_feature[i] = combined_features[j][i] + bin_ranges[j]
+        return new_feature, bin_ranges
+
+    def _greedy_bundling(self, X):
+        n_features = X.shape[1]
+        conflictCnt = torch.zeros((n_features, n_features))
+        
+        for i in range(n_features):
+            for j in range(i+1, n_features):
+                n_conflicts = torch.sum(X[:, i] * X[:, j] > 0)
+                conflictCnt[i, j] = n_conflicts
+                conflictCnt[j, i] = n_conflicts
+
+        degree = conflictCnt.sum(axis=0)
+        searchOrder = torch.argsort(degree, descending=True)
+        
+        bundles = []
+        bundlesConflict = []
+        for i in searchOrder:
+            needNew = True
+            for j in range(len(bundles)):
+                cnt = conflictCnt[bundles[j][-1], i]
+                if cnt + bundlesConflict[j] <= self.max_conflict_rate * len(X):  # K from the paper but different parametrisation
+                    bundles[j].append(i)
+                    bundlesConflict[j] += cnt     
+                    needNew = False
+                    break
+            if needNew:
+                bundles.append([i])
+                bundlesConflict.append(0)
+        return bundles
+    
+    def fit_transform(self, X):
+        self.fit(X)
+        return self.transform(X)
+
+    def transform(self, X):
+        X = self._discretize(X)
+        return self._merge_bundles(X, self.bundles)
+
+    def _discretize(self, X):
+        X_discrete = torch.zeros_like(X)
+        for i in range(X.shape[1]):
+            bins = torch.linspace(self.mins[i], self.maxs[i], self.n_bins)
+            descrete = torch.bucketize(X[:, i].contiguous(), bins)
+            X_discrete[:, i] = torch.clamp(descrete, 0, self.n_bins - 1)  # handle the possible values outside the bounds during transformation phase
+        return X_discrete

DLL/MachineLearning/SupervisedLearning/Trees/_RandomForest.py

@@ -0,0 +1,96 @@
+import torch
+from collections import Counter
+
+from ._DecisionTree import DecisionTree
+from ....Exceptions import NotFittedError
+
+
+class RandomForestClassifier:
+    """
+    RandomForestClassifier implements a classification algorithm fitting many :class:`DecisionTrees <DLL.MachineLearning.SupervisedLearning.Trees.DecisionTree>` to bootstrapped data.
+
+    Args:
+        n_trees (int, optional): The number of trees used for predicting. Defaults to 10. Must be a positive integer.
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 10. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+    """
+    def __init__(self, n_trees=10, max_depth=10, min_samples_split=2):
+        if not isinstance(n_trees, int) or n_trees < 1:
+            raise ValueError("n_trees must be a positive integer.")
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        self.n_trees = n_trees
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.trees = [DecisionTree(max_depth=max_depth, min_samples_split=min_samples_split) for _ in range(n_trees)]
+
+    def fit(self, X, y):
+        """
+        Fits the RandomForestClassifier model to the input data by generating trees, which split the data appropriately.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The labels corresponding to each sample.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix or the label matrix is not a PyTorch tensor.
+            ValueError: If the input matrix or the label matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the label matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The labels must be 1 dimensional with the same number of samples as the input data")
+        for tree in self.trees:
+            tree.fit(*self._bootstrap_sample(X, y))
+
+    def predict(self, X):
+        """
+        Applies the fitted RandomForestClassifier model to the input data, predicting the correct classes.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The predicted labels corresponding to each sample.
+        Raises:
+            NotFittedError: If the RandomForestClassifier model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if self.trees is None:
+            raise NotFittedError("RandomForestClassifier.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.trees[0].n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        predictions = torch.stack([tree.predict(X) for tree in self.trees]).T
+        return torch.tensor([Counter(sample_prediction).most_common(1)[0][0] for sample_prediction in predictions])
+    
+    def predict_proba(self, X):
+        """
+        Applies the fitted RandomForestClassifier model to the input data, predicting the probabilities of each class. Is calculated as the average of each individual trees predicted probabilities.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be classified.
+        Returns:
+            probabilities (torch.Tensor of shape (n_samples, n_classes)): The predicted probabilities corresponding to each sample.
+        Raises:
+            NotFittedError: If the RandomForestClassifier model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if self.trees is None:
+            raise NotFittedError("RandomForestClassifier.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.trees[0].n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        return torch.stack([tree.predict_proba(X) for tree in self.trees]).mean(dim=0)
+
+    def _bootstrap_sample(self, X, y):
+        indices = torch.randint(high=len(y), size=(len(y), 1)).flatten()
+        return X[indices], y[indices]

DLL/MachineLearning/SupervisedLearning/Trees/_RegressionForest.py

@@ -0,0 +1,75 @@
+import torch
+from collections import Counter
+
+from ._RegressionTree import RegressionTree
+from ....Exceptions import NotFittedError
+
+
+class RandomForestRegressor:
+    """
+    RandomForestRegressor implements a regression algorithm fitting many :class:`RegressionTrees <DLL.MachineLearning.SupervisedLearning.Trees.RegressionTree>` to bootstrapped data.
+
+    Args:
+        n_trees (int, optional): The number of trees used for predictiong. Defaults to 10. Must be a positive integer.
+        max_depth (int, optional): The maximum depth of the tree. Defaults to 10. Must be a positive integer.
+        min_samples_split (int, optional): The minimum required samples in a leaf to make a split. Defaults to 2. Must be a positive integer.
+    """
+    def __init__(self, n_trees=10, max_depth=25, min_samples_split=2):
+        if not isinstance(n_trees, int) or n_trees < 1:
+            raise ValueError("n_trees must be a positive integer.")
+        if not isinstance(max_depth, int) or max_depth < 1:
+            raise ValueError("max_depth must be a positive integer.")
+        if not isinstance(min_samples_split, int) or min_samples_split < 1:
+            raise ValueError("min_samples_split must be a positive integer.")
+        self.n_trees = n_trees
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.trees = [RegressionTree(max_depth=max_depth, min_samples_split=min_samples_split) for _ in range(n_trees)]
+
+    def fit(self, X, y):
+        """
+        Fits the RandomForestRegressor model to the input data by generating trees, which split the data appropriately.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The target values corresponding to each sample.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix or the target matrix is not a PyTorch tensor.
+            ValueError: If the input matrix or the target matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or y.shape[0] != X.shape[0]:
+            raise ValueError("The target must be 1 dimensional with the same number of samples as the input data")
+        for tree in self.trees:
+            tree.fit(*self._bootstrap_sample(X, y))
+
+    def predict(self, X):
+        """
+        Applies the fitted RandomForestRegressor model to the input data, predicting the correct values.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be regressed.
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The predicted target values corresponding to each sample.
+        Raises:
+            NotFittedError: If the RandomForestRegressor model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if self.trees is None:
+            raise NotFittedError("RandomForestRegressor.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.trees[0].n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        predictions = torch.stack([tree.predict(X) for tree in self.trees]).T
+        return torch.tensor([Counter(sample_prediction).most_common(1)[0][0] for sample_prediction in predictions])
+    
+    def _bootstrap_sample(self, x, y):
+        indices = torch.randint(high=len(y), size=(len(y), 1)).flatten()
+        return x[indices], y[indices]