forust 0.2.0__tar.gz → 0.2.2__tar.gz

{forust-0.2.0 → forust-0.2.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "py-forust"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -10,6 +10,6 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.17", features = ["extension-module"] }
-forust-ml = { version="0.2.0", path= "local_dependencies/forust-ml" }
+forust-ml = { version="0.2.2", path= "local_dependencies/forust-ml" }
 numpy = "0.17.2"
 ndarray = "0.15.1"

{forust-0.2.0 → forust-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: forust
-Version: 0.2.0
+Version: 0.2.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -47,7 +47,7 @@ pip install forust
 
 To use in a rust project add the following to your Cargo.toml file.
 ```toml
-forust-ml = "0.2.0"
+forust-ml = "0.2.2"
 ```
 
 ## Usage
@@ -105,6 +105,13 @@ model.fit(X, y)
 # Predict on data
 model.predict(X.head())
 # array([-1.94919663,  2.25863229,  0.32963671,  2.48732194, -3.00371813])
+
+# predict contributions
+model.predict_contributions(X.head())
+# array([[-0.63014213,  0.33880048, -0.16520798, -0.07798772, -0.85083578,
+#        -1.07720813],
+#       [ 1.05406709,  0.08825999,  0.21662544, -0.12083538,  0.35209258,
+#        -1.07720813],
 ```
 
 The `fit` method accepts the following arguments.
@@ -123,6 +130,13 @@ The predict method accepts the following arguments.
    passed, the `parallel` attribute of the booster will be used.
    Defaults to `None`.
 
+The `predict_contributions` method will predict with the fitted booster on new data, returning the feature contribution matrix. The last column is the bias term.
+ - `X` ***(FrameLike)***: Either a pandas DataFrame, or a 2 dimensional numpy array, with numeric data.
+ - `parallel` ***(Optional[bool], optional)***: Optionally specify if the predict
+   function should run in parallel on multiple threads. If `None` is
+   passed, the `parallel` attribute of the booster will be used.
+   Defaults to `None`.
+
 ### Inspecting the Model
 
 Once the booster has been fit, each individual tree structure can be retrieved in text form, using the `text_dump` method. This method returns a list, the same length as the number of trees in the model.

{forust-0.2.0 → forust-0.2.2}/README.md

@@ -26,7 +26,7 @@ pip install forust
 
 To use in a rust project add the following to your Cargo.toml file.
 ```toml
-forust-ml = "0.2.0"
+forust-ml = "0.2.2"
 ```
 
 ## Usage
@@ -84,6 +84,13 @@ model.fit(X, y)
 # Predict on data
 model.predict(X.head())
 # array([-1.94919663,  2.25863229,  0.32963671,  2.48732194, -3.00371813])
+
+# predict contributions
+model.predict_contributions(X.head())
+# array([[-0.63014213,  0.33880048, -0.16520798, -0.07798772, -0.85083578,
+#        -1.07720813],
+#       [ 1.05406709,  0.08825999,  0.21662544, -0.12083538,  0.35209258,
+#        -1.07720813],
 ```
 
 The `fit` method accepts the following arguments.
@@ -102,6 +109,13 @@ The predict method accepts the following arguments.
    passed, the `parallel` attribute of the booster will be used.
    Defaults to `None`.
 
+The `predict_contributions` method will predict with the fitted booster on new data, returning the feature contribution matrix. The last column is the bias term.
+ - `X` ***(FrameLike)***: Either a pandas DataFrame, or a 2 dimensional numpy array, with numeric data.
+ - `parallel` ***(Optional[bool], optional)***: Optionally specify if the predict
+   function should run in parallel on multiple threads. If `None` is
+   passed, the `parallel` attribute of the booster will be used.
+   Defaults to `None`.
+
 ### Inspecting the Model
 
 Once the booster has been fit, each individual tree structure can be retrieved in text form, using the `text_dump` method. This method returns a list, the same length as the number of trees in the model.

{forust-0.2.0 → forust-0.2.2}/forust/__init__.py

@@ -35,6 +35,15 @@ class BoosterType:
     ) -> np.ndarray:
         raise NotImplementedError()
 
+    def predict_contributions(
+        self,
+        flat_data: np.ndarray,
+        rows: int,
+        cols: int,
+        parallel: bool = True,
+    ) -> np.ndarray:
+        raise NotImplementedError
+
     def value_partial_dependence(
         self,
         feature: int,
@@ -238,6 +247,37 @@ class GradientBooster:
             parallel=parallel_,
         )
 
+    def predict_contributions(
+        self, X: FrameLike, parallel: Union[bool, None] = None
+    ) -> np.ndarray:
+        """Predict with the fitted booster on new data, returning the feature
+        contribution matrix. The last column is the bias term.
+
+        Args:
+            X (FrameLike): Either a pandas DataFrame, or a 2 dimensional numpy array.
+            parallel (Union[bool, None], optional): Optionally specify if the predict
+                function should run in parallel on multiple threads. If `None` is
+                passed, the `parallel` attribute of the booster will be used.
+                Defaults to `None`.
+
+        Returns:
+            np.ndarray: Returns a numpy array of the predictions.
+        """
+        X_ = X.to_numpy() if isinstance(X, pd.DataFrame) else X
+        if not np.issubdtype(X_.dtype, "float64"):
+            X_ = X_.astype(dtype="float64", copy=False)
+
+        parallel_ = self.parallel if parallel is None else parallel
+        flat_data = X_.ravel(order="F")
+        rows, cols = X_.shape
+        contributions = self.booster.predict_contributions(
+            flat_data=flat_data,
+            rows=rows,
+            cols=cols,
+            parallel=parallel_,
+        )
+        return np.reshape(contributions, (X_.shape[0], X_.shape[1] + 1))
+
     def partial_dependence(self, X: FrameLike, feature: Union[str, int]) -> np.ndarray:
         """Calculate the partial dependence values of a feature. For each unique
         value of the feature, this gives the estimate of the predicted value for that

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/.github/workflows/CI.yml

@@ -5,7 +5,7 @@ jobs:
   windows-build-tests:
     strategy:
       matrix:
-        pyversion: ["3.8", "3.9", "3.10"]
+        pyversion: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: "windows-latest"
     steps:
       - uses: actions/checkout@v2
@@ -58,7 +58,7 @@ jobs:
   macos-build-test:
     strategy:
       matrix:
-        pyversion: ["3.8", "3.9", "3.10"]
+        pyversion: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: "macos-latest"
     steps:
       - uses: actions/checkout@v2
@@ -113,7 +113,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pyversion: ["3.8", "3.9", "3.10"]
+        pyversion: ["3.8", "3.9", "3.10", "3.11"]
         # target: [x86_64, i686]
     steps:
       - uses: actions/checkout@v2

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "forust-ml"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 authors = ["James Inlow <james.d.inlow@gmail.com>"]
 homepage = "https://github.com/jinlow/forust"
@@ -12,8 +12,8 @@ repository = "https://github.com/jinlow/forust"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rayon = "1.5"
-thiserror = "1.0.31"
+rayon = "1.7"
+thiserror = "1.0"
 serde_json = {version = "1.0", features = ["float_roundtrip"] }
 serde = { version = "1.0", features = ["derive"] }
 

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/README.md

@@ -26,7 +26,7 @@ pip install forust
 
 To use in a rust project add the following to your Cargo.toml file.
 ```toml
-forust-ml = "0.2.0"
+forust-ml = "0.2.2"
 ```
 
 ## Usage
@@ -84,6 +84,13 @@ model.fit(X, y)
 # Predict on data
 model.predict(X.head())
 # array([-1.94919663,  2.25863229,  0.32963671,  2.48732194, -3.00371813])
+
+# predict contributions
+model.predict_contributions(X.head())
+# array([[-0.63014213,  0.33880048, -0.16520798, -0.07798772, -0.85083578,
+#        -1.07720813],
+#       [ 1.05406709,  0.08825999,  0.21662544, -0.12083538,  0.35209258,
+#        -1.07720813],
 ```
 
 The `fit` method accepts the following arguments.
@@ -102,6 +109,13 @@ The predict method accepts the following arguments.
    passed, the `parallel` attribute of the booster will be used.
    Defaults to `None`.
 
+The `predict_contributions` method will predict with the fitted booster on new data, returning the feature contribution matrix. The last column is the bias term.
+ - `X` ***(FrameLike)***: Either a pandas DataFrame, or a 2 dimensional numpy array, with numeric data.
+ - `parallel` ***(Optional[bool], optional)***: Optionally specify if the predict
+   function should run in parallel on multiple threads. If `None` is
+   passed, the `parallel` attribute of the booster will be used.
+   Defaults to `None`.
+
 ### Inspecting the Model
 
 Once the booster has been fit, each individual tree structure can be retrieved in text form, using the `text_dump` method. This method returns a list, the same length as the number of trees in the model.

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/rs-example.md

@@ -3,7 +3,7 @@
 To run this example, add the following code to your `Cargo.toml` file.
 ```toml
 [dependencies]
-forust-ml = "0.2.0"
+forust-ml = "0.2.2"
 polars = "0.24"
 reqwest = { version = "0.11", features = ["blocking"] }
 ```

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/src/data.rs

@@ -93,6 +93,7 @@ pub struct Matrix<'a, T> {
 }
 
 impl<'a, T> Matrix<'a, T> {
+    // Defaults to column major
     pub fn new(data: &'a [T], rows: usize, cols: usize) -> Self {
         Matrix {
             data,
@@ -118,6 +119,14 @@ impl<'a, T> Matrix<'a, T> {
         idx
     }
 
+    /// Get access to a row of the data, as an iterator.
+    pub fn get_row_iter(
+        &self,
+        row: usize,
+    ) -> std::iter::StepBy<std::iter::Skip<std::slice::Iter<T>>> {
+        self.data.iter().skip(row).step_by(self.rows)
+    }
+
     /// Get a slice of a column in the matrix.
     ///
     /// * `col` - The index of the column to select.
@@ -137,6 +146,30 @@ impl<'a, T> Matrix<'a, T> {
     }
 }
 
+impl<'a, T> Matrix<'a, T>
+where
+    T: Copy,
+{
+    /// Get a row of the data as a vector.
+    pub fn get_row(&self, row: usize) -> Vec<T> {
+        self.get_row_iter(row).copied().collect()
+    }
+}
+
+/// A lightweight row major matrix, this is primarily
+/// for returning data to the user, it is especially
+/// suited for appending rows to, such as when building
+/// up a matrix of contributions to return to the
+/// user, the added benefit is it will be even
+/// faster to return to numpy.
+// pub struct RowMajorMatrix<T> {
+//     pub data: Vec<T>,
+//     pub rows: usize,
+//     pub cols: usize,
+//     stride1: usize,
+//     stride2: usize,
+// }
+
 impl<'a, T> fmt::Display for Matrix<'a, T>
 where
     T: FromStr + std::fmt::Display,
@@ -212,7 +245,6 @@ impl<T> JaggedMatrix<T> {
             n_records: 0,
         }
     }
-
     /// Get the column of a jagged array.
     pub fn get_col(&self, col: usize) -> &[T] {
         assert!(col < self.ends.len());
@@ -270,6 +302,15 @@ mod tests {
         assert_eq!(m.get_col(1), &vec![5, 6, 7]);
     }
 
+    #[test]
+    fn test_matrix_row() {
+        let v = vec![1, 2, 3, 5, 6, 7];
+        let m = Matrix::new(&v, 3, 2);
+        assert_eq!(m.get_row(2), vec![3, 7]);
+        assert_eq!(m.get_row(0), vec![1, 5]);
+        assert_eq!(m.get_row(1), vec![2, 6]);
+    }
+
     #[test]
     fn test_jaggedmatrix_get_col() {
         let vecs = vec![vec![0], vec![5, 4, 3, 2], vec![4, 5]];

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/src/gradientbooster.rs

@@ -5,6 +5,7 @@ use crate::errors::ForustError;
 use crate::objective::{gradient_hessian_callables, ObjectiveType};
 use crate::splitter::MissingImputerSplitter;
 use crate::tree::Tree;
+use rayon::prelude::*;
 use serde::{Deserialize, Serialize};
 use std::fs;
 
@@ -215,6 +216,84 @@ impl GradientBooster {
         init_preds
     }
 
+    // pub fn predict(&self, data: &Matrix<f64>, parallel: bool) -> Vec<f64> {
+    //     // After we disconvered it's faster materializing the row once, and then
+    //     // Passing that to each tree, let's see if we can do the same with the booster
+    //     // prediction...
+    //     // Clean this up..
+    //     let mut init_preds = vec![self.base_score; data.rows];
+    //     if parallel {
+    //         init_preds.par_iter_mut().enumerate().for_each(|(i, p)| {
+    //             let pred_row = data.get_row(i);
+    //             for t in &self.trees {
+    //                 *p += t.predict_row_from_row_slice(&pred_row);
+    //             }
+    //         });
+    //     } else {
+    //         init_preds.iter_mut().enumerate().for_each(|(i, p)| {
+    //             let pred_row = data.get_row(i);
+    //             for t in &self.trees {
+    //                 *p += t.predict_row_from_row_slice(&pred_row);
+    //             }
+    //         });
+    //     }
+    //     init_preds
+    // }
+
+    /// Generate predictions on data using the gradient booster.
+    ///
+    /// * `data` -  Either a pandas DataFrame, or a 2 dimensional numpy array.
+    pub fn predict_contributions(&self, data: &Matrix<f64>, parallel: bool) -> Vec<f64> {
+        let weights: Vec<Vec<f64>> = if parallel {
+            self.trees
+                .par_iter()
+                .map(|t| t.distribute_leaf_weights())
+                .collect()
+        } else {
+            self.trees
+                .iter()
+                .map(|t| t.distribute_leaf_weights())
+                .collect()
+        };
+        let mut contribs = vec![0.; (data.cols + 1) * data.rows];
+
+        // Add the bias term to every bias value...
+        let bias_idx = data.cols + 1;
+        contribs
+            .iter_mut()
+            .skip(bias_idx - 1)
+            .step_by(bias_idx)
+            .for_each(|v| *v += self.base_score);
+
+        // Clean this up..
+        // materializing a row, and then passing that to all of the
+        // trees seems to be the fastest approach (5X faster), we should test
+        // something like this for normal predictions.
+        if parallel {
+            data.index
+                .par_iter()
+                .zip(contribs.par_chunks_mut(data.cols + 1))
+                .for_each(|(row, c)| {
+                    let r_ = data.get_row(*row);
+                    self.trees.iter().zip(weights.iter()).for_each(|(t, w)| {
+                        t.predict_contributions_row(&r_, c, w);
+                    });
+                });
+        } else {
+            data.index
+                .iter()
+                .zip(contribs.chunks_mut(data.cols + 1))
+                .for_each(|(row, c)| {
+                    let r_ = data.get_row(*row);
+                    self.trees.iter().zip(weights.iter()).for_each(|(t, w)| {
+                        t.predict_contributions_row(&r_, c, w);
+                    });
+                });
+        }
+
+        contribs
+    }
+
     /// Given a value, return the partial dependence value of that value for that
     /// feature in the model.
     ///
@@ -390,6 +469,8 @@ mod tests {
         let sample_weight = vec![1.; y.len()];
         booster.fit(&data, &y, &sample_weight).unwrap();
         let preds = booster.predict(&data, false);
+        let contribs = booster.predict_contributions(&data, false);
+        assert_eq!(contribs.len(), (data.cols + 1) * data.rows);
         println!("{}", booster.trees[0]);
         println!("{}", booster.trees[0].nodes.len());
         println!("{}", booster.trees.last().unwrap().nodes.len());

{forust-0.2.0 → forust-0.2.2}/local_dependencies/forust-ml/src/node.rs

@@ -23,10 +23,11 @@ pub struct SplittableNode {
     pub stop_idx: usize,
     pub lower_bound: f32,
     pub upper_bound: f32,
+    pub is_leaf: bool,
 }
 
 #[derive(Deserialize, Serialize)]
-pub struct ParentNode {
+pub struct Node {
     pub num: usize,
     pub weight_value: f32,
     pub hessian_sum: f32,
@@ -37,6 +38,32 @@ pub struct ParentNode {
     pub missing_node: usize,
     pub left_child: usize,
     pub right_child: usize,
+    pub is_leaf: bool,
+}
+
+impl Node {
+    /// Update all the info that is needed if this node is a
+    /// parent node, this consumes the SplitableNode.
+    pub fn make_parent_node(&mut self, split_node: SplittableNode) {
+        self.is_leaf = false;
+        self.missing_node = split_node.missing_node;
+        self.split_value = split_node.split_value;
+        self.split_feature = split_node.split_feature;
+        self.split_gain = split_node.split_gain;
+        self.left_child = split_node.left_child;
+        self.right_child = split_node.right_child;
+    }
+    /// Get the path that should be traveled down, given a value.
+    pub fn get_child_idx(&self, v: &f64) -> usize {
+        if v.is_nan() {
+            self.missing_node
+        } else if v < &self.split_value {
+            self.left_child
+        } else {
+            //  if v >= &self.split_value
+            self.right_child
+        }
+    }
 }
 
 #[derive(Deserialize, Serialize)]
@@ -47,13 +74,6 @@ pub struct LeafNode {
     pub depth: usize,
 }
 
-#[derive(Deserialize, Serialize)]
-pub enum TreeNode {
-    Parent(ParentNode),
-    Leaf(LeafNode),
-    Splittable(SplittableNode),
-}
-
 impl SplittableNode {
     pub fn from_node_info(
         num: usize,
@@ -81,9 +101,12 @@ impl SplittableNode {
             stop_idx,
             lower_bound: node_info.bounds.0,
             upper_bound: node_info.bounds.1,
+            is_leaf: true,
         }
     }
 
+    /// Create a default splitable node,
+    /// we default to the node being a leaf.
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         num: usize,
@@ -116,6 +139,7 @@ impl SplittableNode {
             stop_idx,
             lower_bound,
             upper_bound,
+            is_leaf: true,
         }
     }
 
@@ -136,17 +160,10 @@ impl SplittableNode {
             MissingInfo::Branch(_) => todo!(),
             MissingInfo::EmptyBranch => todo!(),
         };
+        self.is_leaf = false;
     }
-    pub fn as_leaf_node(&self) -> TreeNode {
-        TreeNode::Leaf(LeafNode {
-            num: self.num,
-            weight_value: self.weight_value,
-            hessian_sum: self.hessian_sum,
-            depth: self.depth,
-        })
-    }
-    pub fn as_parent_node(&self) -> TreeNode {
-        TreeNode::Parent(ParentNode {
+    pub fn as_node(&self) -> Node {
+        Node {
             num: self.num,
             weight_value: self.weight_value,
             hessian_sum: self.hessian_sum,
@@ -157,47 +174,33 @@ impl SplittableNode {
             split_gain: self.split_gain,
             left_child: self.left_child,
             right_child: self.right_child,
-        })
+            is_leaf: self.is_leaf,
+        }
     }
 }
 
-impl fmt::Display for TreeNode {
+impl fmt::Display for Node {
     // This trait requires `fmt` with this exact signature.
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            TreeNode::Leaf(leaf) => write!(
+        if self.is_leaf {
+            write!(
                 f,
                 "{}:leaf={},cover={}",
-                leaf.num, leaf.weight_value, leaf.hessian_sum
-            ),
-            TreeNode::Parent(parent) => {
-                write!(
-                    f,
-                    "{}:[{} < {}] yes={},no={},missing={},gain={},cover={}",
-                    parent.num,
-                    parent.split_feature,
-                    parent.split_value,
-                    parent.left_child,
-                    parent.right_child,
-                    parent.missing_node,
-                    parent.split_gain,
-                    parent.hessian_sum
-                )
-            }
-            TreeNode::Splittable(node) => {
-                write!(
-                    f,
-                    "SPLITTABLE - {}:[{} < {}] yes={},no={},missing={},gain={},cover={}",
-                    node.num,
-                    node.split_feature,
-                    node.split_value,
-                    node.missing_node,
-                    node.left_child,
-                    node.right_child,
-                    node.split_gain,
-                    node.hessian_sum
-                )
-            }
+                self.num, self.weight_value, self.hessian_sum
+            )
+        } else {
+            write!(
+                f,
+                "{}:[{} < {}] yes={},no={},missing={},gain={},cover={}",
+                self.num,
+                self.split_feature,
+                self.split_value,
+                self.left_child,
+                self.right_child,
+                self.missing_node,
+                self.split_gain,
+                self.hessian_sum
+            )
         }
     }
 }