sciml 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

sciml/__init__.py

@@ -1,2 +1,2 @@
-# coding: utf-8
-__all__ = ["utils", "pipelines"]
+# coding: utf-8
+__all__ = ["utils", "pipelines", "models"]

sciml/models.py

@@ -0,0 +1,277 @@
+import numpy as np
+import copy
+import itertools
+import warnings
+from xgboost import XGBRegressor
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import train_test_split
+
+class SmartForest:
+    """
+    SmartForest: A deep, intelligent decision forest model for complex sequential and tabular data.
+
+    SmartForest blends ideas from deep forests (cascade forest structures), LSTM-style forget gates,
+    and ensemble learning using XGBoost. It is especially suited for time series or structured tabular data 
+    where layer-wise feature expansion and memory-inspired filtering can enhance performance.
+
+    Key Features:
+    -------------
+    - Deep cascade of XGBoost regressors
+    - Optional Multi-Grained Scanning (MGS) for local feature extraction
+    - Forget-gate-inspired mechanism to regulate information flow across layers
+    - Early stopping to prevent overfitting
+    - Full retention of best-performing model (lowest validation RMSE)
+
+    Parameters:
+    -----------
+    n_estimators_per_layer : int
+        Number of XGBoost regressors per layer.
+    
+    max_layers : int
+        Maximum number of layers (depth) in the model.
+
+    early_stopping_rounds : int
+        Number of layers with no improvement before early stopping is triggered.
+
+    param_grid : dict
+        Grid of XGBoost hyperparameters to search over.
+
+    use_gpu : bool
+        If True, use GPU-accelerated training (CUDA required).
+
+    gpu_id : int
+        ID of GPU to use (if use_gpu=True).
+
+    window_sizes : list of int
+        Enables Multi-Grained Scanning if non-empty, with specified sliding window sizes.
+
+    forget_factor : float in [0, 1]
+        Simulates LSTM-style forget gate; higher values forget more past information.
+
+    verbose : int
+        Verbosity level (0 = silent, 1 = progress updates).
+
+    Methods:
+    --------
+    fit(X, y, X_val=None, y_val=None):
+        Train the SmartForest model layer by layer, using optional validation for early stopping.
+
+    predict(X):
+        Make predictions on new data using the trained cascade structure.
+
+    get_best_model():
+        Returns a copy of the best model and the corresponding RMSE from validation.
+
+    Example:
+    --------
+    >>> model = SmartForest(n_estimators_per_layer=5, max_layers=10, window_sizes=[2, 3], forget_factor=0.2)
+    >>> model.fit(X_train, y_train, X_val, y_val)
+    >>> y_pred = model.predict(X_val)
+    >>> best_model, best_rmse = model.get_best_model()
+    """
+    def __init__(self, n_estimators_per_layer = 5, max_layers = 10, early_stopping_rounds = 3, param_grid = None,
+                 use_gpu = False, gpu_id = 0, window_sizes = [], forget_factor = 0, verbose = 1):
+        self.n_estimators_per_layer = n_estimators_per_layer
+        self.max_layers = max_layers
+        self.early_stopping_rounds = early_stopping_rounds
+        self.param_grid = param_grid or {
+            "objective": ["reg:squarederror"],
+            "random_state": [42],
+            'seed': [0],
+            'n_estimators': [100],
+            'max_depth': [6],
+            'min_child_weight': [4],
+            'subsample': [0.8],
+            'colsample_bytree': [0.8],
+            'gamma': [0],
+            'reg_alpha': [0],
+            'reg_lambda': [1],
+            'learning_rate': [0.05],
+        }
+        self.use_gpu = use_gpu
+        self.gpu_id = gpu_id
+        self.window_sizes = window_sizes
+        self.forget_factor = forget_factor
+        self.layers = []
+        self.best_model = None
+        self.best_rmse = float("inf")
+        self.verbose = verbose
+
+    def _get_param_combinations(self):
+        keys, values = zip(*self.param_grid.items())
+        return [dict(zip(keys, v)) for v in itertools.product(*values)]
+
+    def _multi_grained_scanning(self, X, y):
+        new_features = []
+        for window_size in self.window_sizes:
+            if X.shape[1] < window_size:
+                continue
+            for start in range(X.shape[1] - window_size + 1):
+                window = X[:, start:start + window_size]
+                if y is None:
+                    new_features.append(window)
+                    continue
+
+                param_combos = self._get_param_combinations()
+                for params in param_combos:
+                    if self.use_gpu:
+                        params['tree_method'] = 'hist'
+                        params['device'] = 'cuda'
+                    model = XGBRegressor(**params)
+                    model.fit(window, y)
+                    preds = model.predict(window).reshape(-1, 1)
+                    new_features.append(preds)
+        return np.hstack(new_features) if new_features else X
+
+    def _apply_forget_gate(self, X, layer_index):
+        forget_weights = np.random.rand(X.shape[1]) * self.forget_factor
+        return X * (1 - forget_weights)
+
+    def _fit_layer(self, X, y, X_val=None, y_val=None, layer_index=0):
+        layer = []
+        layer_outputs = []
+        param_combos = self._get_param_combinations()
+        X = self._apply_forget_gate(X, layer_index)
+
+        for i in range(self.n_estimators_per_layer):
+            best_rmse = float('inf')
+            best_model = None
+
+            for params in param_combos:
+                if self.use_gpu:
+                    params['tree_method'] = 'hist'
+                    params['device'] = 'cuda'
+
+                params = params.copy()  # Prevent modification from affecting the next loop iteration
+                params['random_state'] = i  # Use a different random seed for each model to enhance diversity
+
+                model = XGBRegressor(**params)
+                model.fit(X, y)
+
+                if X_val is not None:
+                    preds_val = model.predict(X_val)
+                    rmse = np.sqrt(mean_squared_error(y_val, preds_val))
+                    if rmse < best_rmse:
+                        best_rmse = rmse
+                        best_model = model
+                else:
+                    best_model = model
+
+            preds = best_model.predict(X).reshape(-1, 1)
+            layer.append(best_model)
+            layer_outputs.append(preds)
+
+        output = np.hstack(layer_outputs)
+        return layer, output
+
+    def fit(self, X, y, X_val=None, y_val=None):
+        X_current = self._multi_grained_scanning(X, y)
+        X_val_current = self._multi_grained_scanning(X_val, y_val) if X_val is not None else None
+        no_improve_rounds = 0
+
+        for layer_index in range(self.max_layers):
+            if self.verbose: print(f"Training Layer {layer_index + 1}")
+            layer, output = self._fit_layer(X_current, y, X_val_current, y_val, layer_index)
+            self.layers.append(layer)
+            X_current = np.hstack([X_current, output])
+
+            if X_val is not None:
+                val_outputs = []
+                for reg in layer:
+                    n_features = reg.n_features_in_
+                    preds = reg.predict(X_val_current[:, :n_features]).reshape(-1, 1)
+                    val_outputs.append(preds)
+                val_output = np.hstack(val_outputs)
+                X_val_current = np.hstack([X_val_current, val_output])
+
+                y_pred = self.predict(X_val)
+                rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+                if self.verbose: print(f"Validation RMSE: {rmse:.4f}")
+
+                if rmse < self.best_rmse:
+                    self.best_rmse = rmse
+                    self.best_model = copy.deepcopy(self.layers)
+                    no_improve_rounds = 0
+                    if self.verbose: print(f"✅ New best RMSE: {self.best_rmse:.4f}")
+                else:
+                    no_improve_rounds += 1
+                    if no_improve_rounds >= self.early_stopping_rounds:
+                        if self.verbose: print("Early stopping triggered.")
+                        break
+
+    def predict(self, X):
+        X_current = self._multi_grained_scanning(X, None)
+        X_current = self._apply_forget_gate(X_current, layer_index=-1)
+
+        for layer in self.layers:
+            layer_outputs = []
+            for reg in layer:
+                n_features = reg.n_features_in_
+                preds = reg.predict(X_current[:, :n_features]).reshape(-1, 1)
+                layer_outputs.append(preds)
+            output = np.hstack(layer_outputs)
+            X_current = np.hstack([X_current, output])
+
+        final_outputs = []
+        for reg in self.layers[-1]:
+            n_features = reg.n_features_in_
+            final_outputs.append(reg.predict(X_current[:, :n_features]).reshape(-1, 1))
+        return np.mean(np.hstack(final_outputs), axis=1)
+
+    def get_best_model(self):
+        return self.best_model, self.best_rmse
+
+"""
+# ============================== Test Example ==============================
+from sklearn.datasets import load_diabetes
+from sklearn.datasets import fetch_california_housing
+from sklearn.model_selection import train_test_split
+
+
+
+warnings.simplefilter('ignore')
+
+# X, y = load_diabetes(return_X_y=True) # Using diabetes dataset
+X, y = fetch_california_housing(return_X_y=True) # Using house price dataset
+X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)
+
+# Hyperparameter grid
+param_grid = {
+    "objective": ["reg:squarederror"],
+    "random_state": [42],
+    'seed': [0],
+    'n_estimators': [100],
+    'max_depth': [6],
+    'min_child_weight': [4],
+    'subsample': [0.8],
+    'colsample_bytree': [0.8],
+    'gamma': [0],
+    'reg_alpha': [0],
+    'reg_lambda': [1],
+    'learning_rate': [0.05],
+}
+
+# Create the model with Multi-Grained Scanning enabled (with window sizes 2 and 3)
+regr = SmartForest(
+    n_estimators_per_layer = 5,
+    max_layers = 10,
+    early_stopping_rounds = 5,
+    param_grid = param_grid,
+    use_gpu = False,
+    gpu_id = 0,
+    window_sizes = [],  # Enables MGS if e.g., [2, 3], else empty disables MGS.
+    forget_factor = 0.,  # Set forget factor to simulate forget gate behavior
+    verbose = 1
+)
+
+regr.fit(X_train, y_train, X_val, y_val)
+
+# Predict on validation set and evaluate
+y_pred = regr.predict(X_val)
+rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+print("\nFinal RMSE:", rmse)
+
+# Output best model and RMSE
+best_model, best_rmse = regr.get_best_model()
+print("\nBest validation RMSE:", best_rmse)
+"""

sciml/pipelines.py

@@ -1,174 +1,435 @@
-import numpy as np
-import pandas as pd
-from scipy import stats
-from copy import deepcopy
-from tqdm import tqdm
-from sklearn.metrics import mean_squared_error
-from xgboost import XGBRegressor
-
-def get_metrics(df, truth = 'truth', pred = 'pred', return_dict = False):
-    '''
-    Calculate statistical measures between validation and prediction sequences
-    '''
-    df = df[[truth, pred]].copy().dropna()
-    slope, intercept, r_value, p_value, std_err = stats.linregress(df.dropna()[truth], df.dropna()[pred])
-    r2 = r_value**2
-    mse = mean_squared_error(df.dropna()[truth], df.dropna()[pred])
-    rmse = np.sqrt(mse)
-    mbe = np.mean(df.dropna()[pred] - df.dropna()[truth])
-    mae = (df.dropna()[pred] - df.dropna()[truth]).abs().mean()
-    if return_dict:
-        return pd.DataFrame.from_dict([{
-            'r2': r2, 
-            'Slope': slope, 
-            'RMSE': rmse, 
-            'MBE': mbe, 
-            'MAE': mae, 
-            'Intercept': intercept, 
-            'p-value': p_value, 
-            'std_err': std_err
-        }])
-    else:
-        return r2, slope, rmse, mbe, mae, intercept, p_value, std_err
-
-# ===============================================================================================================================
-# Machine learning algorithms
-def train_ml(
-    X_train, y_train, model_name = 'XGB', 
-    xgb_params_user = None, rfr_params_user = None, 
-    mlp_params_user = None, svr_params_user = None,
-    df21_params_user = None,
-    gpu = False, partial_mode = False
-    ):
-    # -------------------------------------------------------------------------
-    # Setup parameters:
-    if xgb_params_user:
-        xgb_params = xgb_params_user
-    else:
-        xgb_params = {
-            "objective": "reg:squarederror",
-            "random_state": 0,
-            'seed': 0,
-            'n_estimators': 100,
-            'max_depth': 6, 
-            'min_child_weight': 4, 
-            'subsample': 0.8, 
-            'colsample_bytree': 0.8, 
-            'gamma': 0, 
-            'reg_alpha': 0, 
-            'reg_lambda': 1,
-            'learning_rate': 0.05, 
-        }
-
-        xgb_gpu_params = {
-            'tree_method': 'gpu_hist',
-            'gpu_id': 0,
-            # "n_gpus": 2, 
-        }
-
-        if gpu: xgb_params.update(xgb_gpu_params)
-
-    if rfr_params_user:
-        rfr_params = rfr_params_user
-    else:
-        rfr_params = {
-            'max_depth': 20,
-            'min_samples_leaf': 3,
-            'min_samples_split': 12,
-            'n_estimators': 100,
-            'n_jobs': -1
-        }
-
-    if df21_params_user:
-        df21_params = df21_params_user
-    else:
-        df21_params = {
-            'random_state': 1, 
-            'verbose' : 0, 
-            'predictor': "xgboost", 
-            'n_jobs' : -1, 
-            'predictor_kwargs' : xgb_params, 
-            'partial_mode' : partial_mode
-        }
-    # -------------------------------------------------------------------------
-    # Run:
-    if model_name == "XGB":
-        from xgboost import XGBRegressor
-        regr = XGBRegressor(**xgb_params)
-    elif model_name == "MLP":
-        from sklearn.neural_network import MLPRegressor
-        regr = MLPRegressor(**mlp_params_user)
-    elif model_name == "RFR":
-        from sklearn.ensemble import RandomForestRegressor
-        regr = RandomForestRegressor(**rfr_params)
-    elif model_name == "SVR":
-        from sklearn.svm import SVR
-        regr = SVR(**svr_params_user)
-    elif model_name == "DF21":
-        from deepforest import CascadeForestRegressor
-        # https://deep-forest.readthedocs.io/en/latest/api_reference.html?highlight=CascadeForestRegressor#cascadeforestregressor
-        # predictor: {"forest", "xgboost", "lightgbm"}
-        # regr = CascadeForestRegressor(random_state = 1, verbose = 0, predictor = "xgboost", n_jobs = -1, predictor_kwargs  = xgb_params, partial_mode = partial_mode)
-        regr = CascadeForestRegressor(**df21_params)
-    regr.fit(X_train, y_train)
-    return regr
-
-def test_ml(X_test, y_test, regr):
-    res = y_test.copy() # y_test is 2D pandas dataframe.
-    res.columns = ['truth']
-    res['pred'] = regr.predict(X_test)
-    return res
-
-def run_ensemble(X_train, y_train, n_models = 10, frac_sample = 0.8):
-    base_params_xgb = {
-        "objective": "reg:squarederror",
-        'seed': 0,
-        "random_state": 0,
-    }
-    params_xgb = deepcopy(base_params_xgb)
-    # dropout-like regularization
-    params_xgb.update({
-        "subsample": 0.8,  # Use 80% of the data for each tree
-        "colsample_bytree": 0.8,  # Use 80% of the features for each tree
-    })
-
-    models = []
-    for i in tqdm(range(n_models)):
-        # Create a bootstrapped dataset
-        y_resampled = y_train.copy().sample(frac = frac_sample, random_state = i)
-        X_resampled = X_train.copy().loc[y_resampled.index]
-        # print(y_resampled.sort_index().index[0], y_resampled.sort_index().index[-1])
-
-        # Train the XGBoost model
-        params_xgb.update({'random_state': i})
-        model = XGBRegressor(**params_xgb)
-        model.fit(X_resampled, y_resampled)
-        models.append(model)
-    return models
-
-# ===============================================================================================================================
-# Deep learning neural networks
-
-try:
-    from tensorflow import keras
-    from tensorflow.keras import layers
-    from tensorflow.keras import models
-    # from keras.layers import Dropout
-    from keras.callbacks import EarlyStopping
-    from scitbx.stutils import *
-except Exception as e:
-    print(e)
-
-def train_lstm(X_train, y_train, nfeature, ntime, verbose = 2, epochs = 200, batch_size = 64):
-    # create and fit the LSTM network
-    model = models.Sequential()
-    model.add(layers.LSTM(64, input_shape=(nfeature, ntime)))
-    model.add(layers.Dropout(0.2))
-    model.add(layers.Dense(16, activation='relu'))
-    model.add(layers.Dropout(0.2))
-    model.add(layers.Dense(1, activation='relu'))
-    model.compile(loss='mean_squared_error', optimizer='adam')
-    # es = EarlyStopping(monitor='loss', mode='min', verbose=1)
-    # model.fit(X_train.reshape(-1, nsites, nfeats), y_train, epochs=100, batch_size=256, verbose=2, callbacks=[es])
-    model.fit(X_train, y_train, epochs = epochs, batch_size = batch_size, verbose=verbose)
-    return model
+import numpy as np
+import pandas as pd
+from scipy import stats
+from copy import deepcopy
+from tqdm import tqdm
+from sklearn.metrics import mean_squared_error
+from xgboost import XGBRegressor
+
+def get_metrics(df, truth = 'truth', pred = 'pred', return_dict = False):
+    '''
+    Calculate statistical measures between validation and prediction sequences
+    '''
+    df = df[[truth, pred]].copy().dropna()
+    slope, intercept, r_value, p_value, std_err = stats.linregress(df.dropna()[truth], df.dropna()[pred])
+    r2 = r_value**2
+    mse = mean_squared_error(df.dropna()[truth], df.dropna()[pred])
+    rmse = np.sqrt(mse)
+    mbe = np.mean(df.dropna()[pred] - df.dropna()[truth])
+    mae = (df.dropna()[pred] - df.dropna()[truth]).abs().mean()
+    if return_dict:
+        return pd.DataFrame.from_dict([{
+            'r2': r2, 
+            'Slope': slope, 
+            'RMSE': rmse, 
+            'MBE': mbe, 
+            'MAE': mae, 
+            'Intercept': intercept, 
+            'p-value': p_value, 
+            'std_err': std_err
+        }])
+    else:
+        return r2, slope, rmse, mbe, mae, intercept, p_value, std_err
+
+# ===============================================================================================================================
+# Machine learning algorithms
+def train_ml(
+    X_train, y_train, model_name = 'XGB', 
+    xgb_params_user = None, rfr_params_user = None, 
+    mlp_params_user = None, svr_params_user = None,
+    df21_params_user = None,
+    gpu = False, partial_mode = False
+    ):
+    # -------------------------------------------------------------------------
+    # Setup parameters:
+    if xgb_params_user:
+        xgb_params = xgb_params_user
+    else:
+        xgb_params = {
+            "objective": "reg:squarederror",
+            "random_state": 0,
+            'seed': 0,
+            'n_estimators': 100,
+            'max_depth': 6, 
+            'min_child_weight': 4, 
+            'subsample': 0.8, 
+            'colsample_bytree': 0.8, 
+            'gamma': 0, 
+            'reg_alpha': 0, 
+            'reg_lambda': 1,
+            'learning_rate': 0.05, 
+        }
+
+        xgb_gpu_params = {
+            'tree_method': 'gpu_hist',
+            'gpu_id': 0,
+            # "n_gpus": 2, 
+        }
+
+        if gpu: xgb_params.update(xgb_gpu_params)
+
+    if rfr_params_user:
+        rfr_params = rfr_params_user
+    else:
+        rfr_params = {
+            'max_depth': 20,
+            'min_samples_leaf': 3,
+            'min_samples_split': 12,
+            'n_estimators': 100,
+            'n_jobs': -1
+        }
+
+    if df21_params_user:
+        df21_params = df21_params_user
+    else:
+        df21_params = {
+            'random_state': 1, 
+            'verbose' : 0, 
+            'predictor': "xgboost", 
+            'n_jobs' : -1, 
+            'predictor_kwargs' : xgb_params, 
+            'partial_mode' : partial_mode
+        }
+    # -------------------------------------------------------------------------
+    # Run:
+    if model_name == "XGB":
+        from xgboost import XGBRegressor
+        regr = XGBRegressor(**xgb_params)
+    elif model_name == "MLP":
+        from sklearn.neural_network import MLPRegressor
+        regr = MLPRegressor(**mlp_params_user)
+    elif model_name == "RFR":
+        from sklearn.ensemble import RandomForestRegressor
+        regr = RandomForestRegressor(**rfr_params)
+    elif model_name == "SVR":
+        from sklearn.svm import SVR
+        regr = SVR(**svr_params_user)
+    elif model_name == "DF21":
+        from deepforest import CascadeForestRegressor
+        # https://deep-forest.readthedocs.io/en/latest/api_reference.html?highlight=CascadeForestRegressor#cascadeforestregressor
+        # predictor: {"forest", "xgboost", "lightgbm"}
+        # regr = CascadeForestRegressor(random_state = 1, verbose = 0, predictor = "xgboost", n_jobs = -1, predictor_kwargs  = xgb_params, partial_mode = partial_mode)
+        regr = CascadeForestRegressor(**df21_params)
+    regr.fit(X_train, y_train)
+    return regr
+
+def test_ml(X_test, y_test, regr):
+    res = y_test.copy() # y_test is 2D pandas dataframe.
+    res.columns = ['truth']
+    res['pred'] = regr.predict(X_test)
+    return res
+
+def run_ensemble(X_train, y_train, n_models = 10, frac_sample = 0.8):
+    base_params_xgb = {
+        "objective": "reg:squarederror",
+        'seed': 0,
+        "random_state": 0,
+    }
+    params_xgb = deepcopy(base_params_xgb)
+    # dropout-like regularization
+    params_xgb.update({
+        "subsample": 0.8,  # Use 80% of the data for each tree
+        "colsample_bytree": 0.8,  # Use 80% of the features for each tree
+    })
+
+    models = []
+    for i in tqdm(range(n_models)):
+        # Create a bootstrapped dataset
+        y_resampled = y_train.copy().sample(frac = frac_sample, random_state = i)
+        X_resampled = X_train.copy().loc[y_resampled.index]
+        # print(y_resampled.sort_index().index[0], y_resampled.sort_index().index[-1])
+
+        # Train the XGBoost model
+        params_xgb.update({'random_state': i})
+        model = XGBRegressor(**params_xgb)
+        model.fit(X_resampled, y_resampled)
+        models.append(model)
+    return models
+
+# ===============================================================================================================================
+# Deep learning neural networks
+
+try:
+    from tensorflow import keras
+    from tensorflow.keras import layers
+    from tensorflow.keras import models
+    # from keras.layers import Dropout
+    from keras.callbacks import EarlyStopping
+    from scitbx.stutils import *
+except Exception as e:
+    print(e)
+
+def train_lstm(X_train, y_train, nfeature, ntime, verbose = 2, epochs = 200, batch_size = 64):
+    # create and fit the LSTM network
+    model = models.Sequential()
+    model.add(layers.LSTM(64, input_shape=(nfeature, ntime)))
+    model.add(layers.Dropout(0.2))
+    model.add(layers.Dense(16, activation='relu'))
+    model.add(layers.Dropout(0.2))
+    model.add(layers.Dense(1, activation='relu'))
+    model.compile(loss='mean_squared_error', optimizer='adam')
+    # es = EarlyStopping(monitor='loss', mode='min', verbose=1)
+    # model.fit(X_train.reshape(-1, nsites, nfeats), y_train, epochs=100, batch_size=256, verbose=2, callbacks=[es])
+    model.fit(X_train, y_train, epochs = epochs, batch_size = batch_size, verbose=verbose)
+    return model
+
+
+'''
+# ========================================================================================================
+import numpy as np
+from xgboost import XGBRegressor
+from sklearn.metrics import mean_squared_error
+
+class XGBoostDeepForestRegressor:
+    def __init__(self, n_estimators_per_layer=2, max_layers=20, early_stopping_rounds=2):
+        self.n_estimators_per_layer = n_estimators_per_layer
+        self.max_layers = max_layers
+        self.early_stopping_rounds = early_stopping_rounds
+        self.layers = []
+
+    def _fit_layer(self, X, y):
+        layer = []
+        layer_outputs = []
+        for _ in range(self.n_estimators_per_layer):
+            reg = XGBRegressor()
+            reg.fit(X, y)
+            preds = reg.predict(X).reshape(-1, 1)
+            layer.append(reg)
+            layer_outputs.append(preds)
+        output = np.hstack(layer_outputs)
+        return layer, output
+
+    def fit(self, X, y, X_val=None, y_val=None):
+        X_current = X.copy()
+        best_rmse = float("inf")
+        no_improve_rounds = 0
+
+        for layer_index in range(self.max_layers):
+            print(f"Training Layer {layer_index + 1}")
+            layer, output = self._fit_layer(X_current, y)
+            self.layers.append(layer)
+            X_current = np.hstack([X_current, output])
+
+            if X_val is not None:
+                y_pred = self.predict(X_val)
+                # rmse = mean_squared_error(y_val, y_pred, squared=False)
+                rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+                print(f"Validation RMSE: {rmse:.4f}")
+
+                if rmse < best_rmse:
+                    best_rmse = rmse
+                    no_improve_rounds = 0
+                else:
+                    no_improve_rounds += 1
+                    if no_improve_rounds >= self.early_stopping_rounds:
+                        print("Early stopping triggered.")
+                        break
+
+    def predict(self, X):
+        X_current = X.copy()
+        for layer in self.layers:
+            layer_outputs = []
+            for reg in layer:
+                n_features = reg.n_features_in_
+                preds = reg.predict(X_current[:, :n_features]).reshape(-1, 1)
+                layer_outputs.append(preds)
+            output = np.hstack(layer_outputs)
+            X_current = np.hstack([X_current, output])
+
+        # Final prediction = average of last layer regressors
+        final_outputs = []
+        for reg in self.layers[-1]:
+            n_features = reg.n_features_in_
+            final_outputs.append(reg.predict(X_current[:, :n_features]).reshape(-1, 1))
+        return np.mean(np.hstack(final_outputs), axis=1)
+
+
+from sklearn.datasets import load_diabetes
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
+
+X, y = load_diabetes(return_X_y=True)
+X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)
+
+df_reg = XGBoostDeepForestRegressor(n_estimators_per_layer=2, max_layers=5)
+df_reg.fit(X_train, y_train, X_val, y_val)
+
+y_pred = df_reg.predict(X_val)
+# rmse = mean_squared_error(y_val, y_pred, squared=False)
+rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+print("Final RMSE:", rmse)
+
+# ----------------------------------------------------------------------------------------------------
+
+import numpy as np
+from xgboost import XGBRegressor
+from sklearn.metrics import mean_squared_error
+import itertools
+
+class XGBoostDeepForestRegressor:
+    def __init__(self, n_estimators_per_layer=2, max_layers=20, early_stopping_rounds=2, param_grid=None, use_gpu=True, gpu_id=0):
+        self.n_estimators_per_layer = n_estimators_per_layer
+        self.max_layers = max_layers
+        self.early_stopping_rounds = early_stopping_rounds
+        self.param_grid = param_grid or {
+            'max_depth': [3],
+            'learning_rate': [0.1],
+            'n_estimators': [100]
+        }
+        self.use_gpu = use_gpu
+        self.gpu_id = gpu_id
+        self.layers = []
+
+    def _get_param_combinations(self):
+        keys, values = zip(*self.param_grid.items())
+        return [dict(zip(keys, v)) for v in itertools.product(*values)]
+
+    def _fit_layer(self, X, y, X_val=None, y_val=None):
+        layer = []
+        layer_outputs = []
+        param_combos = self._get_param_combinations()
+
+        for i in range(self.n_estimators_per_layer):
+            best_rmse = float('inf')
+            best_model = None
+
+            for params in param_combos:
+                # Set GPU support parameters in XGBRegressor
+                if self.use_gpu:
+                    params['tree_method'] = 'hist'  # Use hist method
+                    params['device'] = 'cuda'  # Enable CUDA for GPU
+
+                model = XGBRegressor(**params)
+                model.fit(X, y)
+
+                if X_val is not None:
+                    preds_val = model.predict(X_val)
+                    rmse = np.sqrt(mean_squared_error(y_val, preds_val))
+                    if rmse < best_rmse:
+                        best_rmse = rmse
+                        best_model = model
+                else:
+                    best_model = model
+
+            final_model = best_model
+            preds = final_model.predict(X).reshape(-1, 1)
+            layer.append(final_model)
+            layer_outputs.append(preds)
+
+        output = np.hstack(layer_outputs)
+        return layer, output
+
+    def fit(self, X, y, X_val=None, y_val=None):
+        X_current = X.copy()
+        X_val_current = X_val.copy() if X_val is not None else None
+
+        best_rmse = float("inf")
+        no_improve_rounds = 0
+
+        for layer_index in range(self.max_layers):
+            print(f"Training Layer {layer_index + 1}")
+            layer, output = self._fit_layer(X_current, y, X_val_current, y_val)
+            self.layers.append(layer)
+            X_current = np.hstack([X_current, output])
+
+            if X_val is not None:
+                val_outputs = []
+                for reg in layer:
+                    n_features = reg.n_features_in_
+                    preds = reg.predict(X_val_current[:, :n_features]).reshape(-1, 1)
+                    val_outputs.append(preds)
+                val_output = np.hstack(val_outputs)
+                X_val_current = np.hstack([X_val_current, val_output])
+
+                y_pred = self.predict(X_val)
+                rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+                print(f"Validation RMSE: {rmse:.4f}")
+
+                if rmse < best_rmse:
+                    best_rmse = rmse
+                    no_improve_rounds = 0
+                else:
+                    no_improve_rounds += 1
+                    if no_improve_rounds >= self.early_stopping_rounds:
+                        print("Early stopping triggered.")
+                        break
+
+    def predict(self, X):
+        X_current = X.copy()
+        for layer in self.layers:
+            layer_outputs = []
+            for reg in layer:
+                n_features = reg.n_features_in_
+                preds = reg.predict(X_current[:, :n_features]).reshape(-1, 1)
+                layer_outputs.append(preds)
+            output = np.hstack(layer_outputs)
+            X_current = np.hstack([X_current, output])
+
+        final_outputs = []
+        for reg in self.layers[-1]:
+            n_features = reg.n_features_in_
+            final_outputs.append(reg.predict(X_current[:, :n_features]).reshape(-1, 1))
+        return np.mean(np.hstack(final_outputs), axis=1)
+
+
+from sklearn.datasets import load_diabetes
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
+
+# Load dataset
+X, y = load_diabetes(return_X_y=True)
+X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)
+
+# Hyperparameter grid
+param_grid = {
+    'max_depth': [3, 4],
+    'learning_rate': [0.1, 0.05],
+    'n_estimators': [50, 100]
+}
+
+# Create and fit the model with GPU enabled
+df_reg = XGBoostDeepForestRegressor(
+    n_estimators_per_layer=2,
+    max_layers=5,
+    early_stopping_rounds=2,
+    param_grid=param_grid,
+    use_gpu=True,  # Enable GPU acceleration
+    gpu_id=0  # Default to the first GPU
+)
+
+df_reg.fit(X_train, y_train, X_val, y_val)
+
+# Final evaluation
+y_pred = df_reg.predict(X_val)
+rmse = np.sqrt(mean_squared_error(y_val, y_pred))
+print("Final RMSE:", rmse)
+
+# ----------------------------------------------------------------------------------------------------
+
+xgb_params = {
+    "objective": "reg:squarederror",
+    "random_state": 0,
+    'seed': 0,
+    'n_estimators': 100,
+    'max_depth': 6, 
+    'min_child_weight': 4, 
+    'subsample': 0.8, 
+    'colsample_bytree': 0.8, 
+    'gamma': 0, 
+    'reg_alpha': 0, 
+    'reg_lambda': 1,
+    'learning_rate': 0.05, 
+}
+
+from xgboost import XGBRegressor
+regr = XGBRegressor(**xgb_params)
+
+regr.fit(X_train, y_train)
+y_pred = regr.predict(X_val)
+
+
+from scipy import stats
+
+stats.linregress(y_val, y_pred)
+
+'''

sciml/utils.py

@@ -1,46 +1,46 @@
-import numpy as np
-import pandas as pd
-from sklearn.model_selection import ShuffleSplit
-from sklearn.model_selection import train_test_split
-
-# randomly select sites
-def random_select(ds, count, num, random_state = 0):
-    np.random.seed(random_state)
-    idxs = np.random.choice(np.delete(np.arange(len(ds)), count), num, replace = False)
-    return np.sort(idxs)
-
-def split(Xs, ys, return_index = False, test_size = 0.33, random_state = 42):
-    if return_index:
-        sss = ShuffleSplit(n_splits=1, test_size = test_size, random_state = random_state)
-        sss.get_n_splits(Xs, ys)
-        train_index, test_index = next(sss.split(Xs, ys)) 
-        return (train_index, test_index)
-    else:
-        X_train, X_test, y_train, y_test = train_test_split(
-            Xs, ys, 
-            test_size = test_size, 
-            random_state = random_state
-        )
-        return (X_train, X_test, y_train, y_test)
-    
-def split_cut(Xs, ys, test_ratio = 0.33):
-    assert ys.ndim == 2, 'ys must be 2D!'
-    assert len(Xs) == len(ys), 'Xs and ys should be equally long!'
-    assert type(Xs) == type(ys), 'Xs and ys should be the same data type!'
-    if not type(Xs) in [pd.core.frame.DataFrame, np.ndarray]: raise Exception('Only accept numpy ndarray or pandas dataframe')
-    anchor = int(np.floor(len(ys) * (1 - test_ratio)))
-
-    if type(Xs) == pd.core.frame.DataFrame:
-        X_train = Xs.iloc[0: anchor, :]
-        X_test = Xs.iloc[anchor::, :]
-        y_train = ys.iloc[0: anchor, :]
-        y_test = ys.iloc[anchor::, :]
-    else:
-        X_train = Xs[0: anchor, :]
-        X_test = Xs[anchor::, :]
-        y_train = ys[0: anchor, :]
-        y_test = ys[anchor::, :]
-
-    assert len(X_train) + len(X_test) == len(Xs), 'The sum of train and test lengths must equal to Xs/ys!'
-
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import ShuffleSplit
+from sklearn.model_selection import train_test_split
+
+# randomly select sites
+def random_select(ds, count, num, random_state = 0):
+    np.random.seed(random_state)
+    idxs = np.random.choice(np.delete(np.arange(len(ds)), count), num, replace = False)
+    return np.sort(idxs)
+
+def split(Xs, ys, return_index = False, test_size = 0.33, random_state = 42):
+    if return_index:
+        sss = ShuffleSplit(n_splits=1, test_size = test_size, random_state = random_state)
+        sss.get_n_splits(Xs, ys)
+        train_index, test_index = next(sss.split(Xs, ys)) 
+        return (train_index, test_index)
+    else:
+        X_train, X_test, y_train, y_test = train_test_split(
+            Xs, ys, 
+            test_size = test_size, 
+            random_state = random_state
+        )
+        return (X_train, X_test, y_train, y_test)
+    
+def split_cut(Xs, ys, test_ratio = 0.33):
+    assert ys.ndim == 2, 'ys must be 2D!'
+    assert len(Xs) == len(ys), 'Xs and ys should be equally long!'
+    assert type(Xs) == type(ys), 'Xs and ys should be the same data type!'
+    if not type(Xs) in [pd.core.frame.DataFrame, np.ndarray]: raise Exception('Only accept numpy ndarray or pandas dataframe')
+    anchor = int(np.floor(len(ys) * (1 - test_ratio)))
+
+    if type(Xs) == pd.core.frame.DataFrame:
+        X_train = Xs.iloc[0: anchor, :]
+        X_test = Xs.iloc[anchor::, :]
+        y_train = ys.iloc[0: anchor, :]
+        y_test = ys.iloc[anchor::, :]
+    else:
+        X_train = Xs[0: anchor, :]
+        X_test = Xs[anchor::, :]
+        y_train = ys[0: anchor, :]
+        y_test = ys[anchor::, :]
+
+    assert len(X_train) + len(X_test) == len(Xs), 'The sum of train and test lengths must equal to Xs/ys!'
+
     return (X_train, X_test, y_train, y_test)

{sciml-0.0.7.dist-info → sciml-0.0.9.dist-info}/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2021 Zhu
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2021 Zhu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{sciml-0.0.7.dist-info → sciml-0.0.9.dist-info}/METADATA

@@ -1,13 +1,13 @@
-Metadata-Version: 2.1
-Name: sciml
-Version: 0.0.7
-Summary: draw and basic calculations/conversions
-Home-page: https://github.com/soonyenju/sciml
-Author: Songyan Zhu
-Author-email: zhusy93@gmail.com
-License: MIT Licence
-Keywords: Scientific machine learning wrappers
-Platform: any
-License-File: LICENSE
-
-coming soon
+Metadata-Version: 2.1
+Name: sciml
+Version: 0.0.9
+Summary: draw and basic calculations/conversions
+Home-page: https://github.com/soonyenju/sciml
+Author: Songyan Zhu
+Author-email: zhusy93@gmail.com
+License: MIT Licence
+Keywords: Scientific machine learning wrappers
+Platform: any
+License-File: LICENSE
+
+coming soon

sciml-0.0.9.dist-info/RECORD

@@ -0,0 +1,9 @@
+sciml/__init__.py,sha256=wtdlXERN2ik7NT_TQxFdd2gdodBY9vSU1ClSdeJnLm4,59
+sciml/models.py,sha256=BjbliW-KNfzbNdGNgM7nBdJ2SF2z21qCoAvug_v0FEg,10574
+sciml/pipelines.py,sha256=ReNEkQbdFn04D5G2tbxcA7jdSwACy8SnmZ8bFZI_oqE,15702
+sciml/utils.py,sha256=qCdABaTUu3K0R269jI7D_8SO6AqEjphg03CzdxCJR2k,1876
+sciml-0.0.9.dist-info/LICENSE,sha256=hcunSTJmVgRcUNOa1rKl8axtY3Jsy2B4wXDYtQsrAt0,1081
+sciml-0.0.9.dist-info/METADATA,sha256=S5hG3pP3x4yDPe8AJOKn4R-fIuvL-DL1GSKqGqiImSw,326
+sciml-0.0.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sciml-0.0.9.dist-info/top_level.txt,sha256=dS_7aBCZFKQE3myPy5sh4USjQZCZyGg382-YxUUYcdw,6
+sciml-0.0.9.dist-info/RECORD,,

{sciml-0.0.7.dist-info → sciml-0.0.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

sciml-0.0.7.dist-info/RECORD

@@ -1,8 +0,0 @@
-sciml/__init__.py,sha256=Asqzx08kEOBLv_IRE20VlHxZu9XgydyrzIMUDRE-qiU,48
-sciml/pipelines.py,sha256=5qfeHdxGhF-GMu-rTiInPv5metXiT32uSENIDFd2Ths,6333
-sciml/utils.py,sha256=u5DzQJV4aCZ-p7sY56Fxzj8WDGYOgn1rOTeGzAw0vwY,1831
-sciml-0.0.7.dist-info/LICENSE,sha256=dX4jBmkgQPWc_TfYkXtKQzVIgZQWFuHZ8vQjV4sEeV4,1060
-sciml-0.0.7.dist-info/METADATA,sha256=363EbWoSVqR9qAdhOfeVD8RiP6DfcalvDiZECJ6LW3s,313
-sciml-0.0.7.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-sciml-0.0.7.dist-info/top_level.txt,sha256=dS_7aBCZFKQE3myPy5sh4USjQZCZyGg382-YxUUYcdw,6
-sciml-0.0.7.dist-info/RECORD,,