denflux 0.1.0__py3-none-any.whl

denflux/core.py

@@ -0,0 +1,812 @@
+# -*- coding: utf-8 -*-
+"""
+DenFlux Core Engine: A process-based approach to correct TDP baseline shifts.
+
+This engine integrates ecophysiological theory with Random Forest 
+machine learning to partition nocturnal sap flux, estimate physiological 
+parameters, and dynamically reconstruct the true zero-flow baseline.
+
+@author: Mianzhi Wang (mianzhi.wang@helsinki.fi)
+"""
+
+import warnings
+warnings.filterwarnings("ignore", category=RuntimeWarning)
+warnings.filterwarnings("ignore", category=UserWarning)
+warnings.filterwarnings("ignore", category=FutureWarning)
+import pandas as pd
+warnings.simplefilter(action="ignore")
+
+import re
+from sklearn.experimental import enable_iterative_imputer
+from sklearn.impute import IterativeImputer
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.model_selection import GroupShuffleSplit
+from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
+import statsmodels.api as sm
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import StandardScaler
+import numpy as np
+import seaborn as sns
+
+def process(df,
+            time_col='Time',
+            dayfirst=False,
+            date_format=None,
+            alpha=0.0119, 
+            beta=1.231, 
+            growstart=5, 
+            growend=9, 
+            time_interval=5, 
+            apply_rolling=True, 
+            rolling_window=30, 
+            night_threshold=10, 
+            Radiation="PAR", 
+            daycheck_start=10, 
+            daycheck_end=16,
+            deci_tree_T=200, depth_T=12, min_leaf_T=5, max_fea_T='sqrt', random_state_T=42,
+            deci_tree_R=200, depth_R=12, min_leaf_R=5, max_fea_R='sqrt', random_state_R=42,
+            expected_r2=0.3,  
+            diagnostic_plots=False):
+    
+    """
+    Parameters:
+    -----------
+    [Data & Time Configuration]
+    df : pandas.DataFrame
+        Input dataframe containing timestamps and required environmental/sap flow variables.
+    time_col : str, default='Time'
+        The exact name of the column containing timestamps.
+    dayfirst : bool, default=False
+        Set to True if dates are formatted as DD/MM/YYYY to prevent parsing ambiguity.
+    date_format : str, default=None
+        Explicit datetime format string (e.g., '%Y-%m-%d %H:%M:%S'). If None, pandas auto-detects.
+    time_interval : int or float, default=5
+        The temporal resolution of the raw data in minutes.
+
+    [Ecophysiological Configuration]
+    alpha : float, default=0.0119
+        Calibration parameter alpha for the Granier formula.
+    beta : float, default=1.231
+        Calibration parameter beta for the Granier formula.
+    growstart : int, default=5
+        The starting month of the growing season (1-12).
+    growend : int, default=9
+        The ending month of the growing season (1-12).
+
+    [Day/Night Boundary & Filtering]
+    Radiation : str, default='PAR'
+        Column name for solar radiation, used to dynamically determine day and night.
+    night_threshold : float or int, default=10
+        Radiation value below which the environment is mathematically considered "nighttime".
+    daycheck_start : int, default=10
+        Start hour (0-23) for daytime leakage checking.
+    daycheck_end : int, default=16
+        End hour (0-23) for daytime leakage checking.
+
+    [Signal Processing]
+    apply_rolling : bool, default=True
+        Whether to apply a rolling average to smooth dendrometer and sap flux data.
+    rolling_window : int or float, default=30 (Unit: min)
+        The window size for the rolling average in minutes.
+
+    [Machine Learning Hyperparameters (Random Forest)]
+    expected_r2 : float, default=0.3
+        Validation R-squared threshold. Triggers a warning if models perform below this.
+    deci_tree_T, deci_tree_R : int, default=200
+        Number of trees in the forest for Transpiration (T) and Rehydration (R) models.
+    depth_T, depth_R : int, default=12
+        Maximum depth of the individual regression estimators.
+    min_leaf_T, min_leaf_R : int, default=5
+        Minimum number of samples required to be at a leaf node.
+    max_fea_T, max_fea_R : str, default='sqrt'
+        Number of features to consider when looking for the best split.
+    random_state_T, random_state_R : int, default=42
+        Seed for the random number generator to ensure reproducibility.
+
+    [Diagnostics]
+    diagnostic_plots : bool, default=False
+        If True, displays matplotlib figures for ML performance and baseline interpolation 
+        to help troubleshoot data issues.
+
+    Returns:
+    --------
+    df : pandas.DataFrame
+        The fully processed dataframe enriched with corrected baseline and J values.
+    """
+    
+    df = df.copy()
+
+    resample_freq = f'{time_interval}min'
+    time_interval_s = time_interval * 60
+    max_gap = int(60 / time_interval)
+    rolling_window = int(rolling_window / time_interval)
+
+    print("\n" + "="*50)
+    print("INITIALIZING DENFLUX PROCESSING ENGINE...")
+    print("="*50)
+
+    # 1. set timestamp 
+    if time_col in df.columns:
+        if date_format:
+            # If the user specifies an absolute format, the parsing will strictly adhere to that format.
+            df['Time'] = pd.to_datetime(df[time_col], format=date_format, errors='coerce')
+        else:
+            # Otherwise, let Pandas parse automatically, but follow the user-provided dayfirst instructions.
+            df['Time'] = pd.to_datetime(df[time_col], dayfirst=dayfirst, errors='coerce')
+        
+        df.dropna(subset=['Time'], inplace=True)
+        df.set_index('Time', inplace=True)
+    elif not isinstance(df.index, pd.DatetimeIndex):
+        raise ValueError(f"The dataframe must have a column named '{time_col}' or a DatetimeIndex.")
+        
+    df = df[(df.index.month >= growstart) & (df.index.month <= growend)]
+    
+    # 2. get VPD (kPa)
+    def get_vpd(df, tair_col='Tair', rh_col='RH', vpd_col='VPD'):
+        df = df.copy()
+
+        # --- RH standardize ---
+        if rh_col not in df.columns:
+            raise KeyError(f"Missing column: {rh_col}")
+        
+        if df[rh_col].max() > 5:  #If the unit is kPa, it is impossible to have values greater than 5 kPa, so 5 is selected as the threshold.
+            df[rh_col] = df[rh_col] / 100.0  # convert to decimal
+
+        # --- VPD standardize ---
+        if vpd_col in df.columns:
+            if df[vpd_col].max() > 10:
+                df[vpd_col] = df[vpd_col] / 1000.0  # convert to kPa
+        else:
+            if tair_col not in df.columns:
+                raise KeyError(f"Missing column: {tair_col}")
+            
+            T = df[tair_col]
+            RH = df[rh_col]
+            es = 0.611 * np.exp((17.502 * T) / (T + 240.97))
+            df[vpd_col] = es * (1 - RH)  
+
+        return df
+
+    df = get_vpd(df)
+
+
+    #####
+
+    # 3. adjust time resolution 
+    df = df.resample(resample_freq).mean().interpolate(
+        method='linear',
+        limit=max_gap,  # average data with high resolution, Interpolate data with low resolution 
+        limit_direction='both'
+    )
+
+
+    # 4. calculate dden
+    # 4.1 find col start with 'den'
+    den_cols = [col for col in df.columns if col.startswith('den')]
+
+    for col in den_cols:
+        # get den number, e.g., den2 = 2
+        suffix = col.replace('den', '')
+        new_col = f'dden{suffix}'
+
+        # calculate difference
+        df[new_col] = df[col].diff() / time_interval_s
+
+        # remove original col
+        df.drop(columns=col, inplace=True)
+
+
+    df = df[(df.index.month >= growstart) & (df.index.month <= growend)]
+
+    # ---------------------------------------------------------
+    print("\n[Step 1/6] Validating Day/Night Split & Dendrometer Dynamics...")
+    print("  [INFO] Nocturnal stems are expected to expand (positive dden).")
+    print("         High negative ratios (>30%) may indicate improper night_threshold or daytime leakage.")
+    
+    dden_cols = [col for col in df.columns if col.startswith('dden') and col != 'dden']
+    for col in dden_cols:
+        dden = df[[col, Radiation]].copy()
+        dden = dden[dden[Radiation] < night_threshold]
+        dden = dden[dden[col] < night_threshold]  
+        
+        neg_ratio = (dden[col] < 0).mean()
+        tree_id = col.replace('dden', '')
+        
+        if neg_ratio > 0.3:
+            print(f"  [WARNING] Tree {tree_id:<4} : {neg_ratio:>5.1%} negative values")
+        else:
+            print(f"  [-] Tree {tree_id:<4}       : {neg_ratio:>5.1%} negative values")
+
+
+    # 5. rolling average to dden and SFD (optional)
+
+    # 5.1 get dden and SFD cols names
+    dden_sfd_cols = [col for col in df.columns if col.startswith("dden") or col.startswith("SFD")]
+
+    # 5.2 apply rolling average (centered)
+    if apply_rolling and rolling_window > 1:
+        df[dden_sfd_cols] = df[dden_sfd_cols].rolling(window=rolling_window, center=True, min_periods=1).mean()
+        
+
+    # 6. calculate NOY (night of year)
+    df['is_night'] = df[Radiation] < night_threshold
+    df['NOY'] = np.nan
+
+    # 6.1 get DOY (day of year)
+    df ['Time'] = df.index
+    df['DOY']=df['Time'].dt.strftime('%j').astype(int) 
+    df=df.drop(['Time'],axis=1)
+
+    # 6.2 get NOY (night of year)
+    current_noy = np.nan
+    in_night = False
+
+    for i in range(1, len(df)):
+        curr = df.iloc[i]
+
+        if (not in_night) and (curr[Radiation] < night_threshold) and (curr.name.hour >= 12):
+            current_noy = curr['DOY']
+            in_night = True
+            df.iat[i, df.columns.get_loc('NOY')] = current_noy
+        elif in_night and curr[Radiation] < night_threshold:
+            df.iat[i, df.columns.get_loc('NOY')] = current_noy
+        elif in_night and curr[Radiation] >= night_threshold:
+            in_night = False
+
+    # 6.3 delete auxiliary column
+    df=df.drop(['is_night'],axis=1)
+
+    # 6.4 keep DOY only if Radiation is higher than night_threshold
+    df['DOY'] = df['DOY'].where(df[Radiation] >= night_threshold)
+
+    # 6.5 get whole_day
+    df['whole_day'] = df['NOY'].combine_first(df['DOY'])
+
+    # 6.6 add year prefix
+    year = df.index.year
+    for col in ['NOY', 'DOY', 'whole_day']:
+        df[col] = year * 1000 + df[col].astype('Int64') 
+
+    # ==========================================================
+    # Daytime Leakage Filter
+    # ==========================================================
+    print("\n[Step 2/6] Running Internal Daytime Leakage Check...")
+    print("  [INFO] Checking if low-radiation daytime periods were incorrectly classified as night.")
+
+    df_temp_hour = df.index.hour
+    leaked_rows = df[(df['NOY'].notna()) & 
+                     (df_temp_hour >= daycheck_start) & 
+                     (df_temp_hour <= daycheck_end)]
+
+    if not leaked_rows.empty:
+        leak_counts_per_noy = leaked_rows.groupby('NOY').size()
+        threshold_rows = int(60 / time_interval)
+        bad_noys = leak_counts_per_noy[leak_counts_per_noy > threshold_rows].index.tolist()
+        
+        if bad_noys:
+            total_noys_before = df['NOY'].nunique()
+            df.loc[df['NOY'].isin(bad_noys), 'NOY'] = np.nan
+            total_noys_after = df['NOY'].nunique()
+            removed_count = total_noys_before - total_noys_after
+            
+            print(f"  [WARNING] Found daytime leakage!")
+            print(f"            - Corrupted NOYs (>{threshold_rows} points in daytime): {bad_noys}")
+            print(f"            - Action: Removed {removed_count} corrupted NOYs.")
+            print(f"            - Safe NOYs remaining: {total_noys_after}")
+        else:
+            print("  [-] Passed: Minor daytime leakage detected (under 60 mins). No nights removed.")
+    else:
+        print("  [-] Passed: No daytime leakage detected.")
+    # ==========================================================
+
+    # 7. calculate preliminary SFD, using dTmax of each day as baseline
+
+    # 7.1 find dTmax of each day as baseline
+    dT_cols = [col for col in df.columns if col.startswith('dT') and not col.startswith('dTmax')]
+
+    for col in dT_cols:
+        suffix = col.replace('dT', '')
+        new_col = f'dTmax{suffix}'
+        df[new_col] = df.groupby('whole_day')[col].transform('max')
+        
+    # 7.2 calculate preliminary SFD
+    for col in df.columns:
+        if col.startswith('dT') and not col.startswith('dTmax'):
+            suffix = col.replace('dT', '')  # e.g., '1' from 'dT1'
+            dT_col = col
+            dTmax_col = f'dTmax{suffix}'
+            sfd_col = f'SFD{suffix}'
+
+            if dTmax_col in df.columns:
+                with np.errstate(divide='ignore', invalid='ignore'):
+                    # customize alpha and beta
+                    df[sfd_col] = alpha * ((df[dTmax_col] - df[dT_col]) / df[dT_col]) ** beta
+                    df[sfd_col] = df[sfd_col].replace([np.inf, -np.inf], np.nan)
+                    
+        
+    # 8. multiple linear regression model to estimate gc and Frs
+    print("\n[Step 3/6] Building Night-by-Night Linear Models...")
+    print("  [INFO] Estimating physiological parameters (gt, Frs) to find baseline anchors.")
+    night = df[df['NOY'].notna()].copy()
+
+    # 8.1 read ddenX cols
+    dden_cols = [col for col in night.columns if col.startswith('dden')]
+
+    # 8.2 save results in dataframe
+    all_results = []
+
+    def get_outlier_mask(series, multiplier=3.0):
+        Q1 = series.quantile(0.25)
+        Q3 = series.quantile(0.75)
+        IQR = Q3 - Q1
+        lower_bound = Q1 - multiplier * IQR
+        upper_bound = Q3 + multiplier * IQR
+        return (series < lower_bound) | (series > upper_bound)
+
+    for dden_col in dden_cols:
+        suffix = dden_col.replace('dden', '')  
+        sfd_col = f'SFD{suffix}'
+
+        if sfd_col not in night.columns:
+            continue  
+
+        x_cols = ['VPD', dden_col]
+        y_col = sfd_col
+
+        # 8.3 remove NaN
+        df_fit = night.dropna(subset=[*x_cols, y_col]).copy()
+        df_fit['intercept'] = 1.0
+        
+        # 8.4 group by NOY
+        grouped = df_fit.groupby('NOY')
+        results = []
+
+        for name, group in grouped:
+            try:
+                model = sm.OLS(group[y_col], group[x_cols + ['intercept']])
+                result = model.fit()
+
+                results.append({
+                    'NOY': name,
+                    'SFD': sfd_col,
+                    'dden': dden_col,
+                    'int': result.params['intercept'],
+                    'Frs': result.params[dden_col],
+                    'gc': result.params['VPD'],
+                    'p_int': result.pvalues['intercept'],
+                    'p_Frs': result.pvalues[dden_col],
+                    'p_gc': result.pvalues['VPD'],
+                    'R2': result.rsquared,
+                    'R2_adj': result.rsquared_adj,
+                    'AIC': result.aic
+                })
+            except Exception as e:
+                print(f"Skipping NOY={name} for {sfd_col}: {e}")
+                continue
+
+        result_df = pd.DataFrame(results)
+
+        if result_df.empty:
+            continue
+        
+        required_cols = ['p_Frs', 'p_gc', 'Frs', 'gc', 'R2_adj']
+        if not all(col in result_df.columns for col in required_cols):
+            print(f"WARNING: Incomplete regression results for {sfd_col}. Missing required columns. Skipping.")
+            continue
+
+        # 8.5 filter good nights
+        result_df['flag'] = (
+            (result_df['p_Frs'] < 0.05) &
+            (result_df['p_gc'] < 0.05) &
+            (result_df['Frs'] > 0) &
+            (result_df['gc'] > 0) &
+            (result_df['R2_adj'] > 0.5)
+        ).astype(int)
+
+        # 8.6 print good night
+        good_rate = (result_df['flag'] == 1).mean()
+        print(f"  [-] Tree {suffix:<4}       : Valid nights for linear model: {good_rate:>5.1%}")
+
+        # 8.7 save only good ones
+        result_df = result_df[result_df['flag'] == 1]
+        
+        if result_df.empty:
+            print(f"  [WARNING] Tree {suffix:<4} : No valid nights found. Skipping this tree entirely.")
+            continue
+
+        all_results.append(result_df)
+        
+        # 9. calculate T and R 
+        night[f'gc{suffix}'] = night['NOY'].map(result_df.set_index('NOY')['gc'])
+        night[f'Frs{suffix}'] = night['NOY'].map(result_df.set_index('NOY')['Frs'])
+
+        # 9.2 calculate
+        night[f'T{suffix}'] = night['VPD'] * night[f'gc{suffix}']
+        night[f'R{suffix}'] = night[dden_col] * night[f'Frs{suffix}']
+
+        # 9.3 outliers cleaning
+        mask_T = get_outlier_mask(night[f'T{suffix}'], multiplier=3.0)
+        mask_R = get_outlier_mask(night[f'R{suffix}'], multiplier=3.0)
+
+        outlier_mask = mask_T | mask_R
+        outliers_count = outlier_mask.sum()
+
+        if outliers_count > 0:
+            print(f"Tree {suffix}: Found {outliers_count} extreme outlier rows in T or R. Replacing with NaN for RF imputation.")
+            night.loc[outlier_mask, [f'T{suffix}', f'R{suffix}']] = np.nan
+
+    if all_results:
+       final_results = pd.concat(all_results, ignore_index=True)
+
+    # 9.4 After exiting the for loop, delete all model parameter columns at once
+    night.drop(columns=[col for col in night.columns if col.startswith(('gc', 'Frs', 'intercept', 'DOY', 'whole_day'))], errors='ignore', inplace=True)
+
+    # 10. Random Forest (train independently by sample number)
+
+    # Define the plotting function
+    def plot_academic_performance(df_plot, target_name):
+        if not diagnostic_plots:
+            return 
+        sns.set_theme(style="ticks", context="paper", font_scale=1.2)
+        fig, axes = plt.subplots(1, 2, figsize=(12, 5.5))
+        
+        palette_colors = {'Training': '#4C72B0', 'Validation': '#C44E52'}
+        markers_style = {'Training': 'o', 'Validation': '^'} 
+
+        sns.scatterplot(
+            data=df_plot, x='Actual', y='Predicted', 
+            hue='Split', style='Split',
+            palette=palette_colors, markers=markers_style,
+            alpha=0.7, edgecolor='w', s=60, ax=axes[0] 
+        )
+        
+        min_val = min(df_plot['Actual'].min(), df_plot['Predicted'].min())
+        max_val = max(df_plot['Actual'].max(), df_plot['Predicted'].max())
+        
+        axes[0].plot([min_val, max_val], [min_val, max_val], 
+                     color='black', linestyle='--', linewidth=1.5, label='1:1 Line')
+        
+        axes[0].set_title(f'1:1 Scatter Plot for {target_name}', fontweight='bold')
+        axes[0].set_xlabel('Actual Value')
+        axes[0].set_ylabel('Predicted Value')
+        axes[0].legend(frameon=True, loc='best')
+
+        sns.scatterplot(
+            data=df_plot, x='Predicted', y='Residual', 
+            hue='Split', style='Split',
+            palette=palette_colors, markers=markers_style,
+            alpha=0.7, edgecolor='w', s=60, ax=axes[1]
+        )
+        
+        axes[1].axhline(0, color='black', linestyle='--', linewidth=1.5, label='Zero Error')
+        
+        axes[1].set_title(f'Residual Plot for {target_name}', fontweight='bold')
+        axes[1].set_xlabel('Predicted Value')
+        axes[1].set_ylabel('Residual (Actual - Predicted)')
+        axes[1].legend(frameon=True, loc='best')
+
+        plt.tight_layout()
+        plt.show()
+
+    # ==========================================================
+    # Identify tree ID based on dden
+    # ==========================================================
+    orig_den_cols = [col for col in night.columns if re.match(r'^dden\d+$', col)]
+    sample_ids = [re.search(r'\d+', col).group() for col in orig_den_cols]
+
+    # save results
+    rf_results = []
+    all_plot_T = []
+    all_plot_R = []
+    
+    print("\n[Step 4/6] Training Machine Learning Models for Non-linear Nights...")
+
+    for sid in sample_ids:
+        T_col = f"T{sid}"
+        R_col = f"R{sid}"
+        dden_col = f"dden{sid}"
+        sfd_col = f"SFD{sid}"
+        
+        # ==========================================================
+        # If the tree fails completely during the OLS phase, the ML step is silently skipped.
+        # ==========================================================
+        if T_col not in night.columns or R_col not in night.columns:
+            print(f"  [WARNING] Tree {sid:<4} : Skipping ML because it was dropped in OLS step due to poor quality.")
+            continue
+        
+        exclude_keywords = [f'dden{i}' for i in sample_ids if i != sid] + \
+                           [f'dT{i}' for i in sample_ids if i != sid] + \
+                           [f'dTmax{i}' for i in sample_ids if i != sid] + \
+                           [f'SFD{i}' for i in sample_ids if i != sid] + \
+                           [f'T{i}' for i in sample_ids if i != sid] + \
+                           [f'R{i}' for i in sample_ids if i != sid]
+        
+        df_train = night.dropna(subset=[T_col, R_col]).copy()
+        df_predict = night[~(night[T_col].notna() & night[R_col].notna())].copy()
+        
+        df_train = df_train[[col for col in df_train.columns if not any(kw in col for kw in exclude_keywords)]]
+        df_predict = df_predict[[col for col in df_predict.columns if not any(kw in col for kw in exclude_keywords)]]
+        
+        y_train_T = df_train[T_col]
+        y_train_R = df_train[R_col]
+        
+        groups = df_train['NOY']
+        X_train_raw = df_train.drop(columns=[T_col, R_col, 'NOY'])
+        feature_cols = X_train_raw.columns
+
+        gss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
+        train_idx, val_idx = next(gss.split(X_train_raw, y_train_T, groups=groups))
+
+        X_train_split_raw = X_train_raw.iloc[train_idx]
+        X_val_split_raw = X_train_raw.iloc[val_idx]
+
+        y_train_T_split = y_train_T.iloc[train_idx]
+        y_val_T_split = y_train_T.iloc[val_idx]
+
+        y_train_R_split = y_train_R.iloc[train_idx]
+        y_val_R_split = y_train_R.iloc[val_idx]
+        
+        scaler = StandardScaler()
+        imputer = IterativeImputer(max_iter=10, random_state=42)
+
+        # Apply pd.DataFrame to prevent sklearn feature name warnings
+        X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train_split_raw), columns=feature_cols)
+        X_train_imputed_scaled = pd.DataFrame(imputer.fit_transform(X_train_scaled), columns=feature_cols)
+        X_train_split = pd.DataFrame(scaler.inverse_transform(X_train_imputed_scaled), columns=feature_cols)
+
+        X_val_scaled = pd.DataFrame(scaler.transform(X_val_split_raw), columns=feature_cols)
+        X_val_imputed_scaled = pd.DataFrame(imputer.transform(X_val_scaled), columns=feature_cols)
+        X_val_split = pd.DataFrame(scaler.inverse_transform(X_val_imputed_scaled), columns=feature_cols)
+
+        df_predict_raw = df_predict[feature_cols].copy()
+
+        # 10.4 build ML model for T
+        model_T = RandomForestRegressor(
+            n_estimators=deci_tree_T,
+            max_depth=depth_T,
+            min_samples_leaf=min_leaf_T,
+            max_features=max_fea_T,
+            random_state=random_state_T
+        )
+        model_T.fit(X_train_split, y_train_T_split)
+
+        importances_T = model_T.feature_importances_
+        if diagnostic_plots:
+            plt.barh(feature_cols, importances_T)
+            plt.xlabel("Feature Importance")
+            plt.title(f"Feature Importance for T{sid} (VPD/RH/Tair/SFD should rank high)")
+            plt.tight_layout()
+            plt.show()
+
+        y_train_pred_T = model_T.predict(X_train_split)
+        y_val_pred_T = model_T.predict(X_val_split)
+        r2_val_t = r2_score(y_val_T_split, y_val_pred_T)
+
+        # === Retrain Model T on 100% of labeled data before final prediction ===
+        # 1. Refit scaler and imputer on the FULL valid dataset (100%)
+        X_full_scaled = pd.DataFrame(scaler.fit_transform(X_train_raw), columns=feature_cols)
+        X_full_imputed_scaled = pd.DataFrame(imputer.fit_transform(X_full_scaled), columns=feature_cols)
+        X_full = pd.DataFrame(scaler.inverse_transform(X_full_imputed_scaled), columns=feature_cols)
+
+        # 2. IMPORTANT: Re-transform the prediction set using the new 100%-fitted preprocessors!
+        df_predict_scaled_new = pd.DataFrame(scaler.transform(df_predict_raw), columns=feature_cols)
+        df_predict_imputed_scaled_new = pd.DataFrame(imputer.transform(df_predict_scaled_new), columns=feature_cols)
+        df_predict_filled = pd.DataFrame(scaler.inverse_transform(df_predict_imputed_scaled_new), columns=feature_cols)
+
+        # 3. Retrain model on full dataset
+        model_T.fit(X_full, y_train_T)
+        
+        # 4. Predict with retrained model and correctly updated prediction features
+        df_predict[f'RF_T{sid}'] = model_T.predict(df_predict_filled)
+        # =====================================================================
+
+        df_plot_T_train = pd.DataFrame({'Actual': y_train_T_split, 'Predicted': y_train_pred_T, 'Split': 'Training', 'Tree_ID': sid})
+        df_plot_T_val = pd.DataFrame({'Actual': y_val_T_split, 'Predicted': y_val_pred_T, 'Split': 'Validation', 'Tree_ID': sid})
+        df_plot_T_current = pd.concat([df_plot_T_train, df_plot_T_val])
+        df_plot_T_current['Residual'] = df_plot_T_current['Actual'] - df_plot_T_current['Predicted']
+        
+        plot_academic_performance(df_plot_T_current, f"T{sid}")
+        
+        all_plot_T.append(df_plot_T_current)
+
+        # 10.5 build ML model for R
+        model_R = RandomForestRegressor(
+            n_estimators=deci_tree_R,
+            max_depth=depth_R,
+            min_samples_leaf=min_leaf_R,
+            max_features=max_fea_R,
+            random_state=random_state_R
+        )
+        model_R.fit(X_train_split, y_train_R_split)
+
+        importances_R = model_R.feature_importances_
+        if diagnostic_plots:
+            plt.barh(feature_cols, importances_R)
+            plt.xlabel("Feature Importance")
+            plt.title(f"Feature Importance for R{sid} (dden/SFD should rank high)")
+            plt.tight_layout()
+            plt.show()
+
+        y_train_pred_R = model_R.predict(X_train_split)
+        y_val_pred_R = model_R.predict(X_val_split)
+        r2_val_r = r2_score(y_val_R_split, y_val_pred_R)
+        
+        print(f"  [-] Tree {sid:<4}        : Model T (Val R²: {r2_val_t:.2f}) | Model R (Val R²: {r2_val_r:.2f})")
+        
+        
+        if r2_val_t < expected_r2 or r2_val_r < expected_r2:
+            print(f"      [WARNING] Validation R² is below the expected threshold ({expected_r2}). Please check data quality and treat results for this tree with caution.")
+
+        # === Retrain Model R on 100% of labeled data before final prediction ===
+        # Retrain on full dataset (X_full and df_predict_filled were already updated in the T step)
+        model_R.fit(X_full, y_train_R)
+        # Predict with retrained model
+        df_predict[f'RF_R{sid}'] = model_R.predict(df_predict_filled)
+        # =====================================================================
+
+        df_predict[f'RF_Jn{sid}'] = df_predict[f'RF_T{sid}'] + df_predict[f'RF_R{sid}']
+        df_train[f'Jn{sid}'] = df_train[T_col] + df_train[R_col]
+
+        df_plot_R_train = pd.DataFrame({'Actual': y_train_R_split, 'Predicted': y_train_pred_R, 'Split': 'Training', 'Tree_ID': sid})
+        df_plot_R_val = pd.DataFrame({'Actual': y_val_R_split, 'Predicted': y_val_pred_R, 'Split': 'Validation', 'Tree_ID': sid})
+        df_plot_R_current = pd.concat([df_plot_R_train, df_plot_R_val])
+        df_plot_R_current['Residual'] = df_plot_R_current['Actual'] - df_plot_R_current['Predicted']
+        
+        plot_academic_performance(df_plot_R_current, f"R{sid}")
+
+        all_plot_R.append(df_plot_R_current)
+
+        df_combined_sample = pd.concat([df_train, df_predict])
+        rf_results.append(df_combined_sample)
+
+    if not rf_results:
+            raise ValueError("All trees failed the OLS quality filter — no data available for machine learning. Check Night Threshold or data quality.")
+            
+    # concat dataframe for output
+    df_combined = pd.concat(rf_results, axis=1)
+    df_combined = df_combined.loc[:, ~df_combined.columns.duplicated()]
+
+    # ==========================================
+    # 10.6 Aggregate plot data for EXPORT only (No global plotting)
+    # ==========================================
+    # Combine all individual tree data into master DataFrames
+    df_plot_T = pd.concat(all_plot_T, ignore_index=True)
+    df_plot_R = pd.concat(all_plot_R, ignore_index=True)
+
+    # 11. correct sap flow shift
+
+    # 11.1 combine Jn and RF_Jn to build model between SFD and Jn at one time
+    # find tree index，e.g., 1, 2, 3（get from cols name）
+    suffixes = sorted(set(col.replace("Jn", "") for col in df_combined.columns if col.startswith("Jn") and not col.startswith("RF")))
+
+    for suffix in suffixes:
+        Jn_col = f"Jn{suffix}"
+        RF_Jn_col = f"RF_Jn{suffix}"
+        Jn_all_col = f"Jn_all{suffix}"
+        
+        if Jn_col in df_combined.columns and RF_Jn_col in df_combined.columns:
+            df_combined[Jn_all_col] = df_combined[Jn_col].combine_first(df_combined[RF_Jn_col])
+
+    # 11.2 build linear regression model to get intercept
+    shift_list = []
+
+    for suffix in suffixes:
+        sfd_col = f"SFD{suffix}"
+        jn_all_col = f"Jn_all{suffix}"
+        
+        if sfd_col not in df_combined.columns or jn_all_col not in df_combined.columns:
+            continue
+
+        for noy, group in df_combined.groupby('NOY'):
+            sub = group[[sfd_col, jn_all_col]].dropna()
+            if len(sub) < 3:
+                continue
+
+            X = sm.add_constant(sub[jn_all_col])
+            y = sub[sfd_col]
+
+            model = sm.OLS(y, X).fit()
+
+            shift_list.append({
+                'x': suffix,
+                'NOY': noy,
+                'slope': model.params[jn_all_col],
+                'int': model.params['const'],
+                'p_slope': model.pvalues[jn_all_col],
+                'p_int': model.pvalues['const'],
+                'R2': model.rsquared
+            })
+    if not shift_list:
+        raise ValueError("No valid nights found for shift correction. Check your data or parameters.")
+        
+    # convert into DataFrame
+    shift = pd.DataFrame(shift_list)
+
+    print("\n[Step 5/6] Deriving True Baseline Anchors...")
+    print("  [INFO] Identifying valid nights where baseline anchor points can be estimated.")
+    
+    # 11.3 filter out good night
+    shift['flag'] = (
+        (shift['p_slope'] < 0.05) &
+        (shift['p_int'] < 0.05) &
+        (shift['R2'] > 0.5)
+    ).astype(int)
+
+    for suffix in suffixes:
+        ratio = (shift[shift['x'] == suffix]['flag'] == 1).mean()
+        print(f"  [-] Tree {suffix:<4}       : Baseline anchor points estimated for {ratio:>5.1%} of nights")
+        
+    shift_good = shift[shift['flag'] == 1]
+
+    # 11.4 mapping int from shift_good into df
+    for suffix in suffixes:
+        shift_sub = shift_good[shift_good['x'] == suffix][['NOY', 'int']].dropna()
+        
+        if shift_sub.empty:
+            continue
+        
+        shift_dict = dict(zip(shift_sub['NOY'], shift_sub['int']))
+        df[f'int{suffix}'] = df['NOY'].map(shift_dict)
+
+    # 11.5 calculate new baseline at dTmax (when preliminary SFD == 0)
+    for suffix in suffixes:
+        int_col = f'int{suffix}'
+        dTmax_col = f'dTmax{suffix}'
+        sfd_col = f'SFD{suffix}'
+        baseline_col = f'Baseline{suffix}'
+
+        if all(col in df.columns for col in [int_col, dTmax_col, sfd_col]):
+            with np.errstate(invalid='ignore', divide='ignore'):
+                mask = df[sfd_col] == 0
+                baseline_val = ((-df[int_col] / alpha) ** (1 / beta) + 1) * df[dTmax_col]
+                df[baseline_col] = np.where(mask, baseline_val, np.nan)
+                
+    # 11.6 interpolate new baseline
+    # Calculate the maximum allowed number of interpolation points
+    max_interp_gap = int((7 * 24 * 60) / time_interval)  # 7 days in data points
+
+    for suffix in suffixes:
+        baseline_col = f'Baseline{suffix}'
+        if baseline_col not in df.columns:
+            continue
+
+        df[baseline_col] = df[baseline_col].interpolate(
+            method='linear',
+            limit=max_interp_gap,
+            limit_direction='both'
+        )
+        
+    # visualize baseline to evaluate
+    if diagnostic_plots:
+        for suffix in suffixes:
+            baseline_col = f'Baseline{suffix}'
+            if baseline_col not in df.columns:
+                continue
+
+            plt.figure(figsize=(10, 4))
+            plt.scatter(df.index, df[baseline_col], s=10, alpha=0.7, color='teal')
+            plt.title(f'Interpolated {baseline_col}')
+            plt.ylabel('Baseline')
+            plt.xlabel('Time')
+            plt.tight_layout()
+            plt.show()
+    
+    print("\n[Step 6/6] Reconstructing Continuous Baseline & Finalizing Sap Flux...")    
+    # 11.7 calculate shift-corrected SFD (J)
+    for suffix in suffixes:
+        baseline_col = f'Baseline{suffix}'
+        dT_col = f'dT{suffix}'
+        J_col = f'J{suffix}'
+        K_col = f'K{suffix}'
+        
+        if baseline_col in df.columns and dT_col in df.columns:
+            with np.errstate(divide='ignore', invalid='ignore'):
+                ratio = (df[baseline_col] - df[dT_col]) / df[dT_col]
+                df[K_col] = ratio.replace([np.inf, -np.inf], np.nan) # K
+                df[J_col] = alpha * (ratio ** beta) # sap flux density
+                df[J_col] = df[J_col].replace([np.inf, -np.inf], np.nan)
+
+    print("\n" + "="*50)
+    print("[SUCCESS] Processing Complete! Data is ready for export.")
+    print("="*50 + "\n")
+    return df    

denflux-0.1.0.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: denflux
+Version: 0.1.0
+Summary: A process-based approach to correct TDP baseline shifts.
+Author-email: Mianzhi Wang <mianzhi.wang@helsinki.fi>
+Project-URL: Homepage, https://github.com/Mianzhi-Wang/DenFlux
+Project-URL: Bug Tracker, https://github.com/Mianzhi-Wang/DenFlux/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: statsmodels>=0.13.0
+Requires-Dist: matplotlib>=3.4.0
+Requires-Dist: seaborn>=0.11.0
+Dynamic: license-file
+
+# DenFlux: A Process-Based Computational Framework for TDP Baseline Correction
+
+[![PyPI Version](https://img.shields.io/pypi/v/denflux.svg)](https://pypi.org/project/denflux/)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**DenFlux** is an open-source Python framework designed to dynamically reconstruct the true zero-flow baseline for Granier thermal dissipation probes (TDP). 
+
+Conventional zero-flow assumption methods often do not account for nocturnal stem rehydration, leading to systematic baseline shifts and the underestimation of sap flux density. By integrating ecophysiological theory with Random Forest (RF) machine learning, DenFlux corrects these shifts to ensure higher accuracy sap flow estimations.
+
+---
+
+## 1. Installation
+
+DenFlux is hosted on the official Python Package Index (PyPI). You can install it instantly via `pip`. This will also automatically install all required dependencies (e.g., `pandas`, `scikit-learn`).
+
+```bash
+pip install denflux
+```
+
+*Tip for upgrading:* To ensure you have the latest analytical algorithms and bug fixes, run `pip install denflux --upgrade`.
+
+## 2. Quick Start (Python API)
+
+DenFlux is built as a transparent, research-grade engine. It hides the complexity of machine learning hyperparameter tuning while keeping the ecological parameters fully accessible.
+
+```python
+import pandas as pd
+from denflux.core import process
+
+# 1. Load your raw dataset
+df_raw = pd.read_csv('data/example_data.csv')
+
+# 2. Run the core processing engine
+df_processed = process(
+    df=df_raw,
+    time_col='Time',          # Specify your timestamp column
+    dayfirst=True,            # Set to True if dates are DD/MM/YYYY
+    Radiation='PAR',          # Solar radiation column for day/night split
+    growstart=5,              # Start month of growing season
+    growend=9,                # End month of growing season
+    night_threshold=10,       # Radiation < 10 is considered night
+    diagnostic_plots=False    # Set to True for troubleshooting & ML evaluation only
+)
+
+# 3. Export the shift-corrected sap flow data
+df_processed.to_csv('data/corrected_sapflow.csv')
+```
+
+**Tip:** To see the full list of all 20+ configurable parameters (including Granier's calibration coefficients $\alpha/\beta$ and advanced RF hyperparameters), run `help(process)` in your Python console.
+
+## 3. Required Data Format
+
+To ensure the physical formulas and machine learning models correctly pair the sap flow sensors with their corresponding dendrometers, `denflux` expects specific column naming conventions:
+
+| Variable category | Expected column name | Example unit | Description |
+| :--- | :--- | :--- | :--- |
+| **Timestamp** | `Time` | `YYYY/MM/DD hh:mm:ss` | The column containing timestamps. *Date format strings can be auto-detected or explicitly configured in the engine*. |
+| **TDP Sensor Data** | `dT1`, `dT2`, `dT3`... | °C | The temperature ($\Delta T$) or voltage ($\Delta V$) difference from TDP sensors. **A numerical suffix is strictly required** to pair it with the corresponding dendrometer. *Specific units are flexible because they cancel out mathematically.* |
+| **Dendrometer Data** | `den1`, `den2`, `den3`... | μm | Cleaned stem radial displacement readings. **The numerical suffix must strictly match** its corresponding TDP sensor (e.g., `den1` pairs with `dT1`). *Specific units are flexible because they cancel out mathematically.*|
+| **VPD** | `VPD` | kPa | Vapor pressure deficit. If not provided, the engine will automatically calculate it using `Tair` and `RH`. |
+| **Air Temperature** | `Tair` | °C | Mandatory *only if* VPD is not provided. |
+| **Relative Humidity** | `RH` | % or decimal (0-1) | Mandatory *only if* VPD is not provided. |
+| **Radiation** | `PAR` | μmol m⁻² s⁻¹ | Solar radiation used to dynamically determine day/night periods based on a user-defined threshold. *Column name can be configured in the engine*. |
+
+*Note: You do not need to alter your raw data files. You can effortlessly rename your columns using `pandas.DataFrame.rename()` immediately after loading your dataset in Python.*
+
+## 4. Interactive Tutorial
+
+For a deep dive into the methodology, including how to visualize the reconstructed baseline, evaluate model performance, and extract specific output variables (such as J, K, and the corrected Baseline), please see our highly detailed interactive tutorial on GitHub:
+
+**[Open the DenFlux Tutorial (Jupyter Notebook)](https://github.com/Mianzhi-Wang/DenFlux/blob/main/DenFlux_Tutorial.ipynb)**
+
+## Bonus: Standalone Desktop App (GUI)
+
+Not a programmer? No problem.
+We provide a user-friendly, standalone desktop executable (.exe) for Windows users. It encapsulates the identical rigorous Python engine behind a modern graphical interface—complete with progress tracking, diagnostic warnings, and one-click CSV exporting.
+
+* **Download:** Go to the [GitHub Releases](https://github.com/Mianzhi-Wang/DenFlux/releases) tab to download the latest `DenFlux_GUI.zip`.
+* **Usage:** Simply unzip and run the executable. No Python installation or coding required.
+
+## 5. The Science: Core Analytical Framework
+
+The DenFlux pipeline processes data through three key phases to eliminate subjective baseline assumptions:
+
+1. **Component Partitioning (OLS Regression):**
+   Initial nocturnal sap flux (J_n0) is partitioned into transpiration (E, driven by VPD) and stem refilling (R, indicated by stem diameter changes), plus a constant intercept (c) representing the potential systematic sap flow shift. This multiple linear regression is applied night-by-night to extract valid physiological parameters.
+
+2. **Machine Learning Imputation:**
+   For statistically complex nights where linear assumptions are unmet, DenFlux deploys a **Random Forest Regressor** to estimate E and R based on learned environmental features.
+
+3. **True Baseline Reconstruction:**
+   By evaluating the night-specific sap flow difference (c'), DenFlux analytically derives the true unknown baseline mathematically linked to Granier's empirical coefficients ($\alpha$ and $\beta$). These calculated anchor points are then interpolated to generate a continuous baseline time series for the final sap flux (J) computation.
+
+## 6. Citation
+
+If you use DenFlux in your research, please cite our manuscript. A permanent DOI will be updated upon publication.
+
+> **[Wang M], et al. (In Review).** *[Paper Title]*. Submitted to [Journal Name].
+
+```bibtex
+@article{denflux_under_review,
+  title={[Paper Title]},
+  author={[Author 1 and Author 2 and Author 3]},
+  journal={Submitted to [Journal Name]},
+  year={2026},
+  note={Under Review}
+}
+```
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.

denflux-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+denflux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+denflux/core.py,sha256=dM3s6rSIkNhNkGAW8LzmAMbkRTEXnwbM20BNc17VKzw,34248
+denflux-0.1.0.dist-info/licenses/LICENSE,sha256=7ecumowriwWLps20WmGDa4f7A3dfQrog2rb9a1n4g1Y,1077
+denflux-0.1.0.dist-info/METADATA,sha256=tqfepZhBnmgwSVVSPcDfZtq1reS2toyC1HIk7mm7HOc,7885
+denflux-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+denflux-0.1.0.dist-info/top_level.txt,sha256=1nQ3h1dIQby9-OsJhzS7fj3YCG_KpuJeowmmvApAYAo,8
+denflux-0.1.0.dist-info/RECORD,,

denflux-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

denflux-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,10 @@
+MIT License
+
+Copyright (c) 2026 Mianzhi Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

denflux-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+denflux