tank-model 3.0.0b3__py3-none-any.whl

tank_core/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+'''
+Tank Hydrologic Model
+=====================
+Python implementation of Tank Hydrologic Model. 
+Tank-Model is a conceptual rainfall-runoff model 
+proposed by Sugawara and Funiyuki (1956).
+'''
+__version__ = '3.0.0b3'
+__author__ = 'Nazmul Ahasan <nzahasan@gmail.com>'

tank_core/arima.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+'''
+Time series error Correction Module for model
+---------------------------------------------
+ARIMA(p,d,q)
+    p - auto-regressive part
+    d - integrated part / differentiation part
+    q - moving average part
+
+'''
+import pickle
+import warnings
+import numpy as np
+from statsmodels.tsa.arima_model import ARIMA
+from statsmodels.tsa.stattools import adfuller
+warnings.filterwarnings("ignore") 
+
+
+class autoARIMA(object):
+
+    '''
+    A wrapper of statsmodels, ARIMA for easier model fitting and generating forecast
+    this fits ARIMA model using brute force with lowest BIC(Bayesian Information Criteria) value.
+      - Not the best way but its the easiest
+    
+    Possible alternative to look at - pyramid-arima
+    '''
+
+    def __init__(self,endog,max_p=5,max_d=5,max_q=5,helpText=True):
+        
+        self.endog        = endog
+        self.max_p        = max_p
+        self.max_d        = max_d
+        self.max_q        = max_q
+        self.helpText     = helpText
+        self.fitted_model = None
+
+
+    def getOrder(self):
+        
+        fittedOrder = {'order':[],'bic':[]}
+
+        # iterate through (p,d,q) values
+        for p in range(self.max_p):
+            for d in range(self.max_d):
+                for q in range(self.max_q):
+                    try:
+                        model = ARIMA(self.endog, order=(p,d,q)).fit(disp=0)
+                        fittedOrder['bic'].append( model.bic )
+                        fittedOrder['order'].append( (p,d,q) )
+                    except: 
+                        continue
+        
+        # find order with lowest bic value 
+        bestOrder = fittedOrder['order'][ fittedOrder['bic'].index( min(fittedOrder['bic']) ) ]
+        
+        if self.helpText == True:
+            print('Lowest BIC value with order ',bestOrder)
+        
+        return bestOrder
+
+    def fit(self):
+        
+        # return a fitted ARIMA model with lowest bic value
+        
+        self.fitted_model = ARIMA(self.endog,order=self.getOrder()).fit(disp=0)
+        
+        return self
+
+    
+    def forecast(self,num_step):
+        
+        # returns forecasted values and confidence limit of the forecast
+
+        if self.fitted_model == None:
+            print('ERROR: Fit the model first')
+            return None
+
+        forecast,_,confLimit = self.fitted_model.forecast(steps=num_step)
+        return (forecast,confLimit)
+
+    def inSamplePlot(self):
+        
+        self.fitted_model.plot_predict()
+        return self
+
+    def saveModel(self,fileName):
+        with open(fileName,'wb') as outModelFile:
+            pickle.dump(self,outModelFile,pickle.HIGHEST_PROTOCOL)
+
+    def loadModel(self,fileName):
+        with open(fileName,'rb') as inModelFile:
+            self = pickle.load(inModelFile)
+
+
+    
+
+    # N.B. predict() returns differentiated value by default
+    # use typ='levels' for prediction in endog level/scale 

tank_core/bias_correction.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+from scipy.stats import gamma
+
+class gqm():
+    
+    '''
+    Gamma Quantile Mapping (Piani et.el. 2009)
+    ------------------------------------------
+    Parametric bias correction using gamma distribution
+
+    cdf(obs) + C_obs = cdf(sim) + C_sim
+    cdf(obs) = cdf(sim) + C_sim - C_obs
+    cdf(obs) = cdf(sim) + delC
+     
+    >> obs = cdf^-1 [ cdf(sim) + delC ]
+    
+    here, C_sim/C_obs = fraction of dry days / days with 0 rainfall
+    
+    :: in gamma fit force location parameter to 0
+    '''
+    def __init__(self):
+
+        self.obs_param = dict(a = None, b = None, c = None)
+        self.sim_param = dict(a = None, b = None, c = None)
+     
+    def fit(self,obs_data,sim_data):
+
+        #: estimates parameters from provided data
+        #: dry day fraction
+        self.obs_param['c'] = (obs_data[obs_data==0].shape[0]) / (obs_data.shape[0])  
+        self.sim_param['c'] = (sim_data[sim_data==0].shape[0]) / (sim_data.shape[0])
+
+        #: fit gamma with non zero values with floc=0
+        self.obs_param['a'], _, self.obs_param['b'] = gamma.fit(obs_data[obs_data>0], floc=0)
+        self.sim_param['a'], _, self.sim_param['b'] = gamma.fit(sim_data[sim_data>0], floc=0)
+        
+        
+        return self
+
+    def set_params(self,obs_param_arr:np.ndarray, sim_param_arr:np.ndarray)->object:
+
+        self.obs_param['a'] = obs_param_arr[0]
+        self.obs_param['b'] = obs_param_arr[1]
+        self.obs_param['c'] = obs_param_arr[2]
+
+        self.sim_param['a'] = sim_param_arr[0]
+        self.sim_param['b'] = sim_param_arr[1]
+        self.sim_param['c'] = sim_param_arr[2]
+
+        return self
+
+    def correct(self,simVal:float)->float:
+        
+        delC = self.sim_param['c']-self.obs_param['c']
+        simValCDF = gamma.cdf(simVal,a=self.sim_param['a'],scale=self.sim_param['b'],loc=0)
+        
+        #: return the bias corrected value
+        return gamma.ppf( (simValCDF + delC),a=self.obs_param['a'], scale=self.obs_param['b'], loc=0)

tank_core/channel_routing.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+'''
+Channel routing methods
+'''
+
+import numpy as np 
+
+
+def muskingum(in_flow:np.ndarray, del_t:float, k:float, x:float) -> np.ndarray:
+
+    '''
+    Returns routed flow using Muskingum routing method
+    (Lumped channel routing)
+    
+    Units:
+    ------
+    in_flow  - m^3/s
+    del_t    - hr
+    x        - hr
+    '''
+
+    # calculate time-step
+    n_step:int = in_flow.shape[0]
+    
+    # create a zero array of out_flow
+    out_flow:np.ndarray = np.zeros(n_step, dtype=np.float64)
+
+    C0:float = (-k*x+0.5*del_t) / (k*(1-x)+0.5*del_t)
+    C1:float = (k*x+0.5 *del_t) / (k*(1-x)+0.5*del_t)
+    C2:float = (k*(1-x) - 0.5*del_t) / (k*(1-x)+0.5*del_t)
+
+    # constraints check
+    if (C0+C1+C2) > 1 or x >0.5 or (del_t/k + x) > 1:
+        print("WARNING-MUSKINGUM-01: violates k, x constraints")
+
+    # initial condition
+    out_flow[0] = in_flow[0]
+
+    for t in np.arange(1,n_step):
+
+        out_flow[t] = C0 * in_flow[t] + C1 * in_flow[t-1] + C2 * out_flow[t-1]
+
+    return out_flow
+

tank_core/computation_helpers.py

@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+'''
+Helper functions for computation
+'''
+import numpy as np
+import pandas as pd
+from queue import Queue
+from scipy.optimize import minimize
+from . import utils
+from .tank_basin import tank_discharge
+from .channel_routing import muskingum
+from .cost_functions import (PBIAS, R2, RMSE, NSE,)
+from . import global_config as gc
+
+
+
+def build_computation_stack(project:dict) -> list:
+
+    '''
+        Traverse project tree and build computation stack
+        First traverse the tree with queue(fifo) ass all the 
+        upstream node needs to be computed before computing
+        a specific node, while traversing node put them in stack
+
+        N.B. Upstream is child node.
+    '''
+    computation_stack = []
+
+    node_queue:Queue[str] = Queue()
+
+    # en-queue root note
+    for root_node in project['root_node']:
+        node_queue.put(root_node)
+
+    while not node_queue.empty():
+
+        # deque node
+        node = node_queue.get()
+
+        # add node to the top of computation stack
+        computation_stack.append(node)
+
+        # get child nodes and en-queue child nodes
+        if project['basin_def'][node].get('upstream',False):
+            child_nodes = project['basin_def'][node]['upstream']
+
+            for child in child_nodes:
+                node_queue.put(child)
+
+    return computation_stack
+
+
+def compute_project(
+        basin:dict, 
+        precipitation:pd.DataFrame, 
+        evapotranspiration:pd.DataFrame, 
+        del_t:float
+    )->tuple:
+    
+    '''
+    Computes project for provided precipitation and evapotranspiration data
+    
+    Note: 
+        - assumes all kind of time-step check has been completed
+        - and contains no null / missing data
+    '''
+    computation_stack = build_computation_stack(basin)
+    
+    n_step = len(precipitation.index)
+    
+    computation_result = pd.DataFrame(index=precipitation.index)
+    
+    model_states = dict(
+        time = precipitation.index.to_numpy()
+    )
+
+    while len(computation_stack) > 0:
+
+        # pop node from top of the node
+        curr_node_name = computation_stack.pop()
+
+        curr_node_def = basin['basin_def'][curr_node_name]
+
+        # if node is subbasin return tank discharge
+        if curr_node_def['type'] == 'Subbasin':
+            
+            computation_result[curr_node_name], basin_states = tank_discharge(
+                precipitation = precipitation[curr_node_name].to_numpy(),
+                evapotranspiration = evapotranspiration[curr_node_name].to_numpy(), 
+                del_t = del_t,
+                area = curr_node_def['area'],
+                ** curr_node_def['parameters']
+            )
+            # store basin states in  model states for dumping later
+            model_states[curr_node_name] = basin_states
+        
+        # if node is reach return sum of routed flow for each upstream node
+        elif curr_node_def['type'] == 'Reach':
+
+            sum_node = np.zeros(n_step, dtype=np.float64)
+            
+            for us_node_name in curr_node_def['upstream']:
+                sum_node += muskingum( 
+                    in_flow= computation_result[us_node_name].to_numpy(),
+                    del_t=del_t,
+                    ** curr_node_def['parameters']
+                )
+
+            computation_result[curr_node_name] = sum_node
+        
+        # if node is sink/junction return sum each upstream nodes
+        elif curr_node_def['type'] in ['Sink','Junction']:
+
+            sum_node = np.zeros(n_step, dtype=np.float64)
+            
+            for us_node_name in curr_node_def['upstream']:
+                sum_node += computation_result[us_node_name].to_numpy()
+            
+            computation_result[curr_node_name] = sum_node
+    
+    return computation_result, model_states
+
+
+def compute_statistics(basin:dict, result:pd.DataFrame, discharge:pd.DataFrame)->dict:
+
+    merged = merge_obs_sim(observed=discharge, simulated=result)
+
+    merged_keys = merged.keys()
+    
+    statistics = dict()
+    
+    for node in basin['basin_def'].keys():
+
+        obs_key, sim_key = f'{node}_obs', f'{node}_sim'
+
+        if obs_key in merged_keys and sim_key in merged_keys:
+            statistics[node]={
+                "RMSE": RMSE(merged[obs_key].to_numpy(), merged[sim_key].to_numpy()),
+                "NSE" : NSE(sim=merged[sim_key].to_numpy(), obs=merged[obs_key].to_numpy() ),
+                "R2"  : R2(merged[sim_key].to_numpy(), merged[obs_key].to_numpy() ),
+                "PBIAS"  : PBIAS(sim=merged[sim_key].to_numpy(), obs=merged[obs_key].to_numpy() )
+            }
+
+    return statistics
+
+
+
+# creates a single list of parameter stacking each nodes parameter
+def parameter_stack(basin:dict) -> tuple:
+    '''
+    Stacks all parameters into a list of a given basin
+    '''
+    node_order_type = []
+    stacked_parameter = []
+    basin_def = basin['basin_def']
+
+    for node in basin_def.keys():
+
+        node_type = basin_def[node]['type']
+        
+        if node_type in ['Subbasin', 'Reach']:
+            
+            if node_type == 'Subbasin':
+                stacked_parameter.extend(
+                    utils.tank_param_dict2list(basin_def[node]['parameters'])
+                )
+            
+            elif node_type == 'Reach':
+                stacked_parameter.extend(
+                    utils.muskingum_param_dict2list(basin_def[node]['parameters'])
+                )
+            # append to node order
+            node_order_type.append((node, node_type ))
+            
+                
+    # node order, stacked_parameter
+    return (node_order_type, stacked_parameter) 
+
+# Unstacks parameters stacked by parameter_stack function 
+def parameter_unstack(node_order_type:list, stacked_parameter:list) -> dict:
+    '''
+    returns unstacked parameters of a basin for provided unstacked parameters
+    '''
+    unstacked_parameter = dict()
+    
+    # later have to change this if other routing method is added
+    conv_fn = {
+        'Subbasin': utils.tank_param_list2dict,
+        'Reach': utils.muskingum_param_list2dict
+    }
+    offset = 0
+    for node, node_type  in node_order_type:
+
+        num_parameter = gc.NUM_PARAMETER[node_type]
+        unstacked_parameter[node] = conv_fn[node_type](stacked_parameter[offset:offset+num_parameter])
+        offset += num_parameter 
+
+        
+    return unstacked_parameter
+
+def update_basin_with_unstacked_parameter(basin:dict, unstacked_parameter:dict)->dict:
+    '''
+    returns updated basin for provided unstacked parameters
+    '''
+    for node in unstacked_parameter.keys():
+        
+        basin['basin_def'][node]['parameters'] = unstacked_parameter[node]
+
+
+    return basin
+
+def update_basin_with_stacked_parameter(basin:dict, node_order_type:list, stacked_parameter:list)->dict:
+    '''
+    returns updated basin for provided stacked parameters
+    '''
+    # check if stacked parameter length is okay
+    conv_fn = {
+        'Subbasin': utils.tank_param_list2dict,
+        'Reach': utils.muskingum_param_list2dict
+    }
+    offset = 0
+    for node, node_type  in node_order_type:
+
+        num_parameter = gc.NUM_PARAMETER[node_type]
+        basin['basin_def'][node]['parameters'] = conv_fn[node_type](stacked_parameter[offset:offset+num_parameter])
+        offset += num_parameter
+    
+
+    return basin
+
+def merge_obs_sim(observed:pd.DataFrame, simulated:pd.DataFrame) -> pd.DataFrame:
+    ''' 
+    Inner joins observed and simulated output with their index (time) 
+    ref: https://pandas.pydata.org/docs/user_guide/merging.html#database-style-dataframe-or-named-series-joining-merging
+    '''
+    return pd.merge(
+        simulated, observed, 
+        how='inner', 
+        left_index=True, 
+        right_index=True, 
+        suffixes=('_sim', '_obs')
+    )
+    
+
+def stat_by_stacked_parameter(
+        stacked_parameter:list, 
+        node_order_type:list, 
+        basin:dict,
+        rainfall:pd.DataFrame, 
+        evapotranspiration:pd.DataFrame, 
+        discharge:pd.DataFrame, 
+        del_t:float
+    )->float:
+    '''
+    Returns model performance statistics for stacked parameters
+    (right now set to nse only)
+    '''
+    updated_basin = update_basin_with_stacked_parameter(basin, node_order_type, stacked_parameter)
+    
+    result, _ = compute_project(updated_basin, rainfall, evapotranspiration, del_t)
+
+    merged = merge_obs_sim(observed=discharge, simulated=result)
+
+    #@need-fixing : fix for multiple root node
+    root_node = updated_basin['root_node'][0]
+
+    sim_key, obs_key = f'{root_node}_sim', f'{root_node}_obs' 
+
+    _nse = NSE(sim=merged[sim_key].to_numpy(), obs=merged[obs_key].to_numpy())
+    
+
+    return 1 - _nse
+
+
+
+def optimize_project(
+        basin:dict, 
+        precipitation:pd.DataFrame, 
+        evapotranspiration:pd.DataFrame, 
+        discharge:pd.DataFrame, 
+        del_t:float
+    )->dict:
+    '''
+    Optimizes parameters of a basin and returns updated basin file
+    '''
+
+    node_order_type, stacked_parameter = parameter_stack(basin)
+    
+    upper_bound_stacked = list()
+    lower_bound_stacked = list()
+    
+    for _, node_type in node_order_type:
+
+        if node_type == 'Subbasin':
+            
+            upper_bound_stacked.extend(gc.tank_ub.tolist())
+            lower_bound_stacked.extend(gc.tank_lb.tolist())
+
+        if node_type == 'Reach':
+            upper_bound_stacked.extend(gc.muskingum_ub.tolist())
+            lower_bound_stacked.extend(gc.muskingum_lb.tolist())
+    
+    initial_guess = np.array(stacked_parameter)
+
+    param_bounds = np.column_stack((lower_bound_stacked,upper_bound_stacked))
+    
+    optim_func_static_args = (node_order_type, basin,precipitation,evapotranspiration, discharge, del_t)
+    
+    optimizer = minimize(
+        fun = stat_by_stacked_parameter, 
+        x0 = initial_guess,
+        args = optim_func_static_args,
+        method = 'L-BFGS-B',
+        bounds = param_bounds
+    )
+    
+    return update_basin_with_stacked_parameter(basin, node_order_type, optimizer.x)
+

tank_core/cost_functions.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+'''
+Necessary cost function for model calibration and validation
+    R2:
+        Pearson’s correlation coefficient(R) squared
+        Range [0 to 1]
+        Higher is better maximum possible value =1
+    
+    NSE: 
+        Nash-Sutcliffe Efficiency coefficient
+        Range [-inf to 1]
+        NSE value 1 means a perfect model
+        NSE > 0.6 is considered as a good model.
+    
+    MSE: 
+        Mean Squared Error
+        Lower is better
+    
+    RMSE:    
+        Root Mean Squared Error
+        squared root of MSE
+        Lower is better
+    
+    PBIAS:
+        Percent bias
+        Lower is better 
+        +ve values indicate underestimation
+        -ve values indicate model overestimation
+    KGE:
+        Kling-Gupta efficiency
+        KGE = 1 indicates perfect agreement
+        KGE -ve is bad model
+'''
+
+import numpy as np
+from scipy.stats import pearsonr
+from .utils import shape_alike
+
+def get_clean_pairs(sim: np.ndarray, obs: np.ndarray, min_valid: int = 2):
+    
+    mask = ~(np.isnan(sim) | np.isnan(obs))
+
+    if mask.sum() < min_valid:
+        raise ValueError(f'only {mask.sum()} valid (non-NaN) pairs, need >= {min_valid}')
+
+    return sim[mask], obs[mask]
+
+def R2(x:np.ndarray, y:np.ndarray)->float:
+    '''
+    Pearson correlation coefficient (R^2)
+    '''
+    # check & calculate sample shape
+    if not shape_alike(x,y):
+        raise Exception('shape mismatch between x and y')
+    
+    n = x.shape[0]
+
+    NU = (n * ((x*y).sum()) - (x.sum()) * (y.sum()))**2
+    DE = (n * ((x**2).sum()) - (x.sum())**2 ) * ( n * ((y**2).sum()) - (y.sum())**2 ) 
+
+    return NU/DE
+
+
+def NSE(sim:np.ndarray, obs:np.ndarray)->float:
+    '''
+    Nash Schutliff Efficiency (NSE) coefficient
+    '''
+    # N.B. sim and obs is not interchangeable for NSE
+
+    if not shape_alike(sim,obs):
+        raise Exception('shape mismatch between sim and obs')
+
+    obs_mean = obs.mean()
+
+    return 1 - (  np.square( obs - sim).sum() / np.square(obs-obs_mean).sum() )
+
+def MSE(x:np.ndarray, y:np.ndarray)->float:
+    '''
+    Mean squared error
+    '''
+    if not shape_alike(x,y): 
+        raise Exception('shape mismatch between x and y')
+
+    return ((x - y)**2).sum() / x.shape[0]
+
+def RMSE(x:np.ndarray, y:np.ndarray)->float:
+    '''
+    Root mean squared error = sqrt(mse)
+    '''
+    if not shape_alike(x,y):
+        raise Exception('shape mismatch between x and y')
+
+    return np.sqrt(MSE(x,y))
+
+def PBIAS(sim:np.ndarray, obs:np.ndarray)->float:
+    '''
+    Percentage Bias
+    '''
+    if not shape_alike(sim,obs):
+        raise Exception('shape mismatch between x and y')
+    
+    return (obs-sim).sum() * 100 / obs.sum()
+
+def KGE(sim:np.ndarray, obs:np.ndarray)->float:
+    '''
+    Kling-Gupta efficiency
+    '''
+    
+    if not shape_alike(sim,obs):
+        raise Exception('shape mismatch between sim and obs')
+    
+    eMean = (np.mean(sim) / np.mean(obs)) - 1 
+    eVar = (np.std(sim) / np.std(obs)) - 1 
+    eCor = pearsonr(sim, obs).statistic - 1 
+
+    return 1 - np.sqrt(eMean**2 + eVar**2 + eCor**2)