modelflowib 2.73__py3-none-any.whl

modelBLfunk.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Mar  2 17:01:49 2018
+
+@author: hanseni
+
+Functions placed here are included in the Pyfs business language 
+
+"""
+from math import exp, log, sqrt, tanh, erf,isclose
+from numpy import transpose , array
+from scipy.stats import norm,lognorm
+from scipy.stats import gamma
+import inspect 
+
+try:
+    # raise ImportError("Simulating ImportError for numba.")
+
+    from numba import jit
+except ImportError:
+    print("Numba is not available. No worry")
+
+    def jit(*args, **kwargs):
+        def wrapper(func):
+            # print('jit called')
+            return func
+        return wrapper
+    
+@jit("f8(f8)",nopython=True)
+def logit_inverse(number):
+    ''' A function which returns the logit of a number 
+    
+    takes care of extreme values 
+    '''
+    if number > 100:
+        return 1.0
+    elif number < -100:
+        return 0.0
+    else: 
+        return 1/(1+exp(-number))    
+    
+classfunk = []
+try:
+    from cvxopt import matrix
+    from model_cvx import mv_opt, mv_opt_prop
+    classfunk = ['TRANS']    # names a classfunk which can be called 
+except:
+    print('ModelFlow info: CVXopt not installed. Only matters if you are incorporating optimization')
+    pass
+
+from numpy import array
+
+from model_financial_stability import lifetime_credit_loss
+
+def sum_excel(*arg):
+    ''' a functions which sums the arguments used in models franslated from excel 
+    '''
+    return sum(arg)
+    
+def logit(number):
+    ''' A function which returns the logit of a number 
+    '''
+    return(-log(1.0/number-1.0)) 
+
+# @jit("f8(f8)")
+# def logit_inverse(number):
+#     ''' A function which returns the logit of a number 
+    
+#     takes care of extreme values 
+#     '''
+#     if number > 100:
+#         return 1.0
+#     elif number < -100:
+#         return 0.0
+#     else: 
+#         return 1/(1+exp(-number))
+
+def normcdf(input,mu=0.0,sigma=1.0):
+    return norm.cdf(input,mu,sigma)    
+
+
+def qgamma(q,a,loc):
+    res = gamma.ppf(q,a,loc,scale=1)
+    return res
+
+
+def cdf_lognorm_econ(x, mu, sigma, eps=1e-12):
+    """
+    Econ / GAMS-style log-normal cumulative distribution function (Newton-safe).
+
+    Computes the CDF of a log-normal distribution using the mean-parameterized
+    (econ / GAMS) formulation, where `mu` is the mean of the level variable X,
+    not the mean of log(X).
+
+    Mathematically:
+        F(x) = Φ((ln(x / mu) + 0.5 * sigma**2) / sigma)
+
+    where Φ(·) is the standard normal CDF.
+
+    This implementation is numerically safe for use in Newton solvers and
+    numerical differentiation by applying a soft lower bound to x and using
+    a stable normal CDF.
+
+    Parameters
+    ----------
+    x : float
+        Evaluation point. May be zero or negative during numerical
+        differentiation.
+    mu : float
+        Mean of the level variable X (E[X] = mu).
+    sigma : float
+        Standard deviation of ln(X).
+    eps : float, optional
+        Small positive floor used to ensure log(x) is well-defined.
+        Default is 1e-12.
+
+    Returns
+    -------
+    float
+        Value of the log-normal CDF in the interval [0, 1].
+
+    Notes
+    -----
+    - The median of the distribution is mu * exp(-0.5 * sigma**2),
+      not mu.
+    - As x → 0, the function smoothly approaches 0 with a vanishing
+      derivative, which is critical for Newton stability.
+    - Equivalent to the standard log-mean parameterization with:
+          mu_log = ln(mu) - 0.5 * sigma**2
+    """
+    from scipy.special import ndtr
+
+    x_safe = x if x > eps else eps
+    z = (log(x_safe / mu) + 0.5 * sigma**2) / sigma
+    return float(ndtr(z))
+
+def part_exp_lognorm(k, mu, std, eps=1e-12):
+    """
+    Safe scalar equivalent of the GAMS PartExpLogNorm function.
+
+    Parameters
+    ----------
+    k : float
+        Marginal input
+    mu : float
+        Average input (scale)
+    std : float
+        Dispersion / smoothing parameter
+    eps : float, optional
+        Small positive number to avoid log/division errors
+
+    Returns
+    -------
+    float
+        Smoothed, bounded transformation
+    """
+    # protect against division by zero
+    mu_safe = mu if abs(mu) > eps else eps
+
+    # protect against log(0)
+    ratio = abs(k / mu_safe)
+    ratio_safe = ratio if ratio > eps else eps
+
+    # protect against std = 0
+    std_safe = std if abs(std) > eps else eps
+
+    return mu_safe * erf((log(ratio_safe) - 0.5 * std_safe**2) / std_safe)
+
+
+
+def clognorm(input,mu=0.0,sigma=1.0):
+    res = lognorm.cdf(input,mu,sigma)
+    return res    
+
+if __name__ == '__main__' and 1:
+    xx = logit_inverse(-3*10
+                      )
+    print(xx)
+
+

model_Excel.py

@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Feb 12 07:04:02 2016
+
+@author: ibh
+
+Takes all formula's  from a excel work book and translates each to the equivalent expression. 
+Openpyxl is the fastest library but it can not deal all values. Therefor xlwings is also used. 
+But only to read repeated formula's which inly will show as '='
+
+
+
+Also defines function used when using xlwings to automate excel.
+
+These are used in :any:`modeldump_excel` and :any:`modelload_excel`  
+
+Some of the docstring are not very informative, to be improved. 
+
+
+
+"""
+import pandas as pd 
+import networkx as nx 
+import openpyxl
+from openpyxl import load_workbook
+from openpyxl.formula import Tokenizer
+from openpyxl.utils import get_column_letter
+from openpyxl.utils import cols_from_range,rows_from_range
+# try:
+#     import xlwings as xw
+# except:
+#     ...
+# import networkx as nx
+
+import matplotlib.pylab  as plt 
+import seaborn as sns
+from pathlib import Path
+
+import modelclass as mc
+
+DEBUG = 0
+
+def findequations(name):
+    '''Takes all formula's  from a excel work book and translates each to the equivalent expression. 
+    
+        Multicell ranges are expanded to a comma separated list. \n
+        The ordinary operators and the SUM function can be handled. If you need more functions. You have to impelent them in the modelclass. 
+        
+        In the model each cell reference is prefixed by <sheet name>_
+        
+        Openpyxl is the fastest library and it has a tokenizer
+        but it can not read all values. 
+        
+        Therefor xlwings is used to read repeated formula's which Openpyxl will show as '='
+
+    input:
+        :name: Location of a excel sheeet
+    
+         
+        Returns:
+             :modeldic: A dictionary with formulars keyed by cell reference   
+         
+    '''
+    outdic={}
+    wb = load_workbook(name, read_only=True,data_only=False) # to read the spresdsheet first save as xml then write it again 
+    try:
+        wb2 = xw.Book(name)    # the same worksheet in xlwings
+    except:
+        ... 
+    # breakpoint()
+    
+    allsheets = wb.sheetnames
+    for wsname in allsheets:
+        ws=wb[wsname]
+        try:
+            ws2=wb2.sheets(wsname)   # the same sheet but in xlwings
+        except: 
+            ...
+            
+        formulacell = [c for row in ws.rows for c in row if c.value != None and c.data_type == 'f'] 
+        for cell in formulacell:
+            cellref=get_column_letter(cell.column)+str(cell.row)
+            if DEBUG : print('This cell:',cellref,cell.data_type,cell.value)
+            if cell.value == '=' :
+                print(f'Repeat cell = so xlwings has to be imported {cell=}')
+                frml = cell.value if cell.value != '=' else ws2.range(cellref).formula # To avoid emty repeating formula'rs 
+            else:
+                frml = cell.value 
+            tok=Tokenizer(frml)
+            if DEBUG and False : print("\n".join("%19s%15s%9s" % (t.value, t.type, t.subtype) for t in tok.items))
+            # left hand term is <worksheet>!<column><row>= 
+            lhs=wstrans(wsname) + get_column_letter(cell.column)+str(cell.row)
+            out=[lhs+'=']
+            for t in tok.items:            
+                if t.subtype == "RANGE":
+                    #Find or create the sheetname  
+                    sheet0 = t.value.split('!')[0] if '!' in t.value else wsname
+                    sheet  =  wstrans(sheet0)
+                    # print(t.value,'---->')
+                    # Get all the cells in the range columwize 
+                    # the nested list comprehension makes the list works for square ranges. 
+                    # the split construct drops the sheet name from the range name if any 
+                    thisrange=[sheet+i for subtupler in cols_from_range((t.value.split('!')[-1])) for i in subtupler]
+
+                    # put a ',' between each element i the list 
+                    thistext=','.join(thisrange)
+                    #print(thisrange)
+                    out=out + [thistext]
+                else:
+                    out.append(t.value)
+            #create the equation and get rid of the !
+            equation=''.join(out).replace('!','_')
+            outdic[lhs]=equation
+            #print(equation)
+    try:        
+        wb2.close()
+    except: 
+        ... 
+    return outdic   
+    
+def showcells(name):
+    '''Finds values in a excel workbook with a value different from 0 
+    '''
+    wb = load_workbook(name, read_only=True,data_only=False) # to read the spresdsheet first save as xml then write it again 
+    allsheets = wb.sheetnames
+    for wsname in allsheets:
+        ws=wb[wsname]
+        for row in ws.rows:
+            for c in row:
+                if c.value != None:
+                    print(wsname,get_column_letter(c.column ),c.row,c.data_type,c.value)
+
+
+def findvalues(name):
+    '''Finds numerical values in a excel workbook with a value different from 0 
+    '''
+    wb = load_workbook(name, read_only=True,data_only=True) # to read the spresdsheet first save as xml then write it again 
+    allsheets = wb.sheetnames
+    values=[]
+    for wsname in allsheets:
+        ws=wb[wsname]
+        twsname = wstrans(wsname)
+        values+=[(twsname+get_column_letter(c.column )+str(c.row),c.value) 
+           for row in ws.rows for c in row 
+             if c.value != None and c.data_type == 'n' ]  
+    return values 
+    
+def wstrans(wsname):
+    'Translates workspace names'
+    res = '_'+wsname.replace("'","").replace(' - ','_').replace(' ','_').replace('-','_')+'_'
+    return res.upper() 
+
+def findcoordinates(name):
+    '''Finds the cell references matching the codes in a LCR workbook from EBA 
+    
+    This is needed for the mapping of the raw data to the excel cell refereces. 
+    
+    input:
+        :name: Location of a excel sheeet
+    
+    returns:
+        :coldf: Dataframe with mapping between excel column and EBA columns_code
+        :rowdf: Dataframe with row with mapping between excel row and EBS data row_code
+    '''
+    wb = load_workbook(name, read_only=True,data_only=True) # to read the spresdsheet first save as xml then write it again 
+    allsheets = wb.sheetnames
+    
+    colcodes=[]
+    rowcodes=[]
+    
+    for wsname in allsheets:
+        ws=wb[wsname]
+        try:
+        #find the anchor for the row and columns id, the first 
+            cell = [c for row in ws.rows for c in row if c.value and c.data_type == 's' and 'Row'== c.value][0]  
+            # find the numeric values in the column below the anchor, only the digits and remenber python index starts with 0 while excels index starts with 1 
+     
+            rowcodes += [(wsname , c.value , c.row ) for r in ws.rows for c in r  
+                        if c.value and c.column == cell.column and c.row > cell.row and 
+                              ( c.data_type == 'n' or (c.data_type == 's' and c.value.isdigit()) )]
+            
+    #         finds the numeric values in the row at the right of the anchor. 
+            # c.rows returns a generator (probably because te potential for a huge number) therefor the list(c.rows)
+            colcodes += [(wsname , c.value , get_column_letter(c.column ) )  for c in list(ws.rows)[cell.row-1] 
+                        if c.value and c.column > cell.column and (c.data_type == 'n' or (c.data_type == 's' and c.value.isdigit()))] 
+        except:  # Ok this ws did not have an ancor cell 
+            pass
+    coldf = pd.DataFrame(colcodes,columns=['sheet','colcode','col'])
+    rowdf = pd.DataFrame(rowcodes,columns=['sheet','rowcode','row'])
+    return coldf,rowdf
+
+def getexcelmodel(name):
+    ''' Creates a model instance from a excel sheet
+    SUM is replaced by SUM_EXCEL which is a function in the modelclass 
+    
+    In the excel formulars this function accepts ordinary operators and SUM in excel sheets 
+    
+    input:
+        :name: Location of a excel sheeet
+    
+    returns:
+        :model: A model instance with the formulars of the excel sheet
+        :para: A list of values in the sheet which matches exogeneous variables in the model 
+
+    '''
+    modelname = Path(name).stem
+    eqdic   = findequations(name)
+    eqdic2  = {i : eq.replace('SUM(','SUM_EXCEL(')  for i,eq in eqdic.items()}                
+    fdic    = {i : 'Frml xx '+eqdic2[i] + r' $ ' for i,eq in eqdic2.items()}                
+    f       = '\n'.join([eq for i,eq in fdic.items()])                
+    _mmodel=mc.model(f,modelname=modelname)               
+    zz = findvalues(name)
+    para = [z for z in zz if z[0] in _mmodel.exogene]   # find all values which match a exogeneous variable in model
+    return _mmodel,para
+
+#%% now functions related to running xlsheets 
+
+
+def indextrans(index):
+    '''
+    Transforms a period index to excel acceptable datatype 
+    
+    
+    '''    
+    out = [i.year if type(index) == pd.core.indexes.period.PeriodIndex 
+                     else int(i) for i in index]
+    return out 
+
+def df_to_sheet(name,df,wb,after=None):
+    '''
+    Dataframe to sheet 
+
+    Args:
+        name (TYPE): DESCRIPTION.
+        df (TYPE): DESCRIPTION.
+        wb (TYPE): DESCRIPTION.
+        after (TYPE, optional): DESCRIPTION. Defaults to None.
+
+    Returns:
+        sht (TYPE): DESCRIPTION.
+
+    '''
+    try:
+        wb.sheets[name].delete()
+    except:
+        pass  
+    
+    try:
+        sht = wb.sheets.add(name,after=after)
+    except Exception as e :
+        print('no sheet added',str(e))
+    df_ = df.copy()
+    df_.index = indextrans(df.index)
+    sht.range('A1').value = df_.T
+    active_window = wb.app.api.ActiveWindow
+    active_window.FreezePanes = False
+    active_window.SplitColumn = 1
+    active_window.SplitRow = 1
+    active_window.FreezePanes = True
+    sht.autofit(axis="columns")
+    sht[(2,25)].select()
+    return sht
+
+def obj_to_sheet(name,obj,wb,after=None):
+    '''
+    An python object to sheet
+
+    Args:
+        name (TYPE): DESCRIPTION.
+        obj (TYPE): DESCRIPTION.
+        wb (TYPE): DESCRIPTION.
+        after (TYPE, optional): DESCRIPTION. Defaults to None.
+
+    Returns:
+        None.
+
+    '''
+    # breakpoint()
+    try:
+        wb.sheets[name].delete()
+    except:
+        pass
+    
+    try:
+        sht = wb.sheets.add(name,after=after)
+    except Exception as e:
+        print(str(e))
+        print('no sheet added ')
+    sht.range('A1').value=obj
+    
+
+def sheet_to_df(wb,name):
+    '''
+    Sheet to df
+
+    Args:
+        wb (TYPE): DESCRIPTION.
+        name (TYPE): DESCRIPTION.
+
+    Returns:
+        df (TYPE): DESCRIPTION.
+
+    '''
+    df = wb.sheets[name].range('A1').options(pd.DataFrame, expand='table').value.T
+    df.index = indextrans(df.index)
+    return df 
+
+def sheet_to_dict(wb,name,integers=None):
+    ''' transform the named sheet to a python dict. If we need a integer it has to be in the integer set'''
+    
+    integers_ = {'max_iterations'} if isinstance(None,type(None)) else integers 
+    try:
+        out = wb.sheets[name].range('A1').options(dict,expand='table').value
+        out2 = {k : int(v) if k in integers_ else v for k,v in out.items()}
+    except:
+        out2={}
+    return out2
+         
+
+
+
+    
+if __name__ == '__main__':
+    testxls=Path('exceltest/lcrberegning2.xlsx')
+    mmodel,para = getexcelmodel(testxls)
+    eq=mmodel.equations
+    mmodel.draw('_LCR_C62',up=10,down=1,HR=0,pdf=1)  # The LCR 
+    mmodel.draw('_LCR_C25',up=4,down=1,pdf=1)  # liquid assets
+    mmodel.draw('_LCR_C10',up=2,pdf=1)  # Leel 1 covered bonds 
+    
+    c,r = findcoordinates(testxls)
+    xx = findequations(testxls)

model_cvx.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon May 26 21:11:18 2014
+
+@author: Ib Hansen
+
+A good explanation of quadradic programming in cvxopt is in 
+http://courses.csail.mit.edu/6.867/wiki/images/a/a7/Qp-cvxopt.pdf
+
+This exampel calculates the efficient forntier in a small example 
+the example is based on a mean variance model for Indonesian Rupia running in Excel 
+
+"""
+
+import numpy as np
+import pandas as pd
+from cvxopt         import matrix,  spdiag
+from cvxopt.solvers import qp    ,  options 
+
+
+def MV_test(lprint=True):
+    '''  Test a mean variance model for Indonesian Rupia
+    ''' 
+    P= matrix([
+    [0.01573635,	0.01436816,	0.01045556],
+    [0.01436816,	0.02289016,	0.01172995],
+    [0.01045556,	0.01172995,	0.01748076]])   # the covariance matrix
+
+    q       =  matrix([0.048,0.040,0.035])      # return vector 
+    bsum=1.0
+    
+    wsum1=20.   # weighted sum should be less than: 
+    weights1=matrix([2.5 , 1 , 1 ] ,(3,1))     
+    
+    wsum2= 1    # weightet sum should be greater than: 
+    weights2=matrix([0.2 , 2.4 , 1 ] ,(3,1))     
+    
+    
+    hmin    = -matrix([0. , 0 , 0])
+    
+    hmax    =  matrix([1. ,1. , 1.])
+    
+    
+    options['show_progress'] = False
+    
+    riskaversions = [r/100. for r in range(101)]             # compute 100 points on the efficient frontier 
+    portefolios   = [mv_opt(P,q,riskaversion,bsum,[[weights1],[-weights2]],[wsum1,-wsum2],hmin,hmax) for riskaversion in riskaversions] # minimize risk and maximize return 
+    p_return      = [100  * x.T * q           for x in portefolios]
+    risk          = [100 *( x.T * P *x)**0.5  for x in portefolios]
+    res           = [list(r)+list(p)+list(x)  for r,p,x in zip(risk,p_return,portefolios) ]  # a row in the Dataframe 
+
+    columns=['risk','return']+['Asset'+str(i)  for i,temp in enumerate(q)]   # to handle a number of assets 
+    results=pd.DataFrame(res,columns=columns)                 # create an empty pandas.Dataframe 
+    return results     
+    
+def mv_opt(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,boundsmax,maximize=True,lprint=False,solget=None):
+    ''' Performs mean variance optimization by calling a 
+    quadratic optimization function from the cvxopt 
+    library 
+    
+    '''
+    yield_multiplier = -1 if maximize else 1
+
+    q_size   =  len(qq)
+    P        =  matrix(2.0*(1.0-riskaversion)*PP)                 # to 
+    q        =  matrix(yield_multiplier*riskaversion*qq)
+    Gmin     = -matrix(np.eye(q_size))
+    hmin     = -matrix(boundsmin)
+    Gmax     =  matrix(np.eye(q_size))
+    hmax     =  matrix(boundsmax)
+    if weights:
+        Gweights =  matrix(weights)
+        hweights =  matrix(weigthtedsum)
+        G        =  matrix([Gmin,Gmax,Gweights.T])                       # creates the combined inequalities 
+        h        =  matrix([hmin,hmax,hweights])
+    else: 
+        G        =  matrix([Gmin,Gmax])                       # creates the combined inequalities 
+        h        =  matrix([hmin,hmax])
+        
+    A        =  matrix(1.,(1,q_size)) if bsum else None                                # sum of shares equal to bsum 
+    b        =  matrix([bsum])        if bsum else None   
+    options['show_progress'] = False
+    options['refinement']=10
+    sol      = qp(P,q,G,h,A,b) 
+    if solget:
+        return sol
+    else:                                     # minimize risk and maximize return 
+        x        = sol['x'] 
+        res      = x
+        return res                                                       # get the solution 
+
+def mv_opt_bs(msigma,vreturn,riskaversion,budget,risk_weights,capital,lcr_weights,lcr,leverage_weights,equity,boundsmin,boundsmax,lprint=False,solget=None):
+    '''
+    Performs balance sheet optimization using mean variance optimization 
+    '''
+    res = mv_opt(msigma,vreturn,riskaversion,budget,[risk_weights,-lcr_weights,leverage_weights],[capital,-lcr,equity],boundsmin,boundsmax,lprint=False,solget=None)
+    return res
+    
+
+def mv_opt_prop(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,boundsmax,probability=None,lprint=False):
+    ''' select a numner of assets/liabilities which. when the selection is feasible an Mean variance optimazation is performed\n
+    the selection is based on probabilities '''
+    q_size           =  len(qq)
+    selectsize       = 2 
+    newboundsmax     = matrix(0.,(q_size,1))
+    prop             = list(probability/sum(probability)) if probability else [1./q_size for i in range(q_size)]
+# Find a feasible set of banks  
+    while selectsize < q_size : 
+        selected               = [ int(i) for i in np.random.choice(q_size,selectsize,replace=False,p=prop)] # select banks 
+        selectvector           =  matrix(0.,(q_size,1))
+        selectvector[selected] = 1.0                                                      # the selected banks is marked by 1.1
+        newboundsmax           =  matrix([s*bm for s,bm in zip(selectvector,boundsmax)])  # elementwise multiplication, so max=0 if the bank is not selected 
+#        print(sum(newboundsmax))
+        if  sum(newboundsmax) >= bsum and [weigthtedsum] > list(newboundsmax.T*weights):
+            break
+        selectsize=selectsize+1
+    else: 
+        print('*** Constraints to do not allow a solution')
+        raise
+# now optimize 
+    try:
+        sol= mv_opt(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,newboundsmax,lprint=False,solget=True)
+        shares= sol['x']
+    except:
+        print('** The Mean variance problem can not be solved')
+        raise
+    return shares         
+ 
+if __name__ == '__main__': 
+    ib=MV_test()
+    import matplotlib.pyplot as plt
+#    pd.DataFrame.plot(ib,x='risk',y='return')
+    fig, axes = plt.subplots(nrows=3, ncols=1, figsize=(9, 12))
+    ib.plot(x='risk',y=['return'],kind='area'                  ,ax=axes[0])
+    ib.plot(x='risk',y=['Asset0','Asset1','Asset2'],ax=axes[1],kind='line')
+    ib.plot(x='risk',y=['Asset0','Asset1','Asset2'],ax=axes[2],kind='area')
+
+    
+