ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelBLfunk.py ADDED
@@ -0,0 +1,70 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Fri Mar 2 17:01:49 2018
4
+
5
+ @author: hanseni
6
+
7
+ Functions placed here are included in the Pyfs business language
8
+
9
+ """
10
+ from math import exp, log, sqrt
11
+ from numpy import transpose , array
12
+ from scipy.stats import norm,lognorm
13
+ from scipy.stats import gamma
14
+ import inspect
15
+ from numba import jit
16
+ classfunk = []
17
+ try:
18
+ from cvxopt import matrix
19
+ from model_cvx import mv_opt, mv_opt_prop
20
+ classfunk = ['TRANS'] # names a classfunk which can be called
21
+ except:
22
+ print('ModelFlow info: CVXopt not installed. Only matters if you are incorporating optimization')
23
+ pass
24
+
25
+ from numpy import array
26
+
27
+ from model_financial_stability import lifetime_credit_loss
28
+
29
+ def sum_excel(*arg):
30
+ ''' a functions which sums the arguments used in models franslated from excel
31
+ '''
32
+ return sum(arg)
33
+
34
+ def logit(number):
35
+ ''' A function which returns the logit of a number
36
+ '''
37
+ return(-log(1.0/number-1.0))
38
+
39
+ @jit("f8(f8)")
40
+ def logit_inverse(number):
41
+ ''' A function which returns the logit of a number
42
+
43
+ takes care of extreme values
44
+ '''
45
+ if number > 100:
46
+ return 1.0
47
+ elif number < -100:
48
+ return 0.0
49
+ else:
50
+ return 1/(1+exp(-number))
51
+
52
+ def normcdf(input,mu=0.0,sigma=1.0):
53
+ return norm.cdf(input,mu,sigma)
54
+
55
+
56
+ def qgamma(q,a,loc):
57
+ res = gamma.ppf(q,a,loc,scale=1)
58
+ return res
59
+
60
+
61
+
62
+ def clognorm(input,mu=0.0,sigma=1.0):
63
+ res = lognorm.cdf(input,mu,sigma)
64
+ return res
65
+
66
+ if __name__ == '__main__' and 1:
67
+ xx = logit_inverse(3)
68
+ print(xx)
69
+
70
+
model_Excel.py ADDED
@@ -0,0 +1,334 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Fri Feb 12 07:04:02 2016
4
+
5
+ @author: ibh
6
+
7
+ Takes all formula's from a excel work book and translates each to the equivalent expression.
8
+ Openpyxl is the fastest library but it can not deal all values. Therefor xlwings is also used.
9
+ But only to read repeated formula's which inly will show as '='
10
+
11
+
12
+
13
+ Also defines function used when using xlwings to automate excel.
14
+
15
+ These are used in :any:`modeldump_excel` and :any:`modelload_excel`
16
+
17
+ Some of the docstring are not very informative, to be improved.
18
+
19
+
20
+
21
+ """
22
+ import pandas as pd
23
+ import networkx as nx
24
+ import openpyxl
25
+ from openpyxl import load_workbook
26
+ from openpyxl.formula import Tokenizer
27
+ from openpyxl.utils import get_column_letter
28
+ from openpyxl.utils import cols_from_range,rows_from_range
29
+ # try:
30
+ # import xlwings as xw
31
+ # except:
32
+ # ...
33
+ # import networkx as nx
34
+
35
+ import matplotlib.pylab as plt
36
+ import seaborn as sns
37
+ import os
38
+ import sys
39
+ from pathlib import Path
40
+
41
+ import modelclass as mc
42
+
43
+ DEBUG = 0
44
+
45
+ def findequations(name):
46
+ '''Takes all formula's from a excel work book and translates each to the equivalent expression.
47
+
48
+ Multicell ranges are expanded to a comma separated list. \n
49
+ The ordinary operators and the SUM function can be handled. If you need more functions. You have to impelent them in the modelclass.
50
+
51
+ In the model each cell reference is prefixed by <sheet name>_
52
+
53
+ Openpyxl is the fastest library and it has a tokenizer
54
+ but it can not read all values.
55
+
56
+ Therefor xlwings is used to read repeated formula's which Openpyxl will show as '='
57
+
58
+ input:
59
+ :name: Location of a excel sheeet
60
+
61
+
62
+ Returns:
63
+ :modeldic: A dictionary with formulars keyed by cell reference
64
+
65
+ '''
66
+ outdic={}
67
+ wb = load_workbook(name, read_only=True,data_only=False) # to read the spresdsheet first save as xml then write it again
68
+ try:
69
+ wb2 = xw.Book(name) # the same worksheet in xlwings
70
+ except:
71
+ ...
72
+ # breakpoint()
73
+
74
+ allsheets = wb.sheetnames
75
+ for wsname in allsheets:
76
+ ws=wb[wsname]
77
+ try:
78
+ ws2=wb2.sheets(wsname) # the same sheet but in xlwings
79
+ except:
80
+ ...
81
+
82
+ formulacell = [c for row in ws.rows for c in row if c.value != None and c.data_type == 'f']
83
+ for cell in formulacell:
84
+ cellref=get_column_letter(cell.column)+str(cell.row)
85
+ if DEBUG : print('This cell:',cellref,cell.data_type,cell.value)
86
+ if cell.value == '=' :
87
+ print(f'Repeat cell = so xlwings has to be imported {cell=}')
88
+ frml = cell.value if cell.value != '=' else ws2.range(cellref).formula # To avoid emty repeating formula'rs
89
+ else:
90
+ frml = cell.value
91
+ tok=Tokenizer(frml)
92
+ if DEBUG and False : print("\n".join("%19s%15s%9s" % (t.value, t.type, t.subtype) for t in tok.items))
93
+ # left hand term is <worksheet>!<column><row>=
94
+ lhs=wstrans(wsname) + get_column_letter(cell.column)+str(cell.row)
95
+ out=[lhs+'=']
96
+ for t in tok.items:
97
+ if t.subtype == "RANGE":
98
+ #Find or create the sheetname
99
+ sheet0 = t.value.split('!')[0] if '!' in t.value else wsname
100
+ sheet = wstrans(sheet0)
101
+ # print(t.value,'---->')
102
+ # Get all the cells in the range columwize
103
+ # the nested list comprehension makes the list works for square ranges.
104
+ # the split construct drops the sheet name from the range name if any
105
+ thisrange=[sheet+i for subtupler in cols_from_range((t.value.split('!')[-1])) for i in subtupler]
106
+
107
+ # put a ',' between each element i the list
108
+ thistext=','.join(thisrange)
109
+ #print(thisrange)
110
+ out=out + [thistext]
111
+ else:
112
+ out.append(t.value)
113
+ #create the equation and get rid of the !
114
+ equation=''.join(out).replace('!','_')
115
+ outdic[lhs]=equation
116
+ #print(equation)
117
+ try:
118
+ wb2.close()
119
+ except:
120
+ ...
121
+ return outdic
122
+
123
+ def showcells(name):
124
+ '''Finds values in a excel workbook with a value different from 0
125
+ '''
126
+ wb = load_workbook(name, read_only=True,data_only=False) # to read the spresdsheet first save as xml then write it again
127
+ allsheets = wb.sheetnames
128
+ for wsname in allsheets:
129
+ ws=wb[wsname]
130
+ for row in ws.rows:
131
+ for c in row:
132
+ if c.value != None:
133
+ print(wsname,get_column_letter(c.column ),c.row,c.data_type,c.value)
134
+
135
+
136
+ def findvalues(name):
137
+ '''Finds numerical values in a excel workbook with a value different from 0
138
+ '''
139
+ wb = load_workbook(name, read_only=True,data_only=True) # to read the spresdsheet first save as xml then write it again
140
+ allsheets = wb.sheetnames
141
+ values=[]
142
+ for wsname in allsheets:
143
+ ws=wb[wsname]
144
+ twsname = wstrans(wsname)
145
+ values+=[(twsname+get_column_letter(c.column )+str(c.row),c.value)
146
+ for row in ws.rows for c in row
147
+ if c.value != None and c.data_type == 'n' ]
148
+ return values
149
+
150
+ def wstrans(wsname):
151
+ 'Translates workspace names'
152
+ res = '_'+wsname.replace("'","").replace(' - ','_').replace(' ','_').replace('-','_')+'_'
153
+ return res.upper()
154
+
155
+ def findcoordinates(name):
156
+ '''Finds the cell references matching the codes in a LCR workbook from EBA
157
+
158
+ This is needed for the mapping of the raw data to the excel cell refereces.
159
+
160
+ input:
161
+ :name: Location of a excel sheeet
162
+
163
+ returns:
164
+ :coldf: Dataframe with mapping between excel column and EBA columns_code
165
+ :rowdf: Dataframe with row with mapping between excel row and EBS data row_code
166
+ '''
167
+ wb = load_workbook(name, read_only=True,data_only=True) # to read the spresdsheet first save as xml then write it again
168
+ allsheets = wb.sheetnames
169
+
170
+ colcodes=[]
171
+ rowcodes=[]
172
+
173
+ for wsname in allsheets:
174
+ ws=wb[wsname]
175
+ try:
176
+ #find the anchor for the row and columns id, the first
177
+ cell = [c for row in ws.rows for c in row if c.value and c.data_type == 's' and 'Row'== c.value][0]
178
+ # find the numeric values in the column below the anchor, only the digits and remenber python index starts with 0 while excels index starts with 1
179
+
180
+ rowcodes += [(wsname , c.value , c.row ) for r in ws.rows for c in r
181
+ if c.value and c.column == cell.column and c.row > cell.row and
182
+ ( c.data_type == 'n' or (c.data_type == 's' and c.value.isdigit()) )]
183
+
184
+ # finds the numeric values in the row at the right of the anchor.
185
+ # c.rows returns a generator (probably because te potential for a huge number) therefor the list(c.rows)
186
+ colcodes += [(wsname , c.value , get_column_letter(c.column ) ) for c in list(ws.rows)[cell.row-1]
187
+ if c.value and c.column > cell.column and (c.data_type == 'n' or (c.data_type == 's' and c.value.isdigit()))]
188
+ except: # Ok this ws did not have an ancor cell
189
+ pass
190
+ coldf = pd.DataFrame(colcodes,columns=['sheet','colcode','col'])
191
+ rowdf = pd.DataFrame(rowcodes,columns=['sheet','rowcode','row'])
192
+ return coldf,rowdf
193
+
194
+ def getexcelmodel(name):
195
+ ''' Creates a model instance from a excel sheet
196
+ SUM is replaced by SUM_EXCEL which is a function in the modelclass
197
+
198
+ In the excel formulars this function accepts ordinary operators and SUM in excel sheets
199
+
200
+ input:
201
+ :name: Location of a excel sheeet
202
+
203
+ returns:
204
+ :model: A model instance with the formulars of the excel sheet
205
+ :para: A list of values in the sheet which matches exogeneous variables in the model
206
+
207
+ '''
208
+ modelname = Path(name).stem
209
+ eqdic = findequations(name)
210
+ eqdic2 = {i : eq.replace('SUM(','SUM_EXCEL(') for i,eq in eqdic.items()}
211
+ fdic = {i : 'Frml xx '+eqdic2[i] + r' $ ' for i,eq in eqdic2.items()}
212
+ f = '\n'.join([eq for i,eq in fdic.items()])
213
+ _mmodel=mc.model(f,modelname=modelname)
214
+ zz = findvalues(name)
215
+ para = [z for z in zz if z[0] in _mmodel.exogene] # find all values which match a exogeneous variable in model
216
+ return _mmodel,para
217
+
218
+ #%% now functions related to running xlsheets
219
+
220
+
221
+ def indextrans(index):
222
+ '''
223
+ Transforms a period index to excel acceptable datatype
224
+
225
+
226
+ '''
227
+ out = [i.year if type(index) == pd.core.indexes.period.PeriodIndex
228
+ else int(i) for i in index]
229
+ return out
230
+
231
+ def df_to_sheet(name,df,wb,after=None):
232
+ '''
233
+ Dataframe to sheet
234
+
235
+ Args:
236
+ name (TYPE): DESCRIPTION.
237
+ df (TYPE): DESCRIPTION.
238
+ wb (TYPE): DESCRIPTION.
239
+ after (TYPE, optional): DESCRIPTION. Defaults to None.
240
+
241
+ Returns:
242
+ sht (TYPE): DESCRIPTION.
243
+
244
+ '''
245
+ try:
246
+ wb.sheets[name].delete()
247
+ except:
248
+ pass
249
+
250
+ try:
251
+ sht = wb.sheets.add(name,after=after)
252
+ except Exception as e :
253
+ print('no sheet added',str(e))
254
+ df_ = df.copy()
255
+ df_.index = indextrans(df.index)
256
+ sht.range('A1').value = df_.T
257
+ active_window = wb.app.api.ActiveWindow
258
+ active_window.FreezePanes = False
259
+ active_window.SplitColumn = 1
260
+ active_window.SplitRow = 1
261
+ active_window.FreezePanes = True
262
+ sht.autofit(axis="columns")
263
+ sht[(2,25)].select()
264
+ return sht
265
+
266
+ def obj_to_sheet(name,obj,wb,after=None):
267
+ '''
268
+ An python object to sheet
269
+
270
+ Args:
271
+ name (TYPE): DESCRIPTION.
272
+ obj (TYPE): DESCRIPTION.
273
+ wb (TYPE): DESCRIPTION.
274
+ after (TYPE, optional): DESCRIPTION. Defaults to None.
275
+
276
+ Returns:
277
+ None.
278
+
279
+ '''
280
+ # breakpoint()
281
+ try:
282
+ wb.sheets[name].delete()
283
+ except:
284
+ pass
285
+
286
+ try:
287
+ sht = wb.sheets.add(name,after=after)
288
+ except Exception as e:
289
+ print(str(e))
290
+ print('no sheet added ')
291
+ sht.range('A1').value=obj
292
+
293
+
294
+ def sheet_to_df(wb,name):
295
+ '''
296
+ Sheet to df
297
+
298
+ Args:
299
+ wb (TYPE): DESCRIPTION.
300
+ name (TYPE): DESCRIPTION.
301
+
302
+ Returns:
303
+ df (TYPE): DESCRIPTION.
304
+
305
+ '''
306
+ df = wb.sheets[name].range('A1').options(pd.DataFrame, expand='table').value.T
307
+ df.index = indextrans(df.index)
308
+ return df
309
+
310
+ def sheet_to_dict(wb,name,integers=None):
311
+ ''' transform the named sheet to a python dict. If we need a integer it has to be in the integer set'''
312
+
313
+ integers_ = {'max_iterations'} if isinstance(None,type(None)) else integers
314
+ try:
315
+ out = wb.sheets[name].range('A1').options(dict,expand='table').value
316
+ out2 = {k : int(v) if k in integers_ else v for k,v in out.items()}
317
+ except:
318
+ out2={}
319
+ return out2
320
+
321
+
322
+
323
+
324
+
325
+ if __name__ == '__main__':
326
+ testxls=Path('exceltest\lcrberegning2.xlsx')
327
+ mmodel,para = getexcelmodel(testxls)
328
+ eq=mmodel.equations
329
+ mmodel.draw('_LCR_C62',up=10,down=1,HR=0,pdf=1) # The LCR
330
+ mmodel.draw('_LCR_C25',up=4,down=1) # liquid assets
331
+ mmodel.draw('_LCR_C10',up=2) # Leel 1 covered bonds
332
+
333
+ c,r = findcoordinates(testxls)
334
+ xx = findequations(testxls)
model_cvx.py ADDED
@@ -0,0 +1,132 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Mon May 26 21:11:18 2014
4
+
5
+ @author: Ib Hansen
6
+
7
+ A good explanation of quadradic programming in cvxopt is in
8
+ http://courses.csail.mit.edu/6.867/wiki/images/a/a7/Qp-cvxopt.pdf
9
+
10
+ This exampel calculates the efficient forntier in a small example
11
+ the example is based on a mean variance model for Indonesian Rupia running in Excel
12
+
13
+ """
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+ from cvxopt import matrix, spdiag
18
+ from cvxopt.solvers import qp , options
19
+
20
+
21
+ def MV_test(lprint=True):
22
+ ''' Test a mean variance model for Indonesian Rupia
23
+ '''
24
+ P= matrix([
25
+ [0.01573635, 0.01436816, 0.01045556],
26
+ [0.01436816, 0.02289016, 0.01172995],
27
+ [0.01045556, 0.01172995, 0.01748076]]) # the covariance matrix
28
+
29
+ q = matrix([0.048,0.040,0.035]) # return vector
30
+ bsum=1.0
31
+
32
+ wsum1=20. # weighted sum should be less than:
33
+ weights1=matrix([2.5 , 1 , 1 ] ,(3,1))
34
+
35
+ wsum2= 1 # weightet sum should be greater than:
36
+ weights2=matrix([0.2 , 2.4 , 1 ] ,(3,1))
37
+
38
+
39
+ hmin = -matrix([0. , 0 , 0])
40
+
41
+ hmax = matrix([1. ,1. , 1.])
42
+
43
+
44
+ options['show_progress'] = False
45
+
46
+ riskaversions = [r/100. for r in range(101)] # compute 100 points on the efficient frontier
47
+ portefolios = [mv_opt(P,q,riskaversion,bsum,[[weights1],[-weights2]],[wsum1,-wsum2],hmin,hmax) for riskaversion in riskaversions] # minimize risk and maximize return
48
+ p_return = [100 * x.T * q for x in portefolios]
49
+ risk = [100 *( x.T * P *x)**0.5 for x in portefolios]
50
+ res = [list(r)+list(p)+list(x) for r,p,x in zip(risk,p_return,portefolios) ] # a row in the Dataframe
51
+
52
+ columns=['risk','return']+['Asset'+str(i) for i,temp in enumerate(q)] # to handle a number of assets
53
+ results=pd.DataFrame(res,columns=columns) # create an empty pandas.Dataframe
54
+ return results
55
+
56
+ def mv_opt(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,boundsmax,lprint=False,solget=None):
57
+ ''' Performs mean variance optimization by calling a
58
+ quadratic optimization function from the cvxopt
59
+ library
60
+
61
+ '''
62
+ q_size = len(qq)
63
+ P = matrix(2.0*(1.0-riskaversion)*PP) # to
64
+ q = matrix(-1.0*riskaversion*qq)
65
+ Gmin = -matrix(np.eye(q_size))
66
+ hmin = -matrix(boundsmin)
67
+ Gmax = matrix(np.eye(q_size))
68
+ hmax = matrix(boundsmax)
69
+ Gweights = matrix(weights)
70
+ hweights = matrix(weigthtedsum)
71
+ G = matrix([Gmin,Gmax,Gweights.T]) # creates the combined inequalities
72
+ h = matrix([hmin,hmax,hweights])
73
+ A = matrix(1.,(1,q_size)) if bsum else None # sum of shares equal to bsum
74
+ b = matrix([bsum]) if bsum else None
75
+ options['show_progress'] = False
76
+ options['refinement']=10
77
+ sol = qp(P,q,G,h,A,b)
78
+ if solget:
79
+ return sol
80
+ else: # minimize risk and maximize return
81
+ x = sol['x']
82
+ res = x
83
+ return res # get the solution
84
+
85
+ def mv_opt_bs(msigma,vreturn,riskaversion,budget,risk_weights,capital,lcr_weights,lcr,leverage_weights,equity,boundsmin,boundsmax,lprint=False,solget=None):
86
+ '''
87
+ Performs balance sheet optimization using mean variance optimization
88
+ '''
89
+ res = mv_opt(msigma,vreturn,riskaversion,budget,[risk_weights,-lcr_weights,leverage_weights],[capital,-lcr,equity],boundsmin,boundsmax,lprint=False,solget=None)
90
+ return res
91
+
92
+
93
+ def mv_opt_prop(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,boundsmax,probability=None,lprint=False):
94
+ ''' select a numner of assets/liabilities which. when the selection is feasible an Mean variance optimazation is performed\n
95
+ the selection is based on probabilities '''
96
+ q_size = len(qq)
97
+ selectsize = 2
98
+ newboundsmax = matrix(0.,(q_size,1))
99
+ prop = list(probability/sum(probability)) if probability else [1./q_size for i in range(q_size)]
100
+ # Find a feasible set of banks
101
+ while selectsize < q_size :
102
+ selected = [ int(i) for i in np.random.choice(q_size,selectsize,replace=False,p=prop)] # select banks
103
+ selectvector = matrix(0.,(q_size,1))
104
+ selectvector[selected] = 1.0 # the selected banks is marked by 1.1
105
+ newboundsmax = matrix([s*bm for s,bm in zip(selectvector,boundsmax)]) # elementwise multiplication, so max=0 if the bank is not selected
106
+ # print(sum(newboundsmax))
107
+ if sum(newboundsmax) >= bsum and [weigthtedsum] > list(newboundsmax.T*weights):
108
+ break
109
+ selectsize=selectsize+1
110
+ else:
111
+ print('*** Constraints to do not allow a solution')
112
+ raise
113
+ # now optimize
114
+ try:
115
+ sol= mv_opt(PP,qq,riskaversion,bsum,weights,weigthtedsum,boundsmin,newboundsmax,lprint=False,solget=True)
116
+ shares= sol['x']
117
+ except:
118
+ print('** The Mean variance problem can not be solved')
119
+ raise
120
+ return shares
121
+
122
+ if __name__ == '__main__':
123
+ ib=MV_test()
124
+ import matplotlib.pyplot as plt
125
+ # pd.DataFrame.plot(ib,x='risk',y='return')
126
+ fig, axes = plt.subplots(nrows=3, ncols=1, figsize=(9, 12))
127
+ ib.plot(x='risk',y=['return'],kind='area' ,ax=axes[0])
128
+ ib.plot(x='risk',y=['Asset0','Asset1','Asset2'],ax=axes[1],kind='line')
129
+ ib.plot(x='risk',y=['Asset0','Asset1','Asset2'],ax=axes[2],kind='area')
130
+
131
+
132
+
model_doable.py ADDED
@@ -0,0 +1,40 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Thu Feb 1 00:36:40 2018
4
+
5
+ @author: hanseni
6
+ """
7
+
8
+ import re
9
+ import pandas as pd
10
+ import sys
11
+ import matplotlib.pyplot as plt
12
+ from matplotlib import colors
13
+ import seaborn as sns
14
+ import numpy as np
15
+ from collections import namedtuple
16
+ import os
17
+
18
+ import modelmanipulation as mp
19
+ import modelclass as mc
20
+ import model_Excel as me
21
+ #import databank as pb
22
+ from itertools import groupby,chain
23
+
24
+
25
+ path= 'imf-lcr\\'
26
+ if __name__ == '__main__' :
27
+ # read from the P: drive and save to local else read from local drive (just to work if not on the network)
28
+ tlcr = open(path+'lcr.txt').read() # read the template model
29
+ t2lcr = doable(tlcr)
30
+ #%%
31
+
32
+
33
+ print(t2lcr)
34
+ flcr = mp.explode(mp.tofrml(t2lcr))
35
+ mlcr = mc.model(flcr)
36
+ mlcr.drawall()
37
+
38
+ exp='frml ddd ddd %0_ib %1__ha'
39
+ l = ['{abekat}','{hest}']
40
+ ud = dosubst(l,exp)