ModelFlowIb 1.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ModelFlowIb-1.46.dist-info/METADATA +151 -0
- ModelFlowIb-1.46.dist-info/RECORD +48 -0
- ModelFlowIb-1.46.dist-info/WHEEL +5 -0
- ModelFlowIb-1.46.dist-info/top_level.txt +44 -0
- data/ken_base.pcim +1 -0
- modelBLfunk.py +70 -0
- model_Excel.py +334 -0
- model_cvx.py +132 -0
- model_doable.py +40 -0
- model_dynare.py +164 -0
- model_financial_stability.py +88 -0
- model_ifrs9.py +88 -0
- model_latex.py +505 -0
- model_latex_class.py +478 -0
- model_latex_classpy.py +401 -0
- model_run_numba.py +15 -0
- modelclass.py +7057 -0
- modelclass2.py +463 -0
- modeldash.py +265 -0
- modeldashboot.py +202 -0
- modeldashsidebar.py +442 -0
- modeldekom.py +518 -0
- modeldiff.py +561 -0
- modelgrab.py +318 -0
- modelgrabwf2.py +554 -0
- modelhelp.py +253 -0
- modelinvert.py +211 -0
- modeljupyter.py +821 -0
- modeljupytermagic.py +267 -0
- modelmacrograb.py +98 -0
- modelmanipulation.py +1273 -0
- modelmf.py +265 -0
- modelnet.py +114 -0
- modelnewton.py +819 -0
- modelnormalize.py +348 -0
- modelpattern.py +253 -0
- modelpatternrev.py +72 -0
- modelsandbox.py +76 -0
- modelsandbox_Mixin.py +1565 -0
- modeltodo.py +31 -0
- modeluserfunk.py +97 -0
- modeluserfunk_old.py +90 -0
- modelvis.py +677 -0
- modelwidget.py +610 -0
- modelwidget_input.py +961 -0
- modelwidget_new.py +1005 -0
- modelwidget_new_new.py +1238 -0
- modelwidget_shinypy.py +935 -0
modelgrab.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Created on Mon Jun 10 21:11:08 2019
|
|
4
|
+
|
|
5
|
+
@author: hanseni
|
|
6
|
+
|
|
7
|
+
modules to grab models with different specifications and make them ModelFlow conforme
|
|
8
|
+
|
|
9
|
+
**GrabWbModel** will take a eviews model and transform it to Business logic
|
|
10
|
+
- Create a normalized model, add dampning for the stocastic equations
|
|
11
|
+
- Add add-factors to the stocastic equations
|
|
12
|
+
- Generate BL for a model which calculates add-factors so a solution will match teh existing values
|
|
13
|
+
- Generate BL for the model
|
|
14
|
+
|
|
15
|
+
-grap data from excel sheet
|
|
16
|
+
|
|
17
|
+
- Make model instance for model and add-factor model
|
|
18
|
+
|
|
19
|
+
- Run the model, check that the results match.
|
|
20
|
+
|
|
21
|
+
For debuggging valuesthe last part checs value in the order, in which they are calculated,
|
|
22
|
+
and then displays the input to off mark equations
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import pandas as pd
|
|
26
|
+
import re
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
import functools
|
|
29
|
+
from tqdm import tqdm
|
|
30
|
+
import json
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
from modelclass import model
|
|
35
|
+
import modelmf
|
|
36
|
+
import modelmanipulation as mp
|
|
37
|
+
|
|
38
|
+
import modelnormalize as nz
|
|
39
|
+
|
|
40
|
+
assert 1==1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class GrapWbModel():
|
|
45
|
+
'''This class takes a world bank model specification, variable data and variable description
|
|
46
|
+
and transform it to ModelFlow business language'''
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
frml : str = '' # path to model
|
|
50
|
+
data : str = '' # path to data
|
|
51
|
+
des : any = '' # path to descriptions
|
|
52
|
+
scalars : str = '' # path to scalars
|
|
53
|
+
modelname : str = 'No Name' # modelname
|
|
54
|
+
start : int = 2017
|
|
55
|
+
end : int = 2040
|
|
56
|
+
country_trans : any = lambda x:x[:] # function which transform model specification
|
|
57
|
+
country_df_trans : any = lambda x:x # function which transforms initial dataframe
|
|
58
|
+
from_wf2 : bool = False
|
|
59
|
+
make_fitted : bool = False # if True, a clean equation for fittet variables is created
|
|
60
|
+
fit_start : int =2000 # start of fittet model unless overruled by mfmsa
|
|
61
|
+
fit_end : int = 2100 # end of fittet model unless overruled by mfmsa
|
|
62
|
+
do_add_factor_calc : bool = True # calculate the add factors
|
|
63
|
+
mfmsa : str = '' # path to mfsa options
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def __post_init__(self):
|
|
67
|
+
# breakpoint()
|
|
68
|
+
|
|
69
|
+
print(f'\nProcessing the model:{self.modelname}',flush=True)
|
|
70
|
+
self.rawmodel_org = open(self.frml).read()
|
|
71
|
+
eviewsline = [l for l in self.rawmodel_org.split('\n') if len(l.strip()) >=2
|
|
72
|
+
and not ('*******' in l or '----------' in l)]
|
|
73
|
+
# breakpoint()
|
|
74
|
+
self.rawmodel = self.country_trans(self.rawmodel_org)
|
|
75
|
+
rawmodel6 = self.trans_eviews(self.rawmodel)
|
|
76
|
+
# breakpoint()
|
|
77
|
+
bars = '{desc}: {percentage:3.0f}%|{bar}|{n_fmt}/{total_fmt}'
|
|
78
|
+
if self.from_wf2:
|
|
79
|
+
|
|
80
|
+
orgline = [l for l in rawmodel6.split('\n')]
|
|
81
|
+
line_type = ['ident' if l.startswith('@IDENTITY') else 'stoc' for l in orgline]
|
|
82
|
+
line = [l.replace('@IDENTITY ','').replace(' ','') for l in orgline]
|
|
83
|
+
else:
|
|
84
|
+
line_type = []
|
|
85
|
+
line =[]
|
|
86
|
+
with tqdm(total=len(rawmodel6.split('\n')),desc='Reading original model',bar_format=bars) as pbar:
|
|
87
|
+
for l in rawmodel6.split('\n'):
|
|
88
|
+
if ('*******' in l or '----------' in l) and 'IDEN' in l.upper():
|
|
89
|
+
sec='iden'
|
|
90
|
+
#print(l)
|
|
91
|
+
elif '*******' in l and 'STOC' in l:
|
|
92
|
+
sec='stoc'
|
|
93
|
+
#print(l)
|
|
94
|
+
else:
|
|
95
|
+
line_type.append(sec)
|
|
96
|
+
line.append(l)
|
|
97
|
+
# print(f' {sec} {l[:30]} ....')
|
|
98
|
+
pbar.update(1)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
errline1 = [(l,o,e) for l,o,e in zip(line,orgline,eviewsline) if '@' in l or '.coef' in l ]
|
|
102
|
+
if len(errline1):
|
|
103
|
+
print('Probably errors as @ in lines:')
|
|
104
|
+
for l,o,e in errline1:
|
|
105
|
+
print(f'\nEviews line :{e}')
|
|
106
|
+
print(f'Original line :{o}')
|
|
107
|
+
print(f'New modelflow line:{l}')
|
|
108
|
+
raise Exception('@ in lines ')
|
|
109
|
+
|
|
110
|
+
self.all_frml = [nz.normal(l,add_add_factor=(typ=='stoc'),make_fitted=(typ=='stoc'),make_fixable =(typ=='stoc')) for l,typ in tqdm(zip(line,line_type),desc='Normalizing model',total=len(line),bar_format=bars)]
|
|
111
|
+
self.all_frml_dict = {f.endo_var: f for f in self.all_frml}
|
|
112
|
+
lfname = ["<Z,EXO> " if typ == 'stoc' else '' for typ in line_type ]
|
|
113
|
+
self.rorg = [fname + f.normalized for f,fname in zip(self.all_frml,lfname) ]
|
|
114
|
+
|
|
115
|
+
if self.make_fitted:
|
|
116
|
+
self.rfitmodel = ['<FIT> ' + f.fitted for f in self.all_frml if len(f.fitted)]
|
|
117
|
+
self.mfitmodel = model('\n'.join(self.rfitmodel))
|
|
118
|
+
self.mfitmodel.modelname = self.modelname + ' calc fittet values'
|
|
119
|
+
else:
|
|
120
|
+
self.rfitmodel = []
|
|
121
|
+
|
|
122
|
+
self.rres = [f'{f.calc_add_factor}' for f in self.all_frml if len(f.calc_add_factor)]
|
|
123
|
+
self.rres_tomodel ='\n'.join([f'FRML <CALC_ADD_FACTOR> {f.calc_add_factor}$' for f in self.all_frml if len(f.calc_add_factor)])
|
|
124
|
+
# self.fmodel = mp.exounroll(mp.tofrml ('\n'.join(self.rorg+self.rfitmodel)))+self.rres_tomodel
|
|
125
|
+
self.fmodel = mp.tofrml ('\n'.join(self.rorg+self.rfitmodel))+self.rres_tomodel
|
|
126
|
+
# breakpoint()
|
|
127
|
+
self.fres = ('\n'.join(self.rres))
|
|
128
|
+
self.mmodel = model(self.fmodel,modelname = self.modelname)
|
|
129
|
+
self.mmodel.set_var_description(self.var_description)
|
|
130
|
+
self.mmodel.wb_MFMSAOPTIONS = self.mfmsa_options
|
|
131
|
+
self.mres = model(self.fres,modelname = f'Calculation of add factors for {self.modelname}')
|
|
132
|
+
# breakpoint()
|
|
133
|
+
self.start,self.end = self.mfmsa_start_end
|
|
134
|
+
if self.do_add_factor_calc:
|
|
135
|
+
self.base_input = self.mres.res(self.dfmodel,self.start,self.end)
|
|
136
|
+
else:
|
|
137
|
+
self.base_input = self.dfmodel
|
|
138
|
+
|
|
139
|
+
@staticmethod
|
|
140
|
+
def trans_eviews(rawmodel):
|
|
141
|
+
rawmodel0 = '\n'.join(l for l in rawmodel.upper().split('\n') if len(l.strip()) >=2)
|
|
142
|
+
# trailing and leading "
|
|
143
|
+
rawmodel1 = '\n'.join(l[1:-1] if l.startswith('"') else l for l in rawmodel0.split('\n'))
|
|
144
|
+
# powers
|
|
145
|
+
rawmodel2 = rawmodel1.replace('^','**').replace('""',' ').replace('"',' ').\
|
|
146
|
+
replace('@EXP','exp').replace('@RECODE','recode').replace('@MOVAV','movavg').replace('@LOGIT','logit_inverse') \
|
|
147
|
+
.replace('@MEAN(@PC(','@AVERAGE_GROWTH((').replace('@PCY','PCT_GROWTH').replace('@PC','PCT_GROWTH')\
|
|
148
|
+
.replace('@PMAX','MAX').replace('@TREND','EVIEWS_TREND')
|
|
149
|
+
# @ELEM and @DURING
|
|
150
|
+
# @ELEM and @DURING
|
|
151
|
+
rawmodel3 = nz.elem_trans(rawmodel2)
|
|
152
|
+
rawmodel4 = re.sub(r'@DURING\( *([0-9]+) *\)', r'during_\1',rawmodel3)
|
|
153
|
+
rawmodel5 = re.sub(r'@DURING\( *([0-9]+) *([0-9]+) *\)', r'during_\1_\2',rawmodel4)
|
|
154
|
+
|
|
155
|
+
# during check
|
|
156
|
+
ldur = '\n'.join(l for l in rawmodel5.split('\n') if '@DURING' in l)
|
|
157
|
+
ldur2 = '\n'.join(l for l in rawmodel5.split('\n') if 'during' in l)
|
|
158
|
+
|
|
159
|
+
# check D(
|
|
160
|
+
ld = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z]|^)D\(',l) )
|
|
161
|
+
ld1 = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z0-9_]|^)D\(',l) )
|
|
162
|
+
# breakpoint()
|
|
163
|
+
rawmodel6 = nz.funk_replace('D','DIFF',rawmodel5)
|
|
164
|
+
# did we get all the lines
|
|
165
|
+
ldif = '\n'.join(l for l in rawmodel6.split('\n') if 'DIFF(' in l )
|
|
166
|
+
return rawmodel6
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def var_description(self):
|
|
170
|
+
'''
|
|
171
|
+
Adds var descriptions for add factors, exogenizing dummies and exoggenizing values
|
|
172
|
+
'''
|
|
173
|
+
|
|
174
|
+
if isinstance(self.des,dict):
|
|
175
|
+
return self.des
|
|
176
|
+
|
|
177
|
+
try:
|
|
178
|
+
# breakpoint()
|
|
179
|
+
des_file = Path(self.des)
|
|
180
|
+
if des_file.suffix == '.xlsx':
|
|
181
|
+
trans0 = pd.read_excel(self.des).loc[:,['mnem','Excel']].set_index('mnem').to_dict(orient = 'dict')['Excel']
|
|
182
|
+
var_description = {str(k) : str(v) for k,v in trans0.items() if 'nan' != str(v)}
|
|
183
|
+
else:
|
|
184
|
+
with open(des_file,'rt') as f:
|
|
185
|
+
var_description = json.load(f)
|
|
186
|
+
add_d = { newname : 'Add factor:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_A') in self.mmodel.exogene }
|
|
187
|
+
dummy_d = { newname : 'Exo dummy:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_D') in self.mmodel.exogene }
|
|
188
|
+
exo_d = { newname : 'Exo value:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_X') in self.mmodel.exogene }
|
|
189
|
+
fitted_d = { newname : 'Fitted value:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_FITTED') in self.mmodel.endogene }
|
|
190
|
+
var_description = {**var_description,**add_d,**dummy_d,**exo_d,**fitted_d}
|
|
191
|
+
self.mmodel.set_var_description(var_description)
|
|
192
|
+
except:
|
|
193
|
+
print('*** No variable description',flush=True)
|
|
194
|
+
var_description = {}
|
|
195
|
+
return var_description
|
|
196
|
+
|
|
197
|
+
@functools.cached_property
|
|
198
|
+
def mfmsa_options(self):
|
|
199
|
+
'''Grab the mfmsa options, a world bank speciality'''
|
|
200
|
+
if self.mfmsa:
|
|
201
|
+
with open(self.mfmsa,'rt') as f:
|
|
202
|
+
options = f.read()
|
|
203
|
+
else:
|
|
204
|
+
options = ''
|
|
205
|
+
return options
|
|
206
|
+
|
|
207
|
+
@functools.cached_property
|
|
208
|
+
def mfmsa_start_end(self):
|
|
209
|
+
import xml
|
|
210
|
+
root = xml.etree.ElementTree.fromstring(self.mfmsa_options)
|
|
211
|
+
start = int(root.find('iFace').find('SolveStart').text)
|
|
212
|
+
end = int(root.find('iFace').find('SolveEnd').text)
|
|
213
|
+
return start,end
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@functools.cached_property
|
|
217
|
+
def dfmodel(self):
|
|
218
|
+
'''The original input data enriched with during variablees, variables containing
|
|
219
|
+
values for specific historic years and model specific transformation '''
|
|
220
|
+
# Now the data
|
|
221
|
+
if self.from_wf2:
|
|
222
|
+
df = pd.read_excel(self.data,index_col=0)
|
|
223
|
+
else:
|
|
224
|
+
df = (pd.read_excel(self.data).
|
|
225
|
+
pipe( lambda df : df.rename(columns={c:c.upper() for c in df.columns})).
|
|
226
|
+
pipe( lambda df : df.rename(columns={'_DATE_':'DATEID'})).
|
|
227
|
+
pipe( lambda df : df.set_index('DATEID'))
|
|
228
|
+
)
|
|
229
|
+
df.index = [int(i.year) for i in df.index]
|
|
230
|
+
|
|
231
|
+
try:
|
|
232
|
+
sca = pd.read_excel(self.scalars ,index_col=0,header=None).T.pipe(
|
|
233
|
+
lambda _df : _df.loc[_df.index.repeat(len(df.index)),:]).\
|
|
234
|
+
set_index(df.index)
|
|
235
|
+
df= pd.concat([df,sca],axis=1)
|
|
236
|
+
except:
|
|
237
|
+
print(f'{self.modelname} no Scalars prowided ')
|
|
238
|
+
# breakpoint()
|
|
239
|
+
if self.make_fitted:
|
|
240
|
+
df = self.mfitmodel.res(df,self.fit_start,self.fit_end)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
#% Now set the vars with fixedvalues
|
|
244
|
+
value_vars = self.mmodel.vlist('*_value_*')
|
|
245
|
+
for var,val,year in (v.rsplit('_',2) for v in value_vars) :
|
|
246
|
+
df.loc[:,f'{var}_{val}_{year}'] = df.loc[int(year),var]
|
|
247
|
+
self.showvaluevars = df[value_vars]
|
|
248
|
+
|
|
249
|
+
#% now set the values of the dummies
|
|
250
|
+
# breakpoint()
|
|
251
|
+
during_vars = self.mmodel.vlist('*during_*')
|
|
252
|
+
for varname,(dur,per) in ((v,v.split('_',1)) for v in during_vars):
|
|
253
|
+
df.loc[:,varname]=0
|
|
254
|
+
# print(varname,dur,per)
|
|
255
|
+
pers = per.split('_')
|
|
256
|
+
if len(pers) == 1:
|
|
257
|
+
df.loc[int(pers[0]),varname] = 1
|
|
258
|
+
else:
|
|
259
|
+
df.loc[int(pers[0]):int(pers[1]),varname]=1.
|
|
260
|
+
self.showduringvars = df[during_vars]
|
|
261
|
+
# breakpoint()
|
|
262
|
+
df_out = self.mmodel.insertModelVar(df).pipe(self.country_df_trans).fillna(0.0)
|
|
263
|
+
return df_out
|
|
264
|
+
|
|
265
|
+
def __call__(self):
|
|
266
|
+
|
|
267
|
+
return self.mmodel,self.base_input
|
|
268
|
+
|
|
269
|
+
def test_model(self,start=None,end=None,maxvar=1_000_000, maxerr=100,tol=0.0001,showall=False):
|
|
270
|
+
'''
|
|
271
|
+
Compares a straight calculation with the input dataframe.
|
|
272
|
+
|
|
273
|
+
shows which variables dont have the same value
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
df (TYPE): dataframe to run.
|
|
277
|
+
start (TYPE, optional): start period. Defaults to None.
|
|
278
|
+
end (TYPE, optional): end period. Defaults to None.
|
|
279
|
+
maxvar (TYPE, optional): how many variables are to be chekked. Defaults to 1_000_000.
|
|
280
|
+
maxerr (TYPE, optional): how many errors to check Defaults to 100.
|
|
281
|
+
tol (TYPE, optional): check for absolute value of difference. Defaults to 0.0001.
|
|
282
|
+
showall (TYPE, optional): show more . Defaults to False.
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
None.
|
|
286
|
+
|
|
287
|
+
'''
|
|
288
|
+
_start = start if start else self.start
|
|
289
|
+
_end = end if end else self.end
|
|
290
|
+
# breakpoint()
|
|
291
|
+
|
|
292
|
+
resresult = self.mmodel(self.base_input,_start,_end,reset_options=True,silent=0,solver='base_res')
|
|
293
|
+
self.mmodel.basedf = self.dfmodel
|
|
294
|
+
pd.options.display.float_format = '{:.10f}'.format
|
|
295
|
+
err=0
|
|
296
|
+
print(f'\nChekking residuals for {self.mmodel.name} {_start} to {_end}')
|
|
297
|
+
for i,v in enumerate(self.mmodel.solveorder):
|
|
298
|
+
# if v.endswith('_FITTED'): continue
|
|
299
|
+
if i > maxvar : break
|
|
300
|
+
if err > maxerr : break
|
|
301
|
+
check = self.mmodel.get_values(v,pct=True).T
|
|
302
|
+
check.columns = ['Before check','After calculation','Difference','Pct']
|
|
303
|
+
# breakpoint()
|
|
304
|
+
if (check.Difference.abs() >= tol).any():
|
|
305
|
+
err=err+1
|
|
306
|
+
maxdiff = check.Difference.abs().max()
|
|
307
|
+
maxpct = check.Pct.abs().max()
|
|
308
|
+
# breakpoint()
|
|
309
|
+
print('\nVariable with residuals above threshold')
|
|
310
|
+
print(f"{v}, Max difference:{maxdiff:15.8f} Max Pct {maxpct:15.10f}% It is number {i} in the solveorder and error number {err}")
|
|
311
|
+
if showall:
|
|
312
|
+
print(f'\n{self.mmodel.allvar[v]["frml"]}')
|
|
313
|
+
print(f'\nResult of equation \n {check}')
|
|
314
|
+
print(f'\nEquation values before calculations: \n {self.mmodel.get_eq_values(v,last=False,showvar=1)} \n')
|
|
315
|
+
self.mmodel.oldkwargs = {}
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
|