ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelgrab.py ADDED
@@ -0,0 +1,318 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Mon Jun 10 21:11:08 2019
4
+
5
+ @author: hanseni
6
+
7
+ modules to grab models with different specifications and make them ModelFlow conforme
8
+
9
+ **GrabWbModel** will take a eviews model and transform it to Business logic
10
+ - Create a normalized model, add dampning for the stocastic equations
11
+ - Add add-factors to the stocastic equations
12
+ - Generate BL for a model which calculates add-factors so a solution will match teh existing values
13
+ - Generate BL for the model
14
+
15
+ -grap data from excel sheet
16
+
17
+ - Make model instance for model and add-factor model
18
+
19
+ - Run the model, check that the results match.
20
+
21
+ For debuggging valuesthe last part checs value in the order, in which they are calculated,
22
+ and then displays the input to off mark equations
23
+ """
24
+
25
+ import pandas as pd
26
+ import re
27
+ from dataclasses import dataclass
28
+ import functools
29
+ from tqdm import tqdm
30
+ import json
31
+ from pathlib import Path
32
+
33
+
34
+ from modelclass import model
35
+ import modelmf
36
+ import modelmanipulation as mp
37
+
38
+ import modelnormalize as nz
39
+
40
+ assert 1==1
41
+
42
+
43
+ @dataclass
44
+ class GrapWbModel():
45
+ '''This class takes a world bank model specification, variable data and variable description
46
+ and transform it to ModelFlow business language'''
47
+
48
+
49
+ frml : str = '' # path to model
50
+ data : str = '' # path to data
51
+ des : any = '' # path to descriptions
52
+ scalars : str = '' # path to scalars
53
+ modelname : str = 'No Name' # modelname
54
+ start : int = 2017
55
+ end : int = 2040
56
+ country_trans : any = lambda x:x[:] # function which transform model specification
57
+ country_df_trans : any = lambda x:x # function which transforms initial dataframe
58
+ from_wf2 : bool = False
59
+ make_fitted : bool = False # if True, a clean equation for fittet variables is created
60
+ fit_start : int =2000 # start of fittet model unless overruled by mfmsa
61
+ fit_end : int = 2100 # end of fittet model unless overruled by mfmsa
62
+ do_add_factor_calc : bool = True # calculate the add factors
63
+ mfmsa : str = '' # path to mfsa options
64
+
65
+
66
+ def __post_init__(self):
67
+ # breakpoint()
68
+
69
+ print(f'\nProcessing the model:{self.modelname}',flush=True)
70
+ self.rawmodel_org = open(self.frml).read()
71
+ eviewsline = [l for l in self.rawmodel_org.split('\n') if len(l.strip()) >=2
72
+ and not ('*******' in l or '----------' in l)]
73
+ # breakpoint()
74
+ self.rawmodel = self.country_trans(self.rawmodel_org)
75
+ rawmodel6 = self.trans_eviews(self.rawmodel)
76
+ # breakpoint()
77
+ bars = '{desc}: {percentage:3.0f}%|{bar}|{n_fmt}/{total_fmt}'
78
+ if self.from_wf2:
79
+
80
+ orgline = [l for l in rawmodel6.split('\n')]
81
+ line_type = ['ident' if l.startswith('@IDENTITY') else 'stoc' for l in orgline]
82
+ line = [l.replace('@IDENTITY ','').replace(' ','') for l in orgline]
83
+ else:
84
+ line_type = []
85
+ line =[]
86
+ with tqdm(total=len(rawmodel6.split('\n')),desc='Reading original model',bar_format=bars) as pbar:
87
+ for l in rawmodel6.split('\n'):
88
+ if ('*******' in l or '----------' in l) and 'IDEN' in l.upper():
89
+ sec='iden'
90
+ #print(l)
91
+ elif '*******' in l and 'STOC' in l:
92
+ sec='stoc'
93
+ #print(l)
94
+ else:
95
+ line_type.append(sec)
96
+ line.append(l)
97
+ # print(f' {sec} {l[:30]} ....')
98
+ pbar.update(1)
99
+
100
+
101
+ errline1 = [(l,o,e) for l,o,e in zip(line,orgline,eviewsline) if '@' in l or '.coef' in l ]
102
+ if len(errline1):
103
+ print('Probably errors as @ in lines:')
104
+ for l,o,e in errline1:
105
+ print(f'\nEviews line :{e}')
106
+ print(f'Original line :{o}')
107
+ print(f'New modelflow line:{l}')
108
+ raise Exception('@ in lines ')
109
+
110
+ self.all_frml = [nz.normal(l,add_add_factor=(typ=='stoc'),make_fitted=(typ=='stoc'),make_fixable =(typ=='stoc')) for l,typ in tqdm(zip(line,line_type),desc='Normalizing model',total=len(line),bar_format=bars)]
111
+ self.all_frml_dict = {f.endo_var: f for f in self.all_frml}
112
+ lfname = ["<Z,EXO> " if typ == 'stoc' else '' for typ in line_type ]
113
+ self.rorg = [fname + f.normalized for f,fname in zip(self.all_frml,lfname) ]
114
+
115
+ if self.make_fitted:
116
+ self.rfitmodel = ['<FIT> ' + f.fitted for f in self.all_frml if len(f.fitted)]
117
+ self.mfitmodel = model('\n'.join(self.rfitmodel))
118
+ self.mfitmodel.modelname = self.modelname + ' calc fittet values'
119
+ else:
120
+ self.rfitmodel = []
121
+
122
+ self.rres = [f'{f.calc_add_factor}' for f in self.all_frml if len(f.calc_add_factor)]
123
+ self.rres_tomodel ='\n'.join([f'FRML <CALC_ADD_FACTOR> {f.calc_add_factor}$' for f in self.all_frml if len(f.calc_add_factor)])
124
+ # self.fmodel = mp.exounroll(mp.tofrml ('\n'.join(self.rorg+self.rfitmodel)))+self.rres_tomodel
125
+ self.fmodel = mp.tofrml ('\n'.join(self.rorg+self.rfitmodel))+self.rres_tomodel
126
+ # breakpoint()
127
+ self.fres = ('\n'.join(self.rres))
128
+ self.mmodel = model(self.fmodel,modelname = self.modelname)
129
+ self.mmodel.set_var_description(self.var_description)
130
+ self.mmodel.wb_MFMSAOPTIONS = self.mfmsa_options
131
+ self.mres = model(self.fres,modelname = f'Calculation of add factors for {self.modelname}')
132
+ # breakpoint()
133
+ self.start,self.end = self.mfmsa_start_end
134
+ if self.do_add_factor_calc:
135
+ self.base_input = self.mres.res(self.dfmodel,self.start,self.end)
136
+ else:
137
+ self.base_input = self.dfmodel
138
+
139
+ @staticmethod
140
+ def trans_eviews(rawmodel):
141
+ rawmodel0 = '\n'.join(l for l in rawmodel.upper().split('\n') if len(l.strip()) >=2)
142
+ # trailing and leading "
143
+ rawmodel1 = '\n'.join(l[1:-1] if l.startswith('"') else l for l in rawmodel0.split('\n'))
144
+ # powers
145
+ rawmodel2 = rawmodel1.replace('^','**').replace('""',' ').replace('"',' ').\
146
+ replace('@EXP','exp').replace('@RECODE','recode').replace('@MOVAV','movavg').replace('@LOGIT','logit_inverse') \
147
+ .replace('@MEAN(@PC(','@AVERAGE_GROWTH((').replace('@PCY','PCT_GROWTH').replace('@PC','PCT_GROWTH')\
148
+ .replace('@PMAX','MAX').replace('@TREND','EVIEWS_TREND')
149
+ # @ELEM and @DURING
150
+ # @ELEM and @DURING
151
+ rawmodel3 = nz.elem_trans(rawmodel2)
152
+ rawmodel4 = re.sub(r'@DURING\( *([0-9]+) *\)', r'during_\1',rawmodel3)
153
+ rawmodel5 = re.sub(r'@DURING\( *([0-9]+) *([0-9]+) *\)', r'during_\1_\2',rawmodel4)
154
+
155
+ # during check
156
+ ldur = '\n'.join(l for l in rawmodel5.split('\n') if '@DURING' in l)
157
+ ldur2 = '\n'.join(l for l in rawmodel5.split('\n') if 'during' in l)
158
+
159
+ # check D(
160
+ ld = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z]|^)D\(',l) )
161
+ ld1 = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z0-9_]|^)D\(',l) )
162
+ # breakpoint()
163
+ rawmodel6 = nz.funk_replace('D','DIFF',rawmodel5)
164
+ # did we get all the lines
165
+ ldif = '\n'.join(l for l in rawmodel6.split('\n') if 'DIFF(' in l )
166
+ return rawmodel6
167
+
168
+ @property
169
+ def var_description(self):
170
+ '''
171
+ Adds var descriptions for add factors, exogenizing dummies and exoggenizing values
172
+ '''
173
+
174
+ if isinstance(self.des,dict):
175
+ return self.des
176
+
177
+ try:
178
+ # breakpoint()
179
+ des_file = Path(self.des)
180
+ if des_file.suffix == '.xlsx':
181
+ trans0 = pd.read_excel(self.des).loc[:,['mnem','Excel']].set_index('mnem').to_dict(orient = 'dict')['Excel']
182
+ var_description = {str(k) : str(v) for k,v in trans0.items() if 'nan' != str(v)}
183
+ else:
184
+ with open(des_file,'rt') as f:
185
+ var_description = json.load(f)
186
+ add_d = { newname : 'Add factor:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_A') in self.mmodel.exogene }
187
+ dummy_d = { newname : 'Exo dummy:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_D') in self.mmodel.exogene }
188
+ exo_d = { newname : 'Exo value:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_X') in self.mmodel.exogene }
189
+ fitted_d = { newname : 'Fitted value:'+ var_description.get(v,v) for v in self.mmodel.endogene if (newname := v+'_FITTED') in self.mmodel.endogene }
190
+ var_description = {**var_description,**add_d,**dummy_d,**exo_d,**fitted_d}
191
+ self.mmodel.set_var_description(var_description)
192
+ except:
193
+ print('*** No variable description',flush=True)
194
+ var_description = {}
195
+ return var_description
196
+
197
+ @functools.cached_property
198
+ def mfmsa_options(self):
199
+ '''Grab the mfmsa options, a world bank speciality'''
200
+ if self.mfmsa:
201
+ with open(self.mfmsa,'rt') as f:
202
+ options = f.read()
203
+ else:
204
+ options = ''
205
+ return options
206
+
207
+ @functools.cached_property
208
+ def mfmsa_start_end(self):
209
+ import xml
210
+ root = xml.etree.ElementTree.fromstring(self.mfmsa_options)
211
+ start = int(root.find('iFace').find('SolveStart').text)
212
+ end = int(root.find('iFace').find('SolveEnd').text)
213
+ return start,end
214
+
215
+
216
+ @functools.cached_property
217
+ def dfmodel(self):
218
+ '''The original input data enriched with during variablees, variables containing
219
+ values for specific historic years and model specific transformation '''
220
+ # Now the data
221
+ if self.from_wf2:
222
+ df = pd.read_excel(self.data,index_col=0)
223
+ else:
224
+ df = (pd.read_excel(self.data).
225
+ pipe( lambda df : df.rename(columns={c:c.upper() for c in df.columns})).
226
+ pipe( lambda df : df.rename(columns={'_DATE_':'DATEID'})).
227
+ pipe( lambda df : df.set_index('DATEID'))
228
+ )
229
+ df.index = [int(i.year) for i in df.index]
230
+
231
+ try:
232
+ sca = pd.read_excel(self.scalars ,index_col=0,header=None).T.pipe(
233
+ lambda _df : _df.loc[_df.index.repeat(len(df.index)),:]).\
234
+ set_index(df.index)
235
+ df= pd.concat([df,sca],axis=1)
236
+ except:
237
+ print(f'{self.modelname} no Scalars prowided ')
238
+ # breakpoint()
239
+ if self.make_fitted:
240
+ df = self.mfitmodel.res(df,self.fit_start,self.fit_end)
241
+
242
+
243
+ #% Now set the vars with fixedvalues
244
+ value_vars = self.mmodel.vlist('*_value_*')
245
+ for var,val,year in (v.rsplit('_',2) for v in value_vars) :
246
+ df.loc[:,f'{var}_{val}_{year}'] = df.loc[int(year),var]
247
+ self.showvaluevars = df[value_vars]
248
+
249
+ #% now set the values of the dummies
250
+ # breakpoint()
251
+ during_vars = self.mmodel.vlist('*during_*')
252
+ for varname,(dur,per) in ((v,v.split('_',1)) for v in during_vars):
253
+ df.loc[:,varname]=0
254
+ # print(varname,dur,per)
255
+ pers = per.split('_')
256
+ if len(pers) == 1:
257
+ df.loc[int(pers[0]),varname] = 1
258
+ else:
259
+ df.loc[int(pers[0]):int(pers[1]),varname]=1.
260
+ self.showduringvars = df[during_vars]
261
+ # breakpoint()
262
+ df_out = self.mmodel.insertModelVar(df).pipe(self.country_df_trans).fillna(0.0)
263
+ return df_out
264
+
265
+ def __call__(self):
266
+
267
+ return self.mmodel,self.base_input
268
+
269
+ def test_model(self,start=None,end=None,maxvar=1_000_000, maxerr=100,tol=0.0001,showall=False):
270
+ '''
271
+ Compares a straight calculation with the input dataframe.
272
+
273
+ shows which variables dont have the same value
274
+
275
+ Args:
276
+ df (TYPE): dataframe to run.
277
+ start (TYPE, optional): start period. Defaults to None.
278
+ end (TYPE, optional): end period. Defaults to None.
279
+ maxvar (TYPE, optional): how many variables are to be chekked. Defaults to 1_000_000.
280
+ maxerr (TYPE, optional): how many errors to check Defaults to 100.
281
+ tol (TYPE, optional): check for absolute value of difference. Defaults to 0.0001.
282
+ showall (TYPE, optional): show more . Defaults to False.
283
+
284
+ Returns:
285
+ None.
286
+
287
+ '''
288
+ _start = start if start else self.start
289
+ _end = end if end else self.end
290
+ # breakpoint()
291
+
292
+ resresult = self.mmodel(self.base_input,_start,_end,reset_options=True,silent=0,solver='base_res')
293
+ self.mmodel.basedf = self.dfmodel
294
+ pd.options.display.float_format = '{:.10f}'.format
295
+ err=0
296
+ print(f'\nChekking residuals for {self.mmodel.name} {_start} to {_end}')
297
+ for i,v in enumerate(self.mmodel.solveorder):
298
+ # if v.endswith('_FITTED'): continue
299
+ if i > maxvar : break
300
+ if err > maxerr : break
301
+ check = self.mmodel.get_values(v,pct=True).T
302
+ check.columns = ['Before check','After calculation','Difference','Pct']
303
+ # breakpoint()
304
+ if (check.Difference.abs() >= tol).any():
305
+ err=err+1
306
+ maxdiff = check.Difference.abs().max()
307
+ maxpct = check.Pct.abs().max()
308
+ # breakpoint()
309
+ print('\nVariable with residuals above threshold')
310
+ print(f"{v}, Max difference:{maxdiff:15.8f} Max Pct {maxpct:15.10f}% It is number {i} in the solveorder and error number {err}")
311
+ if showall:
312
+ print(f'\n{self.mmodel.allvar[v]["frml"]}')
313
+ print(f'\nResult of equation \n {check}')
314
+ print(f'\nEquation values before calculations: \n {self.mmodel.get_eq_values(v,last=False,showvar=1)} \n')
315
+ self.mmodel.oldkwargs = {}
316
+
317
+
318
+