ModelFlowIb 1.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ModelFlowIb-1.46.dist-info/METADATA +151 -0
- ModelFlowIb-1.46.dist-info/RECORD +48 -0
- ModelFlowIb-1.46.dist-info/WHEEL +5 -0
- ModelFlowIb-1.46.dist-info/top_level.txt +44 -0
- data/ken_base.pcim +1 -0
- modelBLfunk.py +70 -0
- model_Excel.py +334 -0
- model_cvx.py +132 -0
- model_doable.py +40 -0
- model_dynare.py +164 -0
- model_financial_stability.py +88 -0
- model_ifrs9.py +88 -0
- model_latex.py +505 -0
- model_latex_class.py +478 -0
- model_latex_classpy.py +401 -0
- model_run_numba.py +15 -0
- modelclass.py +7057 -0
- modelclass2.py +463 -0
- modeldash.py +265 -0
- modeldashboot.py +202 -0
- modeldashsidebar.py +442 -0
- modeldekom.py +518 -0
- modeldiff.py +561 -0
- modelgrab.py +318 -0
- modelgrabwf2.py +554 -0
- modelhelp.py +253 -0
- modelinvert.py +211 -0
- modeljupyter.py +821 -0
- modeljupytermagic.py +267 -0
- modelmacrograb.py +98 -0
- modelmanipulation.py +1273 -0
- modelmf.py +265 -0
- modelnet.py +114 -0
- modelnewton.py +819 -0
- modelnormalize.py +348 -0
- modelpattern.py +253 -0
- modelpatternrev.py +72 -0
- modelsandbox.py +76 -0
- modelsandbox_Mixin.py +1565 -0
- modeltodo.py +31 -0
- modeluserfunk.py +97 -0
- modeluserfunk_old.py +90 -0
- modelvis.py +677 -0
- modelwidget.py +610 -0
- modelwidget_input.py +961 -0
- modelwidget_new.py +1005 -0
- modelwidget_new_new.py +1238 -0
- modelwidget_shinypy.py +935 -0
modelgrabwf2.py
ADDED
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Created on Wed Mar 30 10:06:26 2022
|
|
4
|
+
|
|
5
|
+
@author: ibhan
|
|
6
|
+
|
|
7
|
+
Module to handle models in wf1 files
|
|
8
|
+
|
|
9
|
+
#. Eviews is started and the wf1 file is loaded.
|
|
10
|
+
|
|
11
|
+
#. Some transformations are performed on data.
|
|
12
|
+
#. The model is unlinked.
|
|
13
|
+
#. The workspace is saved as a wf2 file. Same name with _modelflow appended.
|
|
14
|
+
#. Eviews is closed
|
|
15
|
+
#. The wf2 file is read as a json file.
|
|
16
|
+
#. Relevant objects are extracted.
|
|
17
|
+
#. The MFMSA variable is extracted, to be saved in the dumpfile.
|
|
18
|
+
#. The equations are transformed and normalized to modelflow format and classified into identities and stochastic
|
|
19
|
+
#. Stochastic equations are enriched by add_factor and fixing terms (dummy + fixing value)
|
|
20
|
+
#. For Stochastic equations new fitted variables are generated - without add add_factors and dummies.
|
|
21
|
+
#. A model to generate fitted variables is created
|
|
22
|
+
#. A model to generate add_factors is created.
|
|
23
|
+
#. A model encompassing the original equations, the model for fitted variables and for add_factors is created.
|
|
24
|
+
#. The data series and scalars are shoveled into a Pandas dataframe
|
|
25
|
+
|
|
26
|
+
#. Some special series are generated as the expression can not be incorporated into modelflow model specifications
|
|
27
|
+
#. The model for fitted values is simulated in the specified timespan
|
|
28
|
+
#. The model for add_factors is simulated in the timespan set in MFMSA
|
|
29
|
+
#. The data descriptions are extracted into a dictionary.
|
|
30
|
+
#. Data descriptions for dummies, fixed values, fitted values and add_factors are derived.
|
|
31
|
+
#. Now we have a model and a dataframe with all variables which are needed.
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
import pandas as pd
|
|
38
|
+
import re
|
|
39
|
+
from dataclasses import dataclass, field
|
|
40
|
+
import functools
|
|
41
|
+
from tqdm import tqdm
|
|
42
|
+
import json
|
|
43
|
+
from pathlib import Path
|
|
44
|
+
import gzip
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
from modelclass import model
|
|
48
|
+
import modelmf
|
|
49
|
+
import modelmanipulation as mp
|
|
50
|
+
|
|
51
|
+
import modelnormalize as nz
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def wf1_to_wf2(filename,modelname='',eviews_run_lines= []):
|
|
57
|
+
'''
|
|
58
|
+
- Opens a eviews workfile in wf1 format
|
|
59
|
+
- calculates the eviews_trend
|
|
60
|
+
- unlink a model and
|
|
61
|
+
- writes the workspace back to a wf2 file
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
filename (TYPE): DESCRIPTION.
|
|
65
|
+
modelname (TYPE): default '' then the three first letters of the filenames stem are asumed to be the modelname.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
None.
|
|
69
|
+
|
|
70
|
+
'''
|
|
71
|
+
try:
|
|
72
|
+
import pyeviews as evp
|
|
73
|
+
except:
|
|
74
|
+
raise Exception('You have not pyeviews installed try:\n'+
|
|
75
|
+
'!conda install pyeviews -c conda-forge -y')
|
|
76
|
+
from pathlib import Path
|
|
77
|
+
wfpath = Path(filename)
|
|
78
|
+
if wfpath.suffix not in {'.wf1','.wf2'} :
|
|
79
|
+
raise Exception(f'wf1_to_wf2 expects a .wf1 file as input\nOffending:{wfpath} {wfpath.suffix} ')
|
|
80
|
+
else:
|
|
81
|
+
wf1= wfpath.absolute()
|
|
82
|
+
# wf2 = wf1.with_suffix('.wf2')
|
|
83
|
+
wf2 = (wf1.resolve().parent / (wf1.stem + '_modelflow')).with_suffix('.wf2')
|
|
84
|
+
# breakpoint()
|
|
85
|
+
eviewsapp = evp.GetEViewsApp(instance='new',showwindow=True)
|
|
86
|
+
print(f'\nReading {wf1}')
|
|
87
|
+
if modelname == '':
|
|
88
|
+
modelname = wf1.stem[:3].upper()
|
|
89
|
+
print(f'Assummed model name: {modelname}')
|
|
90
|
+
|
|
91
|
+
evp.Run(fr'wfopen "{wf1}"',eviewsapp)
|
|
92
|
+
evp.Run( r'smpl @all',eviewsapp)
|
|
93
|
+
evp.Run( r'series eviews_trend = @TREND',eviewsapp)
|
|
94
|
+
for eviewsline in eviews_run_lines:
|
|
95
|
+
print(f'Eviewsline to run :{eviewsline}')
|
|
96
|
+
evp.Run(eviewsline,eviewsapp)
|
|
97
|
+
evp.Run(f'{modelname}.unlink @all',eviewsapp)
|
|
98
|
+
print(f'The model: {modelname} is unlinked ')
|
|
99
|
+
evp.Run(fr'wfsave(jf) "{wf2}"',eviewsapp)
|
|
100
|
+
print(f'Writing {wf2}')
|
|
101
|
+
|
|
102
|
+
eviewsapp.hide()
|
|
103
|
+
eviewsapp = None
|
|
104
|
+
evp.Cleanup()
|
|
105
|
+
return wf2,modelname
|
|
106
|
+
|
|
107
|
+
def wf2_to_clean(wf2name,modelname='',save_file = False):
|
|
108
|
+
'''
|
|
109
|
+
Takes a eviews .wf2 file - which is in JSON format - and place a
|
|
110
|
+
dictionary
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
wf2name (TYPE): name of wf2 file .
|
|
114
|
+
modelname (TYPE, optional): Name og model. Defaults to ''.
|
|
115
|
+
save_file (TYPE, optional): save the specification, data and description in a dictionary. Defaults to False.
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
model_all_about (dict): the content of the wf2 file as a dict .
|
|
119
|
+
|
|
120
|
+
'''
|
|
121
|
+
model_all_about = {}
|
|
122
|
+
|
|
123
|
+
if modelname == '':
|
|
124
|
+
modelname = wf2name.stem[:3]
|
|
125
|
+
print(f'Assummed model name: {modelname}')
|
|
126
|
+
else:
|
|
127
|
+
print(f'Model name: {modelname}')
|
|
128
|
+
# with open(wf2name) as f:
|
|
129
|
+
with gzip.open(wf2name, mode="rt") as f:
|
|
130
|
+
|
|
131
|
+
gen0 = f.read()
|
|
132
|
+
# gen0 = gen0.decode("ISO-8859-1")
|
|
133
|
+
all_dict = json.loads(gen0)
|
|
134
|
+
pages = all_dict['_pages']
|
|
135
|
+
first_page = pages[0]
|
|
136
|
+
object_all = [o for o in first_page['_objects']]
|
|
137
|
+
types_set = {o['_type'] for o in object_all}
|
|
138
|
+
object_dict = {_type : [o for o in object_all if o['_type'] == _type ] for _type in types_set}
|
|
139
|
+
object_namecounts = {_type: [o['_name'] for o in this_object] for _type,this_object in object_dict.items() }
|
|
140
|
+
assert 1==1
|
|
141
|
+
# Now extract the model
|
|
142
|
+
thismodel_dict = object_dict['model'][0]
|
|
143
|
+
thismodel_raw = thismodel_dict['data'][0]
|
|
144
|
+
thismodel_raw_list = [l for l in thismodel_raw.split('\n') if len(l) > 1]
|
|
145
|
+
|
|
146
|
+
this_clean = [l for l in thismodel_raw_list if l[:4] not in {'@INN','@ADD'}] # The original frmls
|
|
147
|
+
this_add_vars = [l.split()[1] for l in thismodel_raw_list if l[:4] in {'@ADD'}] # variable with add factors
|
|
148
|
+
this_frml = '\n'.join(this_clean)
|
|
149
|
+
# Take all series and make a dataframe
|
|
150
|
+
index = [int(d[:4]) for d in object_dict['index'][0]['data']]
|
|
151
|
+
series_list = object_dict['series_double']
|
|
152
|
+
series_data = [pd.Series(s['data'],name=s['_name'],index=index) for s in series_list]
|
|
153
|
+
wf_df = pd.concat(series_data,axis=1)
|
|
154
|
+
|
|
155
|
+
var_description = {s['_name']: lab2 for s in series_list
|
|
156
|
+
if (lab2 := s.get('_labels',{}).get('description',''))
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
scalar_list = object_dict['scalar']
|
|
161
|
+
# for scalar in scalar_list:
|
|
162
|
+
# if 'value' in scalar.keys():
|
|
163
|
+
# wf_df.loc[:,scalar['_name']] = scalar['value']
|
|
164
|
+
scalar_data = [pd.Series(s['value'],name=s['_name'],index=index)
|
|
165
|
+
for s in scalar_list if 'value' in s.keys()]
|
|
166
|
+
# breakpoint()
|
|
167
|
+
wf_df = pd.concat([wf_df]+scalar_data,axis=1)
|
|
168
|
+
string_list = object_dict['stringobj']
|
|
169
|
+
# breakpoint()
|
|
170
|
+
mfmsa_dict = {o['_name']:o.get('value','empty') for o in string_list if o.get('_name','').startswith('MFMSA') }
|
|
171
|
+
mfmsa_options = mfmsa_dict.get('MFMSAOPTIONS','')
|
|
172
|
+
|
|
173
|
+
estimates = {eq['_name'].upper()+f'.@COEF({idata+1})': str(data)
|
|
174
|
+
for eq in object_dict['equation7'] for idata,data in enumerate(eq['results']['data']) }
|
|
175
|
+
|
|
176
|
+
# breakpoint()
|
|
177
|
+
model_all_about['modelname'] = modelname
|
|
178
|
+
model_all_about['frml'] = this_frml
|
|
179
|
+
model_all_about['mfmsa_options'] = mfmsa_options
|
|
180
|
+
model_all_about['var_description'] = var_description
|
|
181
|
+
model_all_about['data'] = wf_df
|
|
182
|
+
model_all_about['object_dict_from_wf'] = object_dict
|
|
183
|
+
model_all_about['estimates'] = estimates
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
if save_file:
|
|
187
|
+
with open(f'{wf2name.parent / modelname}_clean.frm','wt') as f:
|
|
188
|
+
f.writelines(this_clean)
|
|
189
|
+
with open(f'{wf2name.parent / modelname}_mfmsa_options','wt') as f:
|
|
190
|
+
f.write(mfmsa_options)
|
|
191
|
+
with open(f'{wf2name.parent / modelname}_var_descriptions.json','wt') as f:
|
|
192
|
+
json.dump(var_description,f,indent=4)
|
|
193
|
+
wf_df.to_excel(f'{wf2name.parent / modelname}soln.xlsx')
|
|
194
|
+
print(f'{wf2name.parent / modelname}.... files are saved')
|
|
195
|
+
|
|
196
|
+
return model_all_about
|
|
197
|
+
|
|
198
|
+
@dataclass
|
|
199
|
+
class GrabWfModel():
|
|
200
|
+
'''This class takes a world bank model specification, variable data and variable description
|
|
201
|
+
and transform it to ModelFlow business language
|
|
202
|
+
|
|
203
|
+
args:
|
|
204
|
+
filename : any = '' #wf1 name
|
|
205
|
+
modelname : any = ''
|
|
206
|
+
eviews_run_lines : list =field(default_factory=list)
|
|
207
|
+
model_all_about : dict = field(default_factory=dict)
|
|
208
|
+
start : any = None # start of testing if not overruled by mfmsa
|
|
209
|
+
end : any = None # end of testing if not overruled by mfmsa
|
|
210
|
+
country_trans : any = lambda x:x[:] # function which transform model specification
|
|
211
|
+
country_df_trans : any = lambda x:x # function which transforms initial dataframe
|
|
212
|
+
make_fitted : bool = False # if True, a clean equation for fittet variables is created
|
|
213
|
+
fit_start : any = 2000 # start of fittet model
|
|
214
|
+
fit_end : any = None # end of fittet model unless overruled by mfmsa
|
|
215
|
+
do_add_factor_calc : bool = True # calculate the add factors
|
|
216
|
+
test_frml : str ='' # a testmodel as string if used no wf processing
|
|
217
|
+
disable_progress : bool = False # Disable progress bar
|
|
218
|
+
|
|
219
|
+
'''
|
|
220
|
+
|
|
221
|
+
filename : any = '' #wf1 name
|
|
222
|
+
modelname : any = ''
|
|
223
|
+
eviews_run_lines : list =field(default_factory=list)
|
|
224
|
+
model_all_about : dict = field(default_factory=dict,init=False)
|
|
225
|
+
start : any = None # start of testing if not overruled by mfmsa
|
|
226
|
+
end : any = None # end of testing if not overruled by mfmsa
|
|
227
|
+
country_trans : any = lambda x:x[:] # function which transform model specification
|
|
228
|
+
country_df_trans : any = lambda x:x # function which transforms initial dataframe
|
|
229
|
+
make_fitted : bool = False # if True, a clean equation for fittet variables is created
|
|
230
|
+
fit_start : any = 2000 # start of fittet model
|
|
231
|
+
fit_end : any = None # end of fittet model unless overruled by mfmsa
|
|
232
|
+
do_add_factor_calc : bool = True # calculate the add factors
|
|
233
|
+
test_frml : str ='' # a testmodel as string if used no wf processing
|
|
234
|
+
disable_progress : bool = False # Disable progress bar
|
|
235
|
+
|
|
236
|
+
def __post_init__(self):
|
|
237
|
+
'''Process the model'''
|
|
238
|
+
|
|
239
|
+
if self.test_frml:
|
|
240
|
+
self.rawmodel_org =self.test_frml
|
|
241
|
+
else:
|
|
242
|
+
# breakpoint()
|
|
243
|
+
wf2name,self.modelname = wf1_to_wf2(self.filename ,modelname=self.modelname,eviews_run_lines= self.eviews_run_lines)
|
|
244
|
+
self.model_all_about = wf2_to_clean(wf2name,modelname=self.modelname)
|
|
245
|
+
|
|
246
|
+
print(f'\nProcessing the model:{self.modelname}',flush=True)
|
|
247
|
+
self.rawmodel_org = self.model_all_about['frml']
|
|
248
|
+
|
|
249
|
+
eviewsline = self.rawmodel_org.split('\n')[:]
|
|
250
|
+
|
|
251
|
+
if '.@coef(' in self.rawmodel_org :
|
|
252
|
+
print('Estimated coifficients are substituted')
|
|
253
|
+
self.rawmodel_org = self.rawmodel_org.upper()
|
|
254
|
+
for k,v in self.model_all_about['estimates'].items():
|
|
255
|
+
self.rawmodel_org = self.rawmodel_org.replace(k,v)
|
|
256
|
+
|
|
257
|
+
self.rawmodel = self.country_trans(self.rawmodel_org)
|
|
258
|
+
rawmodel6 = self.trans_eviews(self.rawmodel)
|
|
259
|
+
bars = '{desc}: {percentage:3.0f}%|{bar}|{n_fmt}/{total_fmt}'
|
|
260
|
+
|
|
261
|
+
orgline = rawmodel6.split('\n')
|
|
262
|
+
line_type = ['ident' if l.startswith('@IDENTITY') else 'stoc' for l in orgline]
|
|
263
|
+
line = [l.replace('@IDENTITY ','').replace(' ','') for l in orgline]
|
|
264
|
+
|
|
265
|
+
# breakpoint()
|
|
266
|
+
|
|
267
|
+
errline1 = [(l,o,e) for l,o,e in zip(line,orgline,eviewsline) if '@' in l or '.coef' in l ]
|
|
268
|
+
if errline1:
|
|
269
|
+
print('Probably errors as @ in lines:')
|
|
270
|
+
for l,o,e in errline1:
|
|
271
|
+
print(f'\nEviews line :{e}')
|
|
272
|
+
print(f'Original line :{o}')
|
|
273
|
+
print(f'New modelflow line:{l}')
|
|
274
|
+
raise Exception('@ in lines ')
|
|
275
|
+
|
|
276
|
+
self.all_frml = [nz.normal(l,add_add_factor=(typ=='stoc'),make_fitted=(typ=='stoc'),make_fixable =(typ=='stoc'),eviews=e)
|
|
277
|
+
for l,typ,e in tqdm(zip(line,line_type,eviewsline),
|
|
278
|
+
desc='Normalizing model',total=len(line),bar_format=bars,disable=self.disable_progress)]
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# print([ f.normalized for f in self.all_frml ])
|
|
282
|
+
self.all_frml_dict = {f.endo_var: f for f in self.all_frml}
|
|
283
|
+
lfname = ["<Z,EXO> " if typ == 'stoc' else '' for typ in line_type ]
|
|
284
|
+
self.rorg = [fname + f.normalized for f,fname in zip(self.all_frml,lfname) ]
|
|
285
|
+
# breakpoint()
|
|
286
|
+
if self.make_fitted:
|
|
287
|
+
self.rfitmodel = ['<FIT> ' + f.fitted for f in self.all_frml if len(f.fitted)]
|
|
288
|
+
self.mfitmodel = model('\n'.join(self.rfitmodel))
|
|
289
|
+
self.mfitmodel.modelname = self.modelname + ' calc fittet values'
|
|
290
|
+
else:
|
|
291
|
+
self.rfitmodel = []
|
|
292
|
+
|
|
293
|
+
self.rres = [f'{f.calc_add_factor}' for f in self.all_frml if len(f.calc_add_factor)]
|
|
294
|
+
self.rres_tomodel ='\n'.join([f'FRML <CALC_ADD_FACTOR> {f.calc_add_factor}$' for f in self.all_frml if len(f.calc_add_factor)])
|
|
295
|
+
# self.fmodel = mp.exounroll(mp.tofrml ('\n'.join(self.rorg+self.rfitmodel)))+self.rres_tomodel
|
|
296
|
+
self.fmodel = mp.tofrml ('\n'.join(self.rorg+self.rfitmodel))+self.rres_tomodel
|
|
297
|
+
# breakpoint()
|
|
298
|
+
self.fres = ('\n'.join(self.rres))
|
|
299
|
+
|
|
300
|
+
self.mmodel = model(self.fmodel,modelname =self.model_all_about['modelname'])
|
|
301
|
+
# self.mmodel.set_var_description(self.model_all_about['var_description'])
|
|
302
|
+
self.mmodel.set_var_description(self.var_description)
|
|
303
|
+
self.mmodel.wb_MFMSAOPTIONS = self.model_all_about['mfmsa_options']
|
|
304
|
+
self.mres = model(self.fres,modelname = f'Calculation of add factors for {self.model_all_about["modelname"]}')
|
|
305
|
+
# breakpoint()
|
|
306
|
+
|
|
307
|
+
try:
|
|
308
|
+
temp_start,temp_end = self.mfmsa_start_end
|
|
309
|
+
except:
|
|
310
|
+
if type(self.start) == type(None) or type(self.end) == type(None):
|
|
311
|
+
raise Exception('Can not read start and end from MFMSA. Provide start and end in call ')
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
self.start = temp_start if type(self.start) == type(None) else self.start
|
|
315
|
+
self.end = temp_end if type(self.end) == type(None) else self.end
|
|
316
|
+
if self.do_add_factor_calc:
|
|
317
|
+
self.base_input = self.mres.res(self.dfmodel,self.start,self.end)
|
|
318
|
+
else:
|
|
319
|
+
self.base_input = self.dfmodel
|
|
320
|
+
|
|
321
|
+
@staticmethod
|
|
322
|
+
def trans_eviews(rawmodel):
|
|
323
|
+
'''
|
|
324
|
+
Takes Eviews specifications and wrangle them into modelflow specifications
|
|
325
|
+
|
|
326
|
+
Args:
|
|
327
|
+
rawmodel (TYPE): a raw model .
|
|
328
|
+
|
|
329
|
+
Returns:
|
|
330
|
+
rawmodel6 (TYPE): a model with the appropiate eviews transformations.
|
|
331
|
+
|
|
332
|
+
'''
|
|
333
|
+
rawmodel0 = '\n'.join(l for l in rawmodel.upper().split('\n') if len(l.strip()) >=2)
|
|
334
|
+
# trailing and leading "
|
|
335
|
+
rawmodel1 = '\n'.join(l[1:-1] if l.startswith('"') else l for l in rawmodel0.split('\n'))
|
|
336
|
+
# powers
|
|
337
|
+
rawmodel2 = rawmodel1.replace('^','**').replace('""',' ').replace('"',' ').\
|
|
338
|
+
replace('@EXP','exp').replace('@RECODE','recode').replace('@MOVAV','movavg').replace('@LOGIT','logit_inverse') \
|
|
339
|
+
.replace('@MEAN(@PC(','@AVERAGE_GROWTH((').replace('@PCY','PCT_GROWTH').replace('@PC','PCT_GROWTH')\
|
|
340
|
+
.replace('@PMAX','MAX').replace('@TREND','EVIEWS_TREND')
|
|
341
|
+
# @ELEM and @DURING
|
|
342
|
+
# @ELEM and @DURING
|
|
343
|
+
rawmodel3 = nz.elem_trans(rawmodel2)
|
|
344
|
+
rawmodel4 = re.sub(r'@DURING\( *([0-9]+) *\)', r'during_\1',rawmodel3)
|
|
345
|
+
rawmodel5 = re.sub(r'@DURING\( *([0-9]+) *([0-9]+) *\)', r'during_\1_\2',rawmodel4)
|
|
346
|
+
|
|
347
|
+
# during check
|
|
348
|
+
ldur = '\n'.join(l for l in rawmodel5.split('\n') if '@DURING' in l)
|
|
349
|
+
ldur2 = '\n'.join(l for l in rawmodel5.split('\n') if 'during' in l)
|
|
350
|
+
|
|
351
|
+
# check D(
|
|
352
|
+
ld = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z]|^)D\(',l) )
|
|
353
|
+
ld1 = '\n'.join(l for l in rawmodel5.split('\n') if re.search(r'([^A-Z0-9_]|^)D\(',l) )
|
|
354
|
+
# breakpoint()
|
|
355
|
+
rawmodel6 = nz.funk_replace('D','DIFF',rawmodel5)
|
|
356
|
+
# did we get all the lines
|
|
357
|
+
ldif = '\n'.join(l for l in rawmodel6.split('\n') if 'DIFF(' in l )
|
|
358
|
+
return rawmodel6
|
|
359
|
+
|
|
360
|
+
@functools.cached_property
|
|
361
|
+
def var_description(self):
|
|
362
|
+
'''
|
|
363
|
+
Adds var descriptions for add factors, exogenizing dummies and exoggenizing values
|
|
364
|
+
'''
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
this = self.model_all_about['var_description']
|
|
370
|
+
this_new = self.mmodel.enrich_var_description(this)
|
|
371
|
+
except:
|
|
372
|
+
print('*** No variable description',flush=True)
|
|
373
|
+
this_new = {}
|
|
374
|
+
return this_new
|
|
375
|
+
|
|
376
|
+
@functools.cached_property
|
|
377
|
+
def mfmsa_options(self):
|
|
378
|
+
'''Grab the mfmsa options, a world bank speciality'''
|
|
379
|
+
options = self.model_all_about.get('mfmsa_options','')
|
|
380
|
+
|
|
381
|
+
return options
|
|
382
|
+
|
|
383
|
+
@functools.cached_property
|
|
384
|
+
def mfmsa_start_end(self):
|
|
385
|
+
'''Finds the start and end from the MFMSA entry'''
|
|
386
|
+
import xml
|
|
387
|
+
root = xml.etree.ElementTree.fromstring(self.mfmsa_options)
|
|
388
|
+
start = (root.find('iFace').find('SolveStart').text)
|
|
389
|
+
end = (root.find('iFace').find('SolveEnd').text)
|
|
390
|
+
return start,end
|
|
391
|
+
|
|
392
|
+
# @property
|
|
393
|
+
|
|
394
|
+
@functools.cached_property
|
|
395
|
+
def dfmodel(self):
|
|
396
|
+
'''The original input data enriched with during variablees, variables containing
|
|
397
|
+
values for specific historic years and model specific transformation '''
|
|
398
|
+
# Now the data
|
|
399
|
+
df = self.model_all_about['data']
|
|
400
|
+
# breakpoint()
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
#% Now set the vars with fixedvalues
|
|
404
|
+
import warnings
|
|
405
|
+
with warnings.catch_warnings():
|
|
406
|
+
warnings.simplefilter(action='ignore')
|
|
407
|
+
|
|
408
|
+
value_vars = self.mmodel.vlist('*_value_*')
|
|
409
|
+
# print(f'{value_vars=}')
|
|
410
|
+
for var,val,year in (v.rsplit('_',2) for v in value_vars) :
|
|
411
|
+
# print(f' {var=} {val=} {year=} {int(year)=} var= {var}_{val}_{year}')
|
|
412
|
+
df.loc[:,f'{var}_{val}_{year}'] = df.loc[int(year),var]
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
self.showvaluevars = df[value_vars]
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
#% now set the values of the dummies
|
|
420
|
+
# breakpoint()
|
|
421
|
+
during_vars = self.mmodel.vlist('*during_*')
|
|
422
|
+
for varname,(dur,per) in ((v,v.split('_',1)) for v in during_vars):
|
|
423
|
+
df.loc[:,varname]=0
|
|
424
|
+
# print(varname,dur,per)
|
|
425
|
+
pers = per.split('_')
|
|
426
|
+
if len(pers) == 1:
|
|
427
|
+
df.loc[int(pers[0]),varname] = 1
|
|
428
|
+
else:
|
|
429
|
+
df.loc[int(pers[0]):int(pers[1]),varname]=1.
|
|
430
|
+
self.showduringvars = df[during_vars]
|
|
431
|
+
|
|
432
|
+
if self.make_fitted:
|
|
433
|
+
# try:
|
|
434
|
+
# temp_start,temp_end = self.mfmsa_start_end
|
|
435
|
+
# except:
|
|
436
|
+
# if type(self.fit_end) == type(None):
|
|
437
|
+
# raise Exception('Can not read end period from MFMSA. Provide fit_end in call ')
|
|
438
|
+
|
|
439
|
+
if type(self.fit_end) == type(None):
|
|
440
|
+
self.fit_end = self.end
|
|
441
|
+
if type(self.fit_start) == type(None):
|
|
442
|
+
self.fit_start = self.start
|
|
443
|
+
df = self.mfitmodel.res(df,self.fit_start,self.fit_end)
|
|
444
|
+
|
|
445
|
+
# breakpoint()
|
|
446
|
+
df_out = self.mmodel.insertModelVar(df).pipe(self.country_df_trans).fillna(0.0)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
return df_out
|
|
450
|
+
|
|
451
|
+
def __call__(self):
|
|
452
|
+
'''returns the model and the base input'''
|
|
453
|
+
return self.mmodel,self.base_input
|
|
454
|
+
|
|
455
|
+
def test_model(self,start=None,end=None,maxvar=1_000_000, maxerr=100,tol=0.0001,showall=False,showinput=False):
|
|
456
|
+
'''
|
|
457
|
+
Compares a straight calculation with the input dataframe.
|
|
458
|
+
|
|
459
|
+
shows which variables dont have the same value
|
|
460
|
+
|
|
461
|
+
Args:
|
|
462
|
+
df (TYPE): dataframe to run.
|
|
463
|
+
start (TYPE, optional): start period. Defaults to None.
|
|
464
|
+
end (TYPE, optional): end period. Defaults to None.
|
|
465
|
+
maxvar (TYPE, optional): how many variables are to be chekked. Defaults to 1_000_000.
|
|
466
|
+
maxerr (TYPE, optional): how many errors to check Defaults to 100.
|
|
467
|
+
tol (TYPE, optional): check for absolute value of difference. Defaults to 0.0001.
|
|
468
|
+
showall (TYPE, optional): show more . Defaults to False.
|
|
469
|
+
showinput (TYPE, optional): show the input values Defaults to False.
|
|
470
|
+
|
|
471
|
+
Returns:
|
|
472
|
+
None.
|
|
473
|
+
|
|
474
|
+
'''
|
|
475
|
+
_start = start if start else self.start
|
|
476
|
+
_end = end if end else self.end
|
|
477
|
+
# breakpoint()
|
|
478
|
+
|
|
479
|
+
resresult = self.mmodel(self.base_input,_start,_end,reset_options=True,silent=0,solver='res')
|
|
480
|
+
self.mmodel.basedf = self.dfmodel
|
|
481
|
+
pd.options.display.float_format = '{:.10f}'.format
|
|
482
|
+
err=0
|
|
483
|
+
pd.set_option('display.max_columns', None)
|
|
484
|
+
pd.set_option("display.width", 1000)
|
|
485
|
+
print(f'\nChekking residuals for {self.mmodel.name} {_start} to {_end}')
|
|
486
|
+
for i,v in enumerate(self.mmodel.solveorder):
|
|
487
|
+
# if v.endswith('_FITTED'): continue
|
|
488
|
+
if i > maxvar : break
|
|
489
|
+
if err > maxerr : break
|
|
490
|
+
check = self.mmodel.get_values(v,pct=True).T
|
|
491
|
+
check.columns = ['Before check','After calculation','Difference','Pct']
|
|
492
|
+
# breakpoint()
|
|
493
|
+
if (check.Difference.abs() >= tol).any():
|
|
494
|
+
err=err+1
|
|
495
|
+
maxdiff = check.Difference.abs().max()
|
|
496
|
+
maxpct = check.Pct.abs().max()
|
|
497
|
+
# breakpoint()
|
|
498
|
+
if err==1 :
|
|
499
|
+
print('\nVariable with residuals above threshold')
|
|
500
|
+
print(f"{v:{self.mmodel.maxnavlen}}, Max difference:{maxdiff:15.8f} Max Pct {maxpct:15.10f}% It is number {i:5} in the solveorder and error number {err}")
|
|
501
|
+
if showall:
|
|
502
|
+
print(f'{self.mmodel.allvar[v]["frml"]}')
|
|
503
|
+
print(f'\nResult of equation \n {check}\n')
|
|
504
|
+
if showinput:
|
|
505
|
+
print(f'\nEquation values before calculations: \n {self.mmodel.get_eq_values(v,last=False,showvar=1)} \n')
|
|
506
|
+
self.mmodel.oldkwargs = {}
|
|
507
|
+
pd.reset_option('max_columns')
|
|
508
|
+
pd.reset_option('max_columns')
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
if __name__ == '__main__':
|
|
512
|
+
#%% Testing
|
|
513
|
+
pak_trans = lambda input : input.replace('- 01*D(','-1*D(')
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
ago_trans = lambda input : input.replace('@MEAN(AGOBNCABFUNDCD/AGONYGDPMKTPCD,"2000 2020")','MEAN_AGOBNCABFUNDCD_DIV_AGONYGDPMKTPCD')
|
|
517
|
+
ago_eviews_run_lines = ['smpl @ALL','series MEAN_AGOBNCABFUNDCD_DIV_AGONYGDPMKTPCD = @MEAN(AGOBNCABFUNDCD/AGONYGDPMKTPCD,"2000 2020")']
|
|
518
|
+
|
|
519
|
+
mda_trans = lambda input: input.replace('_MDAsbbrev.@coef(2)','_MDASBBREV_at_COEF_2')
|
|
520
|
+
mda_eviews_run_lines = ['Scalar _MDASBBREV_at_COEF_2 = _MDASBBREV.@COEF(+2)']
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
filedict = {f.stem[:3].lower():f for f in Path('C:\wb new\Modelflow\ib\developement\original').glob('*.wf1')}
|
|
524
|
+
modelname = 'kaz'
|
|
525
|
+
filename = filedict[modelname]
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
eviews_run_lines= globals().get(f'{modelname}_eviews_run_lines',[])
|
|
529
|
+
country_trans = globals().get(f'{modelname}_trans' ,lambda x : x[:])
|
|
530
|
+
country_df_trans = globals().get(f'{modelname}_df_trans',lambda x : x)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
cmodel = GrabWfModel(filename,
|
|
534
|
+
# eviews_run_lines= eviews_run_lines,
|
|
535
|
+
# country_trans = country_trans,
|
|
536
|
+
country_df_trans = country_df_trans,
|
|
537
|
+
make_fitted = True,
|
|
538
|
+
do_add_factor_calc=True,
|
|
539
|
+
start = 2020,
|
|
540
|
+
end = 2100,
|
|
541
|
+
fit_start = 2000, # Start of calculation of fittet model in baseline
|
|
542
|
+
fit_end = 2030 # end of calc for fittted model, if None taken from mdmfsa options
|
|
543
|
+
)
|
|
544
|
+
# assert 1==2
|
|
545
|
+
cmodel.test_model(cmodel.start,cmodel.end,maxerr=100,tol=0.001,showall=0)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
grab_lookat = cmodel
|
|
549
|
+
mlookat = grab_lookat.mmodel
|
|
550
|
+
lookat_des = mlookat.var_description
|
|
551
|
+
lookat_equations = mlookat.equations
|
|
552
|
+
lookat_all_frml_dict = grab_lookat.all_frml_dict
|
|
553
|
+
base_input = cmodel.base_input
|
|
554
|
+
mlookat(base_input,2020,2022)
|