ModelFlowIb 2.92__tar.gz → 2.94__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modelflowib-2.92 → modelflowib-2.94}/PKG-INFO +1 -1
- {modelflowib-2.92 → modelflowib-2.94}/model_parquet_mixin.py +15 -4
- {modelflowib-2.92 → modelflowib-2.94}/modelcausality.py +4 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelclass.py +74 -25
- {modelflowib-2.92 → modelflowib-2.94}/modelconstruct_estimation.py +54 -1
- {modelflowib-2.92 → modelflowib-2.94}/modelestimator_new.py +95 -7
- modelflowib-2.94/modelfactorize.py +346 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/PKG-INFO +1 -1
- {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/SOURCES.txt +2 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/top_level.txt +2 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelhelp.py +17 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelnewton.py +218 -9
- modelflowib-2.94/modelode.py +552 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelsolver_ng.py +114 -33
- {modelflowib-2.92 → modelflowib-2.94}/pyproject.toml +3 -3
- {modelflowib-2.92 → modelflowib-2.94}/README.md +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/license.md +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelBLfunk.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_Excel.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_cvx.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_dynare.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_financial_stability.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_latex.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/model_latex_class.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelconstruct.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldash.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldashboot.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldashsidebar.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldekom.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldiff.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeldisplay.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelestimation.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/dependency_links.txt +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/requires.txt +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelgrab.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelgrabexcel.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelgrabgdx.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelgrabgekkobank.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelgrabwf2.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelhtml.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelinput_shiny.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelinvert.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeljupyter.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeljupytermagic.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelmacrograb.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelmanipulation.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelmf.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelnet.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelnormalize.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelpattern.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelreport.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modeluserfunk.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelvis.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelwidget.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelwidget_core.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/modelwidget_input.py +0 -0
- {modelflowib-2.92 → modelflowib-2.94}/setup.cfg +0 -0
|
@@ -164,7 +164,7 @@ class Parquet_Mixin:
|
|
|
164
164
|
|
|
165
165
|
# ── dump ──────────────────────────────────────────────────────────────
|
|
166
166
|
|
|
167
|
-
def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
|
|
167
|
+
def modeldump(self, file_path='', keep=False, large=False, compact=False, derivatives=False, **kwargs):
|
|
168
168
|
"""Dump model to disk.
|
|
169
169
|
|
|
170
170
|
Parameters
|
|
@@ -181,6 +181,12 @@ class Parquet_Mixin:
|
|
|
181
181
|
If True (only effective when large=True), store float64
|
|
182
182
|
columns as float32 to roughly halve file size.
|
|
183
183
|
On load the data is restored to float64.
|
|
184
|
+
derivatives : bool
|
|
185
|
+
If True, also store the expressions for the derivatives that a Newton
|
|
186
|
+
solver has found (``model.diff_expressions``). A Newton solver in a later
|
|
187
|
+
session takes them from the model instead of finding them again, which is
|
|
188
|
+
the slow part of its first solve for a large model. They are used only if
|
|
189
|
+
the equations and the solver options are the same.
|
|
184
190
|
|
|
185
191
|
A model instance without a lastdf holds no dataframe, so the feather
|
|
186
192
|
format has nothing to be fast about. Such a model is always written as a
|
|
@@ -194,7 +200,7 @@ class Parquet_Mixin:
|
|
|
194
200
|
|
|
195
201
|
if not large or nodata:
|
|
196
202
|
# ── original JSON/gzip path ──────────────────────────────────
|
|
197
|
-
return super().modeldump(file_path=file_path, keep=keep, **kwargs)
|
|
203
|
+
return super().modeldump(file_path=file_path, keep=keep, derivatives=derivatives, **kwargs)
|
|
198
204
|
|
|
199
205
|
# ── fast feather/zip path ────────────────────────────────────────
|
|
200
206
|
pathname = Path(file_path)
|
|
@@ -216,8 +222,10 @@ class Parquet_Mixin:
|
|
|
216
222
|
'format': 'feather_zip',
|
|
217
223
|
'frml': self.equations,
|
|
218
224
|
'modelname': self.name,
|
|
219
|
-
'oldkwargs': self.oldkwargs,
|
|
225
|
+
'oldkwargs': self.__dict__.get('oldkwargs', {}), # a model never solved has none
|
|
220
226
|
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
227
|
+
'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
|
|
228
|
+
'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
|
|
221
229
|
'var_description': self.var_description,
|
|
222
230
|
'equations_latex': self.equations_latex,
|
|
223
231
|
'wb_MFMSAOPTIONS': (
|
|
@@ -383,8 +391,11 @@ class Parquet_Mixin:
|
|
|
383
391
|
# while threads work, build the model object (parses equations)
|
|
384
392
|
frml = meta['frml']
|
|
385
393
|
modelname = meta['modelname']
|
|
386
|
-
|
|
394
|
+
# a model dumped with straight=True is loaded like that, unless the call says otherwise
|
|
395
|
+
model_kwargs = {**({'straight': True} if meta.get('straight', False) else {}), **kwargs}
|
|
396
|
+
mmodel = cls(frml, modelname=modelname, funks=funks, **model_kwargs)
|
|
387
397
|
mmodel.oldkwargs = meta.get('oldkwargs', {})
|
|
398
|
+
mmodel.diff_expressions = meta.get('diff_expressions', {})
|
|
388
399
|
mmodel.run_on_load = meta.get('run_on_load', False)
|
|
389
400
|
if run is None:
|
|
390
401
|
run = mmodel.run_on_load
|
|
@@ -736,6 +736,10 @@ if AnyWidget is not None:
|
|
|
736
736
|
self.observe(self._refresh, names=self._settings)
|
|
737
737
|
self._refresh()
|
|
738
738
|
|
|
739
|
+
def __repr__(self):
|
|
740
|
+
# the text/plain output, shown where the widget is not run (a static book)
|
|
741
|
+
return f'Interactive causality viewer for {self.var}, run the notebook to use it'
|
|
742
|
+
|
|
739
743
|
def _refresh(self, change=None):
|
|
740
744
|
navn = self.var.upper()
|
|
741
745
|
if navn not in self.mmodel.allvar:
|
|
@@ -116,6 +116,7 @@ from modelhelp import cutout, update_var
|
|
|
116
116
|
from modelnormalize import normal
|
|
117
117
|
from modeldekom import totdif
|
|
118
118
|
from modelhelp import debug_var
|
|
119
|
+
from modelhelp import nonfinite_places
|
|
119
120
|
|
|
120
121
|
|
|
121
122
|
# functions used in BL language
|
|
@@ -1027,7 +1028,8 @@ class BaseModel():
|
|
|
1027
1028
|
convergence = True
|
|
1028
1029
|
for after, before in zip(itafter, itbefore):
|
|
1029
1030
|
# print(before,after)
|
|
1030
|
-
|
|
1031
|
+
# NaN fails before > absconv, so it has to be tested on its own
|
|
1032
|
+
if not np.all(np.isfinite(after)) or (before > absconv and abs(after-before)/abs(before) > relconv):
|
|
1031
1033
|
convergence = False
|
|
1032
1034
|
break
|
|
1033
1035
|
if convergence:
|
|
@@ -3174,7 +3176,7 @@ class Description_Mixin():
|
|
|
3174
3176
|
out = (ind.replace('&','&').replace('<', '<').replace('>', '>').replace('\n', ' ')
|
|
3175
3177
|
.replace('æ', 'æ').replace('ø', 'ø').replace('å', 'å')
|
|
3176
3178
|
.replace('Æ', 'Æ').replace('Ø', 'Ø').replace('Å', 'Å')
|
|
3177
|
-
.replace('$','$')
|
|
3179
|
+
.replace('$','$').replace('"', '"')
|
|
3178
3180
|
)
|
|
3179
3181
|
return out
|
|
3180
3182
|
|
|
@@ -6863,11 +6865,15 @@ class Json_Mixin():
|
|
|
6863
6865
|
|
|
6864
6866
|
return pathname, dump_data
|
|
6865
6867
|
|
|
6866
|
-
def modeldump_base(self, outfile='',keep=False,data=True):
|
|
6868
|
+
def modeldump_base(self, outfile='',keep=False,data=True,derivatives=False):
|
|
6867
6869
|
'''Dumps a model and its lastdf to a json file
|
|
6868
6870
|
|
|
6869
6871
|
if keep=True the model.keep_solutions will alse be dumped
|
|
6870
6872
|
|
|
6873
|
+
if derivatives=True the expressions for the derivatives, which a Newton solver has found
|
|
6874
|
+
(model.diff_expressions), are dumped too. A Newton solver in a later session then takes them
|
|
6875
|
+
from the model instead of finding them again - the slow part of its first solve for a large model
|
|
6876
|
+
|
|
6871
6877
|
if data=False, or the model instance has no lastdf, only the model
|
|
6872
6878
|
structure is dumped and the file has no dataframes'''
|
|
6873
6879
|
# print('dump ready')
|
|
@@ -6928,6 +6934,8 @@ class Json_Mixin():
|
|
|
6928
6934
|
'modelname': self.name,
|
|
6929
6935
|
'oldkwargs': self.__dict__.get('oldkwargs',{}),
|
|
6930
6936
|
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
6937
|
+
'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
|
|
6938
|
+
'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
|
|
6931
6939
|
'var_description': self.var_description,
|
|
6932
6940
|
'equations_latex': self.equations_latex ,
|
|
6933
6941
|
'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
|
|
@@ -7065,8 +7073,11 @@ class Json_Mixin():
|
|
|
7065
7073
|
current_per = None
|
|
7066
7074
|
modelname = input['modelname']
|
|
7067
7075
|
# breakpoint()
|
|
7068
|
-
|
|
7076
|
+
# a model dumped with straight=True is loaded like that, unless the call says otherwise
|
|
7077
|
+
model_kwargs = {**({'straight': True} if input.get('straight', False) else {}), **kwargs}
|
|
7078
|
+
mmodel = cls(frml, modelname=modelname, funks=funks,**model_kwargs)
|
|
7069
7079
|
mmodel.oldkwargs = input.get('oldkwargs',{})
|
|
7080
|
+
mmodel.diff_expressions = input.get('diff_expressions', {})
|
|
7070
7081
|
mmodel.run_on_load = input.get('run_on_load', False)
|
|
7071
7082
|
if run is None:
|
|
7072
7083
|
run = mmodel.run_on_load and has_data
|
|
@@ -7123,12 +7134,12 @@ class Json_Mixin():
|
|
|
7123
7134
|
|
|
7124
7135
|
class Zip_Mixin():
|
|
7125
7136
|
'''This experimental class zips a dumped file '''
|
|
7126
|
-
def modeldump(self, file_path='',keep=False,zip_file=True,data=True):
|
|
7137
|
+
def modeldump(self, file_path='',keep=False,zip_file=True,data=True,derivatives=False):
|
|
7127
7138
|
pathname, dump_data = self._dump_pathname(file_path, self.has_data, data=data)
|
|
7128
7139
|
|
|
7129
7140
|
pathname.parent.mkdir(parents=True, exist_ok=True)
|
|
7130
7141
|
|
|
7131
|
-
json_string = self.modeldump_base(keep=keep,data=dump_data)
|
|
7142
|
+
json_string = self.modeldump_base(keep=keep,data=dump_data,derivatives=derivatives)
|
|
7132
7143
|
|
|
7133
7144
|
|
|
7134
7145
|
if zip_file:
|
|
@@ -7640,7 +7651,8 @@ class Solver_Mixin():
|
|
|
7640
7651
|
# this is how convergence is measured
|
|
7641
7652
|
convplace = [databank.columns.get_loc(c) for c in convvar]
|
|
7642
7653
|
convergence = True
|
|
7643
|
-
|
|
7654
|
+
endovar = list(self.endogene)
|
|
7655
|
+
endoplace = [databank.columns.get_loc(c) for c in endovar]
|
|
7644
7656
|
# breakpoint()
|
|
7645
7657
|
if ldumpvar:
|
|
7646
7658
|
self.dumplist = []
|
|
@@ -7668,6 +7680,7 @@ class Solver_Mixin():
|
|
|
7668
7680
|
values[row, c] = values[row-1, c]
|
|
7669
7681
|
itbefore = values[row, convplace]
|
|
7670
7682
|
self.pro2d(values, values, row, 1.0)
|
|
7683
|
+
corenan = False
|
|
7671
7684
|
for iteration in range(max_iterations):
|
|
7672
7685
|
with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
|
|
7673
7686
|
self.solve2d(values, values, row, alfa)
|
|
@@ -7679,18 +7692,30 @@ class Solver_Mixin():
|
|
|
7679
7692
|
if iteration > first_test:
|
|
7680
7693
|
itafter = values[row, convplace]
|
|
7681
7694
|
select = absconv <= np.abs(itbefore)
|
|
7695
|
+
# NaN drops out of select, so it has to be tested on its own
|
|
7696
|
+
nonfinite = nonfinite_places(itafter)
|
|
7682
7697
|
convergence = (
|
|
7683
|
-
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
|
|
7698
|
+
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
|
|
7684
7699
|
if convergence:
|
|
7685
7700
|
if not silent:
|
|
7686
7701
|
print(
|
|
7687
7702
|
f'{self.periode} Solved in {iteration} iterations')
|
|
7688
7703
|
break
|
|
7704
|
+
if len(nonfinite):
|
|
7705
|
+
corenan = True
|
|
7706
|
+
bad = [convvar[i] for i in nonfinite]
|
|
7707
|
+
print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
7708
|
+
break
|
|
7689
7709
|
itbefore = itafter
|
|
7690
7710
|
else:
|
|
7691
7711
|
print(f'{self.periode} not converged in {iteration} iterations')
|
|
7692
|
-
|
|
7712
|
+
|
|
7693
7713
|
self.epi2d(values, values, row, 1.0)
|
|
7714
|
+
# the convergence test only sees the core, prolog/epilog can also be NaN
|
|
7715
|
+
nonfinite = nonfinite_places(values[row, endoplace])
|
|
7716
|
+
if len(nonfinite) and not corenan:
|
|
7717
|
+
bad = [endovar[i] for i in nonfinite]
|
|
7718
|
+
print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
7694
7719
|
pbar.update()
|
|
7695
7720
|
endtime = time.time()
|
|
7696
7721
|
|
|
@@ -7975,7 +8000,8 @@ class Solver_Mixin():
|
|
|
7975
8000
|
# convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
7976
8001
|
convplace = [self.allvar[c]['startnr'] -
|
|
7977
8002
|
self.allvar[c]['maxlead'] for c in convvar]
|
|
7978
|
-
|
|
8003
|
+
endovar = list(self.endogene)
|
|
8004
|
+
endoplace = [databank.columns.get_loc(c) for c in endovar]
|
|
7979
8005
|
|
|
7980
8006
|
convergence = True
|
|
7981
8007
|
|
|
@@ -8010,6 +8036,7 @@ class Solver_Mixin():
|
|
|
8010
8036
|
itbeforeold = [a[c] for c in convplace]
|
|
8011
8037
|
itbefore = a[convplace]
|
|
8012
8038
|
this_pro1d(a, 1.0)
|
|
8039
|
+
corenan = False
|
|
8013
8040
|
for iteration in range(max_iterations):
|
|
8014
8041
|
with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
|
|
8015
8042
|
this_solve1d(a, alfa)
|
|
@@ -8022,20 +8049,32 @@ class Solver_Mixin():
|
|
|
8022
8049
|
itafter = a[convplace]
|
|
8023
8050
|
# breakpoint()
|
|
8024
8051
|
select = absconv <= np.abs(itbefore)
|
|
8052
|
+
# NaN drops out of select, so it has to be tested on its own
|
|
8053
|
+
nonfinite = nonfinite_places(itafter)
|
|
8025
8054
|
convergence = (
|
|
8026
|
-
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
|
|
8055
|
+
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
|
|
8027
8056
|
if convergence:
|
|
8028
8057
|
if not silent:
|
|
8029
8058
|
print(
|
|
8030
8059
|
f'{self.periode} Solved in {iteration} iterations')
|
|
8031
8060
|
break
|
|
8061
|
+
if len(nonfinite):
|
|
8062
|
+
corenan = True
|
|
8063
|
+
bad = [convvar[i] for i in nonfinite]
|
|
8064
|
+
print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
8065
|
+
break
|
|
8032
8066
|
itbefore = itafter
|
|
8033
8067
|
else:
|
|
8034
8068
|
print(f'{self.periode} not converged in {iteration} iterations')
|
|
8035
|
-
|
|
8069
|
+
|
|
8036
8070
|
this_epi1d(a, 1.0)
|
|
8037
8071
|
|
|
8038
8072
|
self.saveeval3(values, row, a)
|
|
8073
|
+
# the convergence test only sees the core, prolog/epilog can also be NaN
|
|
8074
|
+
nonfinite = nonfinite_places(values[row, endoplace])
|
|
8075
|
+
if len(nonfinite) and not corenan:
|
|
8076
|
+
bad = [endovar[i] for i in nonfinite]
|
|
8077
|
+
print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
8039
8078
|
|
|
8040
8079
|
if ldumpvar:
|
|
8041
8080
|
self.dumpdf = pd.DataFrame(self.dumplist)
|
|
@@ -9617,9 +9656,23 @@ class Solver_Mixin():
|
|
|
9617
9656
|
|
|
9618
9657
|
def errfunk(self, values, linenr, overhead=4, overeq=0):
|
|
9619
9658
|
'''Development function to handle run time errors in model calculations'''
|
|
9620
|
-
|
|
9659
|
+
|
|
9621
9660
|
import pandas as pd
|
|
9622
|
-
|
|
9661
|
+
|
|
9662
|
+
if getattr(self, 'errfunk_defer', False):
|
|
9663
|
+
# a Newton trial step: keep the error point, the solver dumps it only if it gives up
|
|
9664
|
+
self.errfunk_deferred = (values.copy(), linenr, overhead, overeq)
|
|
9665
|
+
return
|
|
9666
|
+
|
|
9667
|
+
if hasattr(self, 'dumplist'):
|
|
9668
|
+
# first, so the returns below keep it, and it must never mask the real error
|
|
9669
|
+
dumplist = self.dumplist
|
|
9670
|
+
del self.dumplist
|
|
9671
|
+
try:
|
|
9672
|
+
self.dumpdf = pd.DataFrame(dumplist, columns=['fair', 'per', 'iteration'] + list(self.dump))
|
|
9673
|
+
except Exception as e:
|
|
9674
|
+
print('>> Could not build .dumpdf:', e)
|
|
9675
|
+
|
|
9623
9676
|
def find_log_arguments(text):
|
|
9624
9677
|
"""Return all arguments inside balanced LOG(...) calls."""
|
|
9625
9678
|
out = []
|
|
@@ -9666,7 +9719,8 @@ class Solver_Mixin():
|
|
|
9666
9719
|
outeq = self.allvar[errvar]['frml']
|
|
9667
9720
|
|
|
9668
9721
|
print('>>:', outeq,)
|
|
9669
|
-
|
|
9722
|
+
per = getattr(self, 'periode', None) # not set if the error comes before the period loop
|
|
9723
|
+
print('In:', per if per is not None else 'unknown period', '\n')
|
|
9670
9724
|
print('Below you will find information which might help you identify the problem')
|
|
9671
9725
|
terms = self.allvar[errvar]['terms']
|
|
9672
9726
|
|
|
@@ -9679,7 +9733,7 @@ class Solver_Mixin():
|
|
|
9679
9733
|
|
|
9680
9734
|
|
|
9681
9735
|
try:
|
|
9682
|
-
basepos = self.errdump.index.get_loc(
|
|
9736
|
+
basepos = self.errdump.index.get_loc(per)
|
|
9683
9737
|
rows = []
|
|
9684
9738
|
seen = set()
|
|
9685
9739
|
|
|
@@ -9718,8 +9772,8 @@ class Solver_Mixin():
|
|
|
9718
9772
|
for arg in log_args:
|
|
9719
9773
|
try:
|
|
9720
9774
|
val = self.errdump.mfcalc(
|
|
9721
|
-
f'<{
|
|
9722
|
-
).loc[
|
|
9775
|
+
f'<{per}> {tmpvar} = {arg}'
|
|
9776
|
+
).loc[per, tmpvar]
|
|
9723
9777
|
status = 'Bad log argument' if pd.isna(val) or val <= 0 else 'ok'
|
|
9724
9778
|
except Exception:
|
|
9725
9779
|
val = ''
|
|
@@ -9735,13 +9789,8 @@ class Solver_Mixin():
|
|
|
9735
9789
|
|
|
9736
9790
|
print('\nA snapshot of the data at the error point is at .errdump')
|
|
9737
9791
|
print('Also .lastdf contains a copy of .errdump for inspecting')
|
|
9738
|
-
|
|
9739
|
-
|
|
9740
|
-
self.dumpdf = pd.DataFrame(self.dumplist)
|
|
9741
|
-
del self.dumplist
|
|
9742
|
-
self.dumpdf.columns = ['fair', 'per', 'iteration'] + self.dump
|
|
9743
|
-
|
|
9744
|
-
|
|
9792
|
+
|
|
9793
|
+
|
|
9745
9794
|
def show_iterations(self, pat='*', per='', last=0, change=False,top=.9):
|
|
9746
9795
|
'''
|
|
9747
9796
|
shows the last iterations for the most recent simulation.
|
|
@@ -2366,7 +2366,8 @@ class Makemodel(BaseExplode):
|
|
|
2366
2366
|
self.markdown_model = (mfmod_list_to_codeblock(self.original_statements))
|
|
2367
2367
|
|
|
2368
2368
|
elif self.type_input == 'single':
|
|
2369
|
-
|
|
2369
|
+
# join the lines with a space, so a tag ending a line stays apart from the equation
|
|
2370
|
+
extracted_frml = ' '.join(line.strip() for line in self.original_statements.splitlines()).strip()
|
|
2370
2371
|
self.markdown_model = ''
|
|
2371
2372
|
|
|
2372
2373
|
else:
|
|
@@ -2685,6 +2686,46 @@ class Makemodel(BaseExplode):
|
|
|
2685
2686
|
cols[name] = ser
|
|
2686
2687
|
return pd.DataFrame(cols)
|
|
2687
2688
|
|
|
2689
|
+
def actual_fitted(self, level: bool = True):
|
|
2690
|
+
"""Actual, fitted and residuals of all estimated equations as one DataFrame.
|
|
2691
|
+
|
|
2692
|
+
The columns have two levels: the endogenous variable, and ``Actual``,
|
|
2693
|
+
``Fitted``, ``Residuals``. Take one equation with
|
|
2694
|
+
``mm.actual_fitted()['CON']``.
|
|
2695
|
+
|
|
2696
|
+
level : bool, default True
|
|
2697
|
+
True: the endogenous variable itself. When the left-hand side is
|
|
2698
|
+
``DLOG(CON)`` this is ``CON``, its one-step fitted value (the
|
|
2699
|
+
estimated equation solved for ``CON``, lagged values are the
|
|
2700
|
+
actual ones) and the residual in the units of ``CON``.
|
|
2701
|
+
False: the left-hand side as it was estimated (``DLOG(CON)``).
|
|
2702
|
+
|
|
2703
|
+
Each equation covers its own estimation sample, so rows outside it
|
|
2704
|
+
are NaN. An equation which can't be solved for its variable is left
|
|
2705
|
+
out with ``level=True``.
|
|
2706
|
+
|
|
2707
|
+
Per-equation access: ``rec['estimator_object'].af_level_df`` (or
|
|
2708
|
+
``.af_df`` and ``.residuals_df``) on the entries of
|
|
2709
|
+
:attr:`estimation_records`; ``modelestimator_new.plot_actual_vs_fitted
|
|
2710
|
+
(est, level=True)`` draws the chart of the report.
|
|
2711
|
+
"""
|
|
2712
|
+
import pandas as pd
|
|
2713
|
+
frames = {}
|
|
2714
|
+
for rec in self.estimation_records:
|
|
2715
|
+
est = rec.get('estimator_object')
|
|
2716
|
+
if not _is_estimator_backend_instance(est):
|
|
2717
|
+
continue
|
|
2718
|
+
frame = est.af_level_df if level else est.af_df.join(est.residuals_df)
|
|
2719
|
+
if frame is None:
|
|
2720
|
+
continue
|
|
2721
|
+
name = getattr(est, 'endo_var', '') or rec.get('frmlname', '')
|
|
2722
|
+
if level: # the variable is the outer column level
|
|
2723
|
+
frame = frame.rename(columns=lambda c: c.removeprefix(f'{est.endo_var}__'))
|
|
2724
|
+
frames[name] = frame
|
|
2725
|
+
if not frames:
|
|
2726
|
+
return pd.DataFrame()
|
|
2727
|
+
return pd.concat(frames, axis=1)
|
|
2728
|
+
|
|
2688
2729
|
@property
|
|
2689
2730
|
def markdown_with_estimation(self) -> str:
|
|
2690
2731
|
"""Original Markdown input with compact estimation tables inserted.
|
|
@@ -3085,6 +3126,18 @@ class Listmodels(BaseExplode):
|
|
|
3085
3126
|
return pd.DataFrame()
|
|
3086
3127
|
return pd.concat(frames, axis=1)
|
|
3087
3128
|
|
|
3129
|
+
def actual_fitted(self, level: bool = True):
|
|
3130
|
+
"""Actual, fitted and residuals of all member Makemodels, combined column-wise.
|
|
3131
|
+
|
|
3132
|
+
See :meth:`Makemodel.actual_fitted` for the layout.
|
|
3133
|
+
"""
|
|
3134
|
+
import pandas as pd
|
|
3135
|
+
frames = [mex.actual_fitted(level=level) for mex in self.makemodels]
|
|
3136
|
+
frames = [f for f in frames if not f.empty]
|
|
3137
|
+
if not frames:
|
|
3138
|
+
return pd.DataFrame()
|
|
3139
|
+
return pd.concat(frames, axis=1)
|
|
3140
|
+
|
|
3088
3141
|
@property
|
|
3089
3142
|
def markdown_with_estimation(self) -> str:
|
|
3090
3143
|
"""Concatenate member Makemodel markdown-with-estimation strings."""
|
|
@@ -1378,6 +1378,67 @@ class EstimatorBackend(Eq_parent, ABC):
|
|
|
1378
1378
|
.rename(columns=lambda s: s.lower().capitalize())
|
|
1379
1379
|
)
|
|
1380
1380
|
|
|
1381
|
+
# ---- A/F and residuals of the endogenous variable itself ---------------
|
|
1382
|
+
|
|
1383
|
+
@cached_property
|
|
1384
|
+
def lhs_is_transformed(self) -> bool:
|
|
1385
|
+
"""True when the left-hand side is a transformation of the endogenous
|
|
1386
|
+
variable (``DLOG(CON)``, ``DIFF(CON)``, ``LOG(CON/P)`` ...), so
|
|
1387
|
+
:attr:`af_df` is not in the units of the variable itself."""
|
|
1388
|
+
lhs = self.org_eq_clean.split("=", 1)[0]
|
|
1389
|
+
return "".join(lhs.split()).upper() != self.endo_var.upper()
|
|
1390
|
+
|
|
1391
|
+
@cached_property
|
|
1392
|
+
def af_level_df(self) -> Optional[pd.DataFrame]:
|
|
1393
|
+
"""Actual, fitted and residuals of the endogenous variable itself.
|
|
1394
|
+
|
|
1395
|
+
:attr:`af_df` and :attr:`residuals_df` are in the units of the
|
|
1396
|
+
left-hand side as it was estimated, e.g. ``DLOG(CON)``. Here the
|
|
1397
|
+
estimated equation is solved for the variable (``CON``):
|
|
1398
|
+
|
|
1399
|
+
* ``Actual`` the variable,
|
|
1400
|
+
* ``Fitted`` the normalized equation with the estimated
|
|
1401
|
+
coefficients, evaluated on the data. Lagged values of the variable
|
|
1402
|
+
are the actual ones, so this is the one-step (static) fit, not a
|
|
1403
|
+
dynamic simulation,
|
|
1404
|
+
* ``Residuals`` ``Actual - Fitted``, in the units of the variable.
|
|
1405
|
+
|
|
1406
|
+
The columns are prefixed with the variable name and ``__``
|
|
1407
|
+
(``CON__Actual``, ``CON__Fitted``, ``CON__Residuals``), so frames of
|
|
1408
|
+
several equations can be joined side by side.
|
|
1409
|
+
|
|
1410
|
+
When the left-hand side is the variable itself the frame holds the
|
|
1411
|
+
same numbers as :attr:`af_df` and :attr:`residuals_df`. ``None`` (and
|
|
1412
|
+
a message) when the equation can't be solved for the variable.
|
|
1413
|
+
"""
|
|
1414
|
+
prefix = f"{self.endo_var}__"
|
|
1415
|
+
if not self.lhs_is_transformed:
|
|
1416
|
+
return self.af_df.join(self.residuals_df).add_prefix(prefix)
|
|
1417
|
+
start, end = self.smpl
|
|
1418
|
+
try:
|
|
1419
|
+
normalized = nz.normal(
|
|
1420
|
+
self.org_eq_baked, add_add_factor=False
|
|
1421
|
+
).normalized
|
|
1422
|
+
lhs, rhs = normalized.split("=", 1)
|
|
1423
|
+
if lhs.strip().upper() != self.endo_var.upper():
|
|
1424
|
+
raise ValueError(f"normalized to {lhs.strip()!r}")
|
|
1425
|
+
res = self.eq_var_df.mfcalc(
|
|
1426
|
+
f"<{start},{end}> FITTED_LEVEL = {rhs}"
|
|
1427
|
+
)
|
|
1428
|
+
out = pd.DataFrame({
|
|
1429
|
+
"Actual": res.loc[start:end, self.endo_var],
|
|
1430
|
+
"Fitted": res.loc[start:end, "FITTED_LEVEL"],
|
|
1431
|
+
})
|
|
1432
|
+
except Exception as e:
|
|
1433
|
+
print(
|
|
1434
|
+
f"{type(self).__name__}({self.endo_var}): no actual/fitted "
|
|
1435
|
+
f"for the variable itself, the equation could not be solved "
|
|
1436
|
+
f"for it ({type(e).__name__}: {str(e)[:100]})"
|
|
1437
|
+
)
|
|
1438
|
+
return None
|
|
1439
|
+
out["Residuals"] = out["Actual"] - out["Fitted"]
|
|
1440
|
+
return out.add_prefix(prefix)
|
|
1441
|
+
|
|
1381
1442
|
@cached_property
|
|
1382
1443
|
def estimation_smpl(self) -> Tuple[Any, Any]:
|
|
1383
1444
|
"""The usable sample, derived from non-missing residuals.
|
|
@@ -2088,6 +2149,11 @@ class LSResult:
|
|
|
2088
2149
|
title_html = self._build_title_html()
|
|
2089
2150
|
summary_html = self.olsmodel._render_summary_html()
|
|
2090
2151
|
plot_html = self._build_plot_html(plot_format)
|
|
2152
|
+
# the same for the endogenous variable itself, when the left-hand
|
|
2153
|
+
# side is a transformation of it (DLOG(CON) -> CON)
|
|
2154
|
+
if (self.olsmodel.lhs_is_transformed
|
|
2155
|
+
and self.olsmodel.af_level_df is not None):
|
|
2156
|
+
plot_html += self._build_plot_html(plot_format, level=True)
|
|
2091
2157
|
return title_html + summary_html + plot_html
|
|
2092
2158
|
|
|
2093
2159
|
def show(self, plot_format: str = "svg") -> None:
|
|
@@ -2124,17 +2190,21 @@ class LSResult:
|
|
|
2124
2190
|
f"<pre><code>{normalized}</code></pre></p>"
|
|
2125
2191
|
)
|
|
2126
2192
|
|
|
2127
|
-
def _build_plot_html(self, plot_format: str) -> str:
|
|
2193
|
+
def _build_plot_html(self, plot_format: str, level: bool = False) -> str:
|
|
2128
2194
|
buf = BytesIO()
|
|
2129
2195
|
plot_actual_vs_fitted(
|
|
2130
2196
|
self.olsmodel, save_to=buf, format=plot_format,
|
|
2131
|
-
figsize=(8, 5), dpi=150,
|
|
2197
|
+
figsize=(8, 5), dpi=150, level=level,
|
|
2132
2198
|
)
|
|
2133
2199
|
buf.seek(0)
|
|
2134
2200
|
b64 = base64.b64encode(buf.read()).decode("utf-8")
|
|
2135
2201
|
mime = "svg+xml" if plot_format == "svg" else "png"
|
|
2202
|
+
heading = (
|
|
2203
|
+
f"Actual vs Fitted Plot: {self.olsmodel.endo_var} itself "
|
|
2204
|
+
"(one-step fit)" if level else "Actual vs Fitted Plot"
|
|
2205
|
+
)
|
|
2136
2206
|
return (
|
|
2137
|
-
"<h3>
|
|
2207
|
+
f"<h3>{heading}</h3>"
|
|
2138
2208
|
f'<img style="max-width:100%; height:auto;" '
|
|
2139
2209
|
f'src="data:image/{mime};base64,{b64}" />'
|
|
2140
2210
|
)
|
|
@@ -2440,28 +2510,46 @@ def plot_actual_vs_fitted(
|
|
|
2440
2510
|
format: str = "png",
|
|
2441
2511
|
figsize: Tuple[float, float] = (7, 3),
|
|
2442
2512
|
dpi: int = 150,
|
|
2513
|
+
level: bool = False,
|
|
2443
2514
|
):
|
|
2444
2515
|
"""Plot Actual vs Fitted (top panel) and Residuals (bottom panel).
|
|
2445
2516
|
|
|
2517
|
+
``level=False`` plots the left-hand side as it was estimated (``af_df``,
|
|
2518
|
+
``residuals_df``), ``level=True`` the endogenous variable itself
|
|
2519
|
+
(``af_level_df``): ``CON`` when the left-hand side is ``DLOG(CON)``.
|
|
2520
|
+
|
|
2446
2521
|
If ``save_to`` is a file-like object, the plot is saved there and the
|
|
2447
2522
|
figure is closed; otherwise it is shown interactively.
|
|
2448
2523
|
"""
|
|
2524
|
+
if level:
|
|
2525
|
+
if emodel.af_level_df is None:
|
|
2526
|
+
raise ValueError(
|
|
2527
|
+
f"{emodel.endo_var}: no actual/fitted for the variable itself"
|
|
2528
|
+
)
|
|
2529
|
+
prefix = f"{emodel.endo_var}__"
|
|
2530
|
+
af_df = residuals_df = emodel.af_level_df.rename(
|
|
2531
|
+
columns=lambda c: c.removeprefix(prefix))
|
|
2532
|
+
title = f"Actual vs Fitted: {emodel.endo_var} itself (one-step fit)"
|
|
2533
|
+
else:
|
|
2534
|
+
af_df, residuals_df = emodel.af_df, emodel.residuals_df
|
|
2535
|
+
title = f"Actual vs Fitted: {emodel.endo_var}"
|
|
2536
|
+
|
|
2449
2537
|
fig = plt.figure(figsize=figsize, dpi=dpi)
|
|
2450
2538
|
gs = GridSpec(3, 1, height_ratios=[2, 0.1, 1])
|
|
2451
2539
|
|
|
2452
2540
|
ax1 = fig.add_subplot(gs[0])
|
|
2453
2541
|
ax2 = fig.add_subplot(gs[2], sharex=ax1)
|
|
2454
2542
|
|
|
2455
|
-
ax1.plot(
|
|
2543
|
+
ax1.plot(af_df.index, af_df["Actual"],
|
|
2456
2544
|
label="Actual", linewidth=2)
|
|
2457
|
-
ax1.plot(
|
|
2545
|
+
ax1.plot(af_df.index, af_df["Fitted"],
|
|
2458
2546
|
label="Fitted", linestyle="--")
|
|
2459
|
-
ax1.set_title(
|
|
2547
|
+
ax1.set_title(title)
|
|
2460
2548
|
ax1.set_ylabel(emodel.endo_var)
|
|
2461
2549
|
ax1.grid(True)
|
|
2462
2550
|
ax1.legend()
|
|
2463
2551
|
|
|
2464
|
-
ax2.plot(
|
|
2552
|
+
ax2.plot(residuals_df.index, residuals_df["Residuals"],
|
|
2465
2553
|
color="gray")
|
|
2466
2554
|
ax2.axhline(0, color="red", linestyle="--", linewidth=1)
|
|
2467
2555
|
ax2.set_title("Residuals")
|