ModelFlowIb 2.92__tar.gz → 2.94__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {modelflowib-2.92 → modelflowib-2.94}/PKG-INFO +1 -1
  2. {modelflowib-2.92 → modelflowib-2.94}/model_parquet_mixin.py +15 -4
  3. {modelflowib-2.92 → modelflowib-2.94}/modelcausality.py +4 -0
  4. {modelflowib-2.92 → modelflowib-2.94}/modelclass.py +74 -25
  5. {modelflowib-2.92 → modelflowib-2.94}/modelconstruct_estimation.py +54 -1
  6. {modelflowib-2.92 → modelflowib-2.94}/modelestimator_new.py +95 -7
  7. modelflowib-2.94/modelfactorize.py +346 -0
  8. {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/PKG-INFO +1 -1
  9. {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/SOURCES.txt +2 -0
  10. {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/top_level.txt +2 -0
  11. {modelflowib-2.92 → modelflowib-2.94}/modelhelp.py +17 -0
  12. {modelflowib-2.92 → modelflowib-2.94}/modelnewton.py +218 -9
  13. modelflowib-2.94/modelode.py +552 -0
  14. {modelflowib-2.92 → modelflowib-2.94}/modelsolver_ng.py +114 -33
  15. {modelflowib-2.92 → modelflowib-2.94}/pyproject.toml +3 -3
  16. {modelflowib-2.92 → modelflowib-2.94}/README.md +0 -0
  17. {modelflowib-2.92 → modelflowib-2.94}/license.md +0 -0
  18. {modelflowib-2.92 → modelflowib-2.94}/modelBLfunk.py +0 -0
  19. {modelflowib-2.92 → modelflowib-2.94}/model_Excel.py +0 -0
  20. {modelflowib-2.92 → modelflowib-2.94}/model_cvx.py +0 -0
  21. {modelflowib-2.92 → modelflowib-2.94}/model_dynare.py +0 -0
  22. {modelflowib-2.92 → modelflowib-2.94}/model_financial_stability.py +0 -0
  23. {modelflowib-2.92 → modelflowib-2.94}/model_latex.py +0 -0
  24. {modelflowib-2.92 → modelflowib-2.94}/model_latex_class.py +0 -0
  25. {modelflowib-2.92 → modelflowib-2.94}/modelconstruct.py +0 -0
  26. {modelflowib-2.92 → modelflowib-2.94}/modeldash.py +0 -0
  27. {modelflowib-2.92 → modelflowib-2.94}/modeldashboot.py +0 -0
  28. {modelflowib-2.92 → modelflowib-2.94}/modeldashsidebar.py +0 -0
  29. {modelflowib-2.92 → modelflowib-2.94}/modeldekom.py +0 -0
  30. {modelflowib-2.92 → modelflowib-2.94}/modeldiff.py +0 -0
  31. {modelflowib-2.92 → modelflowib-2.94}/modeldisplay.py +0 -0
  32. {modelflowib-2.92 → modelflowib-2.94}/modelestimation.py +0 -0
  33. {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/dependency_links.txt +0 -0
  34. {modelflowib-2.92 → modelflowib-2.94}/modelflowib.egg-info/requires.txt +0 -0
  35. {modelflowib-2.92 → modelflowib-2.94}/modelgrab.py +0 -0
  36. {modelflowib-2.92 → modelflowib-2.94}/modelgrabexcel.py +0 -0
  37. {modelflowib-2.92 → modelflowib-2.94}/modelgrabgdx.py +0 -0
  38. {modelflowib-2.92 → modelflowib-2.94}/modelgrabgekkobank.py +0 -0
  39. {modelflowib-2.92 → modelflowib-2.94}/modelgrabwf2.py +0 -0
  40. {modelflowib-2.92 → modelflowib-2.94}/modelhtml.py +0 -0
  41. {modelflowib-2.92 → modelflowib-2.94}/modelinput_shiny.py +0 -0
  42. {modelflowib-2.92 → modelflowib-2.94}/modelinvert.py +0 -0
  43. {modelflowib-2.92 → modelflowib-2.94}/modeljupyter.py +0 -0
  44. {modelflowib-2.92 → modelflowib-2.94}/modeljupytermagic.py +0 -0
  45. {modelflowib-2.92 → modelflowib-2.94}/modelmacrograb.py +0 -0
  46. {modelflowib-2.92 → modelflowib-2.94}/modelmanipulation.py +0 -0
  47. {modelflowib-2.92 → modelflowib-2.94}/modelmf.py +0 -0
  48. {modelflowib-2.92 → modelflowib-2.94}/modelnet.py +0 -0
  49. {modelflowib-2.92 → modelflowib-2.94}/modelnormalize.py +0 -0
  50. {modelflowib-2.92 → modelflowib-2.94}/modelpattern.py +0 -0
  51. {modelflowib-2.92 → modelflowib-2.94}/modelreport.py +0 -0
  52. {modelflowib-2.92 → modelflowib-2.94}/modeluserfunk.py +0 -0
  53. {modelflowib-2.92 → modelflowib-2.94}/modelvis.py +0 -0
  54. {modelflowib-2.92 → modelflowib-2.94}/modelwidget.py +0 -0
  55. {modelflowib-2.92 → modelflowib-2.94}/modelwidget_core.py +0 -0
  56. {modelflowib-2.92 → modelflowib-2.94}/modelwidget_input.py +0 -0
  57. {modelflowib-2.92 → modelflowib-2.94}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.92
3
+ Version: 2.94
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -164,7 +164,7 @@ class Parquet_Mixin:
164
164
 
165
165
  # ── dump ──────────────────────────────────────────────────────────────
166
166
 
167
- def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
167
+ def modeldump(self, file_path='', keep=False, large=False, compact=False, derivatives=False, **kwargs):
168
168
  """Dump model to disk.
169
169
 
170
170
  Parameters
@@ -181,6 +181,12 @@ class Parquet_Mixin:
181
181
  If True (only effective when large=True), store float64
182
182
  columns as float32 to roughly halve file size.
183
183
  On load the data is restored to float64.
184
+ derivatives : bool
185
+ If True, also store the expressions for the derivatives that a Newton
186
+ solver has found (``model.diff_expressions``). A Newton solver in a later
187
+ session takes them from the model instead of finding them again, which is
188
+ the slow part of its first solve for a large model. They are used only if
189
+ the equations and the solver options are the same.
184
190
 
185
191
  A model instance without a lastdf holds no dataframe, so the feather
186
192
  format has nothing to be fast about. Such a model is always written as a
@@ -194,7 +200,7 @@ class Parquet_Mixin:
194
200
 
195
201
  if not large or nodata:
196
202
  # ── original JSON/gzip path ──────────────────────────────────
197
- return super().modeldump(file_path=file_path, keep=keep, **kwargs)
203
+ return super().modeldump(file_path=file_path, keep=keep, derivatives=derivatives, **kwargs)
198
204
 
199
205
  # ── fast feather/zip path ────────────────────────────────────────
200
206
  pathname = Path(file_path)
@@ -216,8 +222,10 @@ class Parquet_Mixin:
216
222
  'format': 'feather_zip',
217
223
  'frml': self.equations,
218
224
  'modelname': self.name,
219
- 'oldkwargs': self.oldkwargs,
225
+ 'oldkwargs': self.__dict__.get('oldkwargs', {}), # a model never solved has none
220
226
  'run_on_load': self.__dict__.get('run_on_load', False),
227
+ 'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
228
+ 'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
221
229
  'var_description': self.var_description,
222
230
  'equations_latex': self.equations_latex,
223
231
  'wb_MFMSAOPTIONS': (
@@ -383,8 +391,11 @@ class Parquet_Mixin:
383
391
  # while threads work, build the model object (parses equations)
384
392
  frml = meta['frml']
385
393
  modelname = meta['modelname']
386
- mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
394
+ # a model dumped with straight=True is loaded like that, unless the call says otherwise
395
+ model_kwargs = {**({'straight': True} if meta.get('straight', False) else {}), **kwargs}
396
+ mmodel = cls(frml, modelname=modelname, funks=funks, **model_kwargs)
387
397
  mmodel.oldkwargs = meta.get('oldkwargs', {})
398
+ mmodel.diff_expressions = meta.get('diff_expressions', {})
388
399
  mmodel.run_on_load = meta.get('run_on_load', False)
389
400
  if run is None:
390
401
  run = mmodel.run_on_load
@@ -736,6 +736,10 @@ if AnyWidget is not None:
736
736
  self.observe(self._refresh, names=self._settings)
737
737
  self._refresh()
738
738
 
739
+ def __repr__(self):
740
+ # the text/plain output, shown where the widget is not run (a static book)
741
+ return f'Interactive causality viewer for {self.var}, run the notebook to use it'
742
+
739
743
  def _refresh(self, change=None):
740
744
  navn = self.var.upper()
741
745
  if navn not in self.mmodel.allvar:
@@ -116,6 +116,7 @@ from modelhelp import cutout, update_var
116
116
  from modelnormalize import normal
117
117
  from modeldekom import totdif
118
118
  from modelhelp import debug_var
119
+ from modelhelp import nonfinite_places
119
120
 
120
121
 
121
122
  # functions used in BL language
@@ -1027,7 +1028,8 @@ class BaseModel():
1027
1028
  convergence = True
1028
1029
  for after, before in zip(itafter, itbefore):
1029
1030
  # print(before,after)
1030
- if before > absconv and abs(after-before)/abs(before) > relconv:
1031
+ # NaN fails before > absconv, so it has to be tested on its own
1032
+ if not np.all(np.isfinite(after)) or (before > absconv and abs(after-before)/abs(before) > relconv):
1031
1033
  convergence = False
1032
1034
  break
1033
1035
  if convergence:
@@ -3174,7 +3176,7 @@ class Description_Mixin():
3174
3176
  out = (ind.replace('&','&#38;').replace('<', '&lt;').replace('>', '&gt;').replace('\n', '&#10;')
3175
3177
  .replace('æ', '&#230;').replace('ø', '&#248;').replace('å', '&#229;')
3176
3178
  .replace('Æ', '&#198;').replace('Ø', '&#216;').replace('Å', '&#197;')
3177
- .replace('$','&#36;')
3179
+ .replace('$','&#36;').replace('"', '&quot;')
3178
3180
  )
3179
3181
  return out
3180
3182
 
@@ -6863,11 +6865,15 @@ class Json_Mixin():
6863
6865
 
6864
6866
  return pathname, dump_data
6865
6867
 
6866
- def modeldump_base(self, outfile='',keep=False,data=True):
6868
+ def modeldump_base(self, outfile='',keep=False,data=True,derivatives=False):
6867
6869
  '''Dumps a model and its lastdf to a json file
6868
6870
 
6869
6871
  if keep=True the model.keep_solutions will alse be dumped
6870
6872
 
6873
+ if derivatives=True the expressions for the derivatives, which a Newton solver has found
6874
+ (model.diff_expressions), are dumped too. A Newton solver in a later session then takes them
6875
+ from the model instead of finding them again - the slow part of its first solve for a large model
6876
+
6871
6877
  if data=False, or the model instance has no lastdf, only the model
6872
6878
  structure is dumped and the file has no dataframes'''
6873
6879
  # print('dump ready')
@@ -6928,6 +6934,8 @@ class Json_Mixin():
6928
6934
  'modelname': self.name,
6929
6935
  'oldkwargs': self.__dict__.get('oldkwargs',{}),
6930
6936
  'run_on_load': self.__dict__.get('run_on_load', False),
6937
+ 'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
6938
+ 'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
6931
6939
  'var_description': self.var_description,
6932
6940
  'equations_latex': self.equations_latex ,
6933
6941
  'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
@@ -7065,8 +7073,11 @@ class Json_Mixin():
7065
7073
  current_per = None
7066
7074
  modelname = input['modelname']
7067
7075
  # breakpoint()
7068
- mmodel = cls(frml, modelname=modelname, funks=funks,**kwargs)
7076
+ # a model dumped with straight=True is loaded like that, unless the call says otherwise
7077
+ model_kwargs = {**({'straight': True} if input.get('straight', False) else {}), **kwargs}
7078
+ mmodel = cls(frml, modelname=modelname, funks=funks,**model_kwargs)
7069
7079
  mmodel.oldkwargs = input.get('oldkwargs',{})
7080
+ mmodel.diff_expressions = input.get('diff_expressions', {})
7070
7081
  mmodel.run_on_load = input.get('run_on_load', False)
7071
7082
  if run is None:
7072
7083
  run = mmodel.run_on_load and has_data
@@ -7123,12 +7134,12 @@ class Json_Mixin():
7123
7134
 
7124
7135
  class Zip_Mixin():
7125
7136
  '''This experimental class zips a dumped file '''
7126
- def modeldump(self, file_path='',keep=False,zip_file=True,data=True):
7137
+ def modeldump(self, file_path='',keep=False,zip_file=True,data=True,derivatives=False):
7127
7138
  pathname, dump_data = self._dump_pathname(file_path, self.has_data, data=data)
7128
7139
 
7129
7140
  pathname.parent.mkdir(parents=True, exist_ok=True)
7130
7141
 
7131
- json_string = self.modeldump_base(keep=keep,data=dump_data)
7142
+ json_string = self.modeldump_base(keep=keep,data=dump_data,derivatives=derivatives)
7132
7143
 
7133
7144
 
7134
7145
  if zip_file:
@@ -7640,7 +7651,8 @@ class Solver_Mixin():
7640
7651
  # this is how convergence is measured
7641
7652
  convplace = [databank.columns.get_loc(c) for c in convvar]
7642
7653
  convergence = True
7643
- endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
7654
+ endovar = list(self.endogene)
7655
+ endoplace = [databank.columns.get_loc(c) for c in endovar]
7644
7656
  # breakpoint()
7645
7657
  if ldumpvar:
7646
7658
  self.dumplist = []
@@ -7668,6 +7680,7 @@ class Solver_Mixin():
7668
7680
  values[row, c] = values[row-1, c]
7669
7681
  itbefore = values[row, convplace]
7670
7682
  self.pro2d(values, values, row, 1.0)
7683
+ corenan = False
7671
7684
  for iteration in range(max_iterations):
7672
7685
  with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
7673
7686
  self.solve2d(values, values, row, alfa)
@@ -7679,18 +7692,30 @@ class Solver_Mixin():
7679
7692
  if iteration > first_test:
7680
7693
  itafter = values[row, convplace]
7681
7694
  select = absconv <= np.abs(itbefore)
7695
+ # NaN drops out of select, so it has to be tested on its own
7696
+ nonfinite = nonfinite_places(itafter)
7682
7697
  convergence = (
7683
- np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
7698
+ np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
7684
7699
  if convergence:
7685
7700
  if not silent:
7686
7701
  print(
7687
7702
  f'{self.periode} Solved in {iteration} iterations')
7688
7703
  break
7704
+ if len(nonfinite):
7705
+ corenan = True
7706
+ bad = [convvar[i] for i in nonfinite]
7707
+ print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
7708
+ break
7689
7709
  itbefore = itafter
7690
7710
  else:
7691
7711
  print(f'{self.periode} not converged in {iteration} iterations')
7692
-
7712
+
7693
7713
  self.epi2d(values, values, row, 1.0)
7714
+ # the convergence test only sees the core, prolog/epilog can also be NaN
7715
+ nonfinite = nonfinite_places(values[row, endoplace])
7716
+ if len(nonfinite) and not corenan:
7717
+ bad = [endovar[i] for i in nonfinite]
7718
+ print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
7694
7719
  pbar.update()
7695
7720
  endtime = time.time()
7696
7721
 
@@ -7975,7 +8000,8 @@ class Solver_Mixin():
7975
8000
  # convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
7976
8001
  convplace = [self.allvar[c]['startnr'] -
7977
8002
  self.allvar[c]['maxlead'] for c in convvar]
7978
- endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
8003
+ endovar = list(self.endogene)
8004
+ endoplace = [databank.columns.get_loc(c) for c in endovar]
7979
8005
 
7980
8006
  convergence = True
7981
8007
 
@@ -8010,6 +8036,7 @@ class Solver_Mixin():
8010
8036
  itbeforeold = [a[c] for c in convplace]
8011
8037
  itbefore = a[convplace]
8012
8038
  this_pro1d(a, 1.0)
8039
+ corenan = False
8013
8040
  for iteration in range(max_iterations):
8014
8041
  with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
8015
8042
  this_solve1d(a, alfa)
@@ -8022,20 +8049,32 @@ class Solver_Mixin():
8022
8049
  itafter = a[convplace]
8023
8050
  # breakpoint()
8024
8051
  select = absconv <= np.abs(itbefore)
8052
+ # NaN drops out of select, so it has to be tested on its own
8053
+ nonfinite = nonfinite_places(itafter)
8025
8054
  convergence = (
8026
- np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
8055
+ np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
8027
8056
  if convergence:
8028
8057
  if not silent:
8029
8058
  print(
8030
8059
  f'{self.periode} Solved in {iteration} iterations')
8031
8060
  break
8061
+ if len(nonfinite):
8062
+ corenan = True
8063
+ bad = [convvar[i] for i in nonfinite]
8064
+ print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
8065
+ break
8032
8066
  itbefore = itafter
8033
8067
  else:
8034
8068
  print(f'{self.periode} not converged in {iteration} iterations')
8035
-
8069
+
8036
8070
  this_epi1d(a, 1.0)
8037
8071
 
8038
8072
  self.saveeval3(values, row, a)
8073
+ # the convergence test only sees the core, prolog/epilog can also be NaN
8074
+ nonfinite = nonfinite_places(values[row, endoplace])
8075
+ if len(nonfinite) and not corenan:
8076
+ bad = [endovar[i] for i in nonfinite]
8077
+ print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
8039
8078
 
8040
8079
  if ldumpvar:
8041
8080
  self.dumpdf = pd.DataFrame(self.dumplist)
@@ -9617,9 +9656,23 @@ class Solver_Mixin():
9617
9656
 
9618
9657
  def errfunk(self, values, linenr, overhead=4, overeq=0):
9619
9658
  '''Development function to handle run time errors in model calculations'''
9620
-
9659
+
9621
9660
  import pandas as pd
9622
-
9661
+
9662
+ if getattr(self, 'errfunk_defer', False):
9663
+ # a Newton trial step: keep the error point, the solver dumps it only if it gives up
9664
+ self.errfunk_deferred = (values.copy(), linenr, overhead, overeq)
9665
+ return
9666
+
9667
+ if hasattr(self, 'dumplist'):
9668
+ # first, so the returns below keep it, and it must never mask the real error
9669
+ dumplist = self.dumplist
9670
+ del self.dumplist
9671
+ try:
9672
+ self.dumpdf = pd.DataFrame(dumplist, columns=['fair', 'per', 'iteration'] + list(self.dump))
9673
+ except Exception as e:
9674
+ print('>> Could not build .dumpdf:', e)
9675
+
9623
9676
  def find_log_arguments(text):
9624
9677
  """Return all arguments inside balanced LOG(...) calls."""
9625
9678
  out = []
@@ -9666,7 +9719,8 @@ class Solver_Mixin():
9666
9719
  outeq = self.allvar[errvar]['frml']
9667
9720
 
9668
9721
  print('>>:', outeq,)
9669
- print('In:', self.periode,'\n')
9722
+ per = getattr(self, 'periode', None) # not set if the error comes before the period loop
9723
+ print('In:', per if per is not None else 'unknown period', '\n')
9670
9724
  print('Below you will find information which might help you identify the problem')
9671
9725
  terms = self.allvar[errvar]['terms']
9672
9726
 
@@ -9679,7 +9733,7 @@ class Solver_Mixin():
9679
9733
 
9680
9734
 
9681
9735
  try:
9682
- basepos = self.errdump.index.get_loc(self.periode)
9736
+ basepos = self.errdump.index.get_loc(per)
9683
9737
  rows = []
9684
9738
  seen = set()
9685
9739
 
@@ -9718,8 +9772,8 @@ class Solver_Mixin():
9718
9772
  for arg in log_args:
9719
9773
  try:
9720
9774
  val = self.errdump.mfcalc(
9721
- f'<{self.periode}> {tmpvar} = {arg}'
9722
- ).loc[self.periode, tmpvar]
9775
+ f'<{per}> {tmpvar} = {arg}'
9776
+ ).loc[per, tmpvar]
9723
9777
  status = 'Bad log argument' if pd.isna(val) or val <= 0 else 'ok'
9724
9778
  except Exception:
9725
9779
  val = ''
@@ -9735,13 +9789,8 @@ class Solver_Mixin():
9735
9789
 
9736
9790
  print('\nA snapshot of the data at the error point is at .errdump')
9737
9791
  print('Also .lastdf contains a copy of .errdump for inspecting')
9738
-
9739
- if hasattr(self, 'dumplist'):
9740
- self.dumpdf = pd.DataFrame(self.dumplist)
9741
- del self.dumplist
9742
- self.dumpdf.columns = ['fair', 'per', 'iteration'] + self.dump
9743
-
9744
-
9792
+
9793
+
9745
9794
  def show_iterations(self, pat='*', per='', last=0, change=False,top=.9):
9746
9795
  '''
9747
9796
  shows the last iterations for the most recent simulation.
@@ -2366,7 +2366,8 @@ class Makemodel(BaseExplode):
2366
2366
  self.markdown_model = (mfmod_list_to_codeblock(self.original_statements))
2367
2367
 
2368
2368
  elif self.type_input == 'single':
2369
- extracted_frml = self.original_statements.replace('\n', '')
2369
+ # join the lines with a space, so a tag ending a line stays apart from the equation
2370
+ extracted_frml = ' '.join(line.strip() for line in self.original_statements.splitlines()).strip()
2370
2371
  self.markdown_model = ''
2371
2372
 
2372
2373
  else:
@@ -2685,6 +2686,46 @@ class Makemodel(BaseExplode):
2685
2686
  cols[name] = ser
2686
2687
  return pd.DataFrame(cols)
2687
2688
 
2689
+ def actual_fitted(self, level: bool = True):
2690
+ """Actual, fitted and residuals of all estimated equations as one DataFrame.
2691
+
2692
+ The columns have two levels: the endogenous variable, and ``Actual``,
2693
+ ``Fitted``, ``Residuals``. Take one equation with
2694
+ ``mm.actual_fitted()['CON']``.
2695
+
2696
+ level : bool, default True
2697
+ True: the endogenous variable itself. When the left-hand side is
2698
+ ``DLOG(CON)`` this is ``CON``, its one-step fitted value (the
2699
+ estimated equation solved for ``CON``, lagged values are the
2700
+ actual ones) and the residual in the units of ``CON``.
2701
+ False: the left-hand side as it was estimated (``DLOG(CON)``).
2702
+
2703
+ Each equation covers its own estimation sample, so rows outside it
2704
+ are NaN. An equation which can't be solved for its variable is left
2705
+ out with ``level=True``.
2706
+
2707
+ Per-equation access: ``rec['estimator_object'].af_level_df`` (or
2708
+ ``.af_df`` and ``.residuals_df``) on the entries of
2709
+ :attr:`estimation_records`; ``modelestimator_new.plot_actual_vs_fitted
2710
+ (est, level=True)`` draws the chart of the report.
2711
+ """
2712
+ import pandas as pd
2713
+ frames = {}
2714
+ for rec in self.estimation_records:
2715
+ est = rec.get('estimator_object')
2716
+ if not _is_estimator_backend_instance(est):
2717
+ continue
2718
+ frame = est.af_level_df if level else est.af_df.join(est.residuals_df)
2719
+ if frame is None:
2720
+ continue
2721
+ name = getattr(est, 'endo_var', '') or rec.get('frmlname', '')
2722
+ if level: # the variable is the outer column level
2723
+ frame = frame.rename(columns=lambda c: c.removeprefix(f'{est.endo_var}__'))
2724
+ frames[name] = frame
2725
+ if not frames:
2726
+ return pd.DataFrame()
2727
+ return pd.concat(frames, axis=1)
2728
+
2688
2729
  @property
2689
2730
  def markdown_with_estimation(self) -> str:
2690
2731
  """Original Markdown input with compact estimation tables inserted.
@@ -3085,6 +3126,18 @@ class Listmodels(BaseExplode):
3085
3126
  return pd.DataFrame()
3086
3127
  return pd.concat(frames, axis=1)
3087
3128
 
3129
+ def actual_fitted(self, level: bool = True):
3130
+ """Actual, fitted and residuals of all member Makemodels, combined column-wise.
3131
+
3132
+ See :meth:`Makemodel.actual_fitted` for the layout.
3133
+ """
3134
+ import pandas as pd
3135
+ frames = [mex.actual_fitted(level=level) for mex in self.makemodels]
3136
+ frames = [f for f in frames if not f.empty]
3137
+ if not frames:
3138
+ return pd.DataFrame()
3139
+ return pd.concat(frames, axis=1)
3140
+
3088
3141
  @property
3089
3142
  def markdown_with_estimation(self) -> str:
3090
3143
  """Concatenate member Makemodel markdown-with-estimation strings."""
@@ -1378,6 +1378,67 @@ class EstimatorBackend(Eq_parent, ABC):
1378
1378
  .rename(columns=lambda s: s.lower().capitalize())
1379
1379
  )
1380
1380
 
1381
+ # ---- A/F and residuals of the endogenous variable itself ---------------
1382
+
1383
+ @cached_property
1384
+ def lhs_is_transformed(self) -> bool:
1385
+ """True when the left-hand side is a transformation of the endogenous
1386
+ variable (``DLOG(CON)``, ``DIFF(CON)``, ``LOG(CON/P)`` ...), so
1387
+ :attr:`af_df` is not in the units of the variable itself."""
1388
+ lhs = self.org_eq_clean.split("=", 1)[0]
1389
+ return "".join(lhs.split()).upper() != self.endo_var.upper()
1390
+
1391
+ @cached_property
1392
+ def af_level_df(self) -> Optional[pd.DataFrame]:
1393
+ """Actual, fitted and residuals of the endogenous variable itself.
1394
+
1395
+ :attr:`af_df` and :attr:`residuals_df` are in the units of the
1396
+ left-hand side as it was estimated, e.g. ``DLOG(CON)``. Here the
1397
+ estimated equation is solved for the variable (``CON``):
1398
+
1399
+ * ``Actual`` the variable,
1400
+ * ``Fitted`` the normalized equation with the estimated
1401
+ coefficients, evaluated on the data. Lagged values of the variable
1402
+ are the actual ones, so this is the one-step (static) fit, not a
1403
+ dynamic simulation,
1404
+ * ``Residuals`` ``Actual - Fitted``, in the units of the variable.
1405
+
1406
+ The columns are prefixed with the variable name and ``__``
1407
+ (``CON__Actual``, ``CON__Fitted``, ``CON__Residuals``), so frames of
1408
+ several equations can be joined side by side.
1409
+
1410
+ When the left-hand side is the variable itself the frame holds the
1411
+ same numbers as :attr:`af_df` and :attr:`residuals_df`. ``None`` (and
1412
+ a message) when the equation can't be solved for the variable.
1413
+ """
1414
+ prefix = f"{self.endo_var}__"
1415
+ if not self.lhs_is_transformed:
1416
+ return self.af_df.join(self.residuals_df).add_prefix(prefix)
1417
+ start, end = self.smpl
1418
+ try:
1419
+ normalized = nz.normal(
1420
+ self.org_eq_baked, add_add_factor=False
1421
+ ).normalized
1422
+ lhs, rhs = normalized.split("=", 1)
1423
+ if lhs.strip().upper() != self.endo_var.upper():
1424
+ raise ValueError(f"normalized to {lhs.strip()!r}")
1425
+ res = self.eq_var_df.mfcalc(
1426
+ f"<{start},{end}> FITTED_LEVEL = {rhs}"
1427
+ )
1428
+ out = pd.DataFrame({
1429
+ "Actual": res.loc[start:end, self.endo_var],
1430
+ "Fitted": res.loc[start:end, "FITTED_LEVEL"],
1431
+ })
1432
+ except Exception as e:
1433
+ print(
1434
+ f"{type(self).__name__}({self.endo_var}): no actual/fitted "
1435
+ f"for the variable itself, the equation could not be solved "
1436
+ f"for it ({type(e).__name__}: {str(e)[:100]})"
1437
+ )
1438
+ return None
1439
+ out["Residuals"] = out["Actual"] - out["Fitted"]
1440
+ return out.add_prefix(prefix)
1441
+
1381
1442
  @cached_property
1382
1443
  def estimation_smpl(self) -> Tuple[Any, Any]:
1383
1444
  """The usable sample, derived from non-missing residuals.
@@ -2088,6 +2149,11 @@ class LSResult:
2088
2149
  title_html = self._build_title_html()
2089
2150
  summary_html = self.olsmodel._render_summary_html()
2090
2151
  plot_html = self._build_plot_html(plot_format)
2152
+ # the same for the endogenous variable itself, when the left-hand
2153
+ # side is a transformation of it (DLOG(CON) -> CON)
2154
+ if (self.olsmodel.lhs_is_transformed
2155
+ and self.olsmodel.af_level_df is not None):
2156
+ plot_html += self._build_plot_html(plot_format, level=True)
2091
2157
  return title_html + summary_html + plot_html
2092
2158
 
2093
2159
  def show(self, plot_format: str = "svg") -> None:
@@ -2124,17 +2190,21 @@ class LSResult:
2124
2190
  f"<pre><code>{normalized}</code></pre></p>"
2125
2191
  )
2126
2192
 
2127
- def _build_plot_html(self, plot_format: str) -> str:
2193
+ def _build_plot_html(self, plot_format: str, level: bool = False) -> str:
2128
2194
  buf = BytesIO()
2129
2195
  plot_actual_vs_fitted(
2130
2196
  self.olsmodel, save_to=buf, format=plot_format,
2131
- figsize=(8, 5), dpi=150,
2197
+ figsize=(8, 5), dpi=150, level=level,
2132
2198
  )
2133
2199
  buf.seek(0)
2134
2200
  b64 = base64.b64encode(buf.read()).decode("utf-8")
2135
2201
  mime = "svg+xml" if plot_format == "svg" else "png"
2202
+ heading = (
2203
+ f"Actual vs Fitted Plot: {self.olsmodel.endo_var} itself "
2204
+ "(one-step fit)" if level else "Actual vs Fitted Plot"
2205
+ )
2136
2206
  return (
2137
- "<h3>Actual vs Fitted Plot</h3>"
2207
+ f"<h3>{heading}</h3>"
2138
2208
  f'<img style="max-width:100%; height:auto;" '
2139
2209
  f'src="data:image/{mime};base64,{b64}" />'
2140
2210
  )
@@ -2440,28 +2510,46 @@ def plot_actual_vs_fitted(
2440
2510
  format: str = "png",
2441
2511
  figsize: Tuple[float, float] = (7, 3),
2442
2512
  dpi: int = 150,
2513
+ level: bool = False,
2443
2514
  ):
2444
2515
  """Plot Actual vs Fitted (top panel) and Residuals (bottom panel).
2445
2516
 
2517
+ ``level=False`` plots the left-hand side as it was estimated (``af_df``,
2518
+ ``residuals_df``), ``level=True`` the endogenous variable itself
2519
+ (``af_level_df``): ``CON`` when the left-hand side is ``DLOG(CON)``.
2520
+
2446
2521
  If ``save_to`` is a file-like object, the plot is saved there and the
2447
2522
  figure is closed; otherwise it is shown interactively.
2448
2523
  """
2524
+ if level:
2525
+ if emodel.af_level_df is None:
2526
+ raise ValueError(
2527
+ f"{emodel.endo_var}: no actual/fitted for the variable itself"
2528
+ )
2529
+ prefix = f"{emodel.endo_var}__"
2530
+ af_df = residuals_df = emodel.af_level_df.rename(
2531
+ columns=lambda c: c.removeprefix(prefix))
2532
+ title = f"Actual vs Fitted: {emodel.endo_var} itself (one-step fit)"
2533
+ else:
2534
+ af_df, residuals_df = emodel.af_df, emodel.residuals_df
2535
+ title = f"Actual vs Fitted: {emodel.endo_var}"
2536
+
2449
2537
  fig = plt.figure(figsize=figsize, dpi=dpi)
2450
2538
  gs = GridSpec(3, 1, height_ratios=[2, 0.1, 1])
2451
2539
 
2452
2540
  ax1 = fig.add_subplot(gs[0])
2453
2541
  ax2 = fig.add_subplot(gs[2], sharex=ax1)
2454
2542
 
2455
- ax1.plot(emodel.af_df.index, emodel.af_df["Actual"],
2543
+ ax1.plot(af_df.index, af_df["Actual"],
2456
2544
  label="Actual", linewidth=2)
2457
- ax1.plot(emodel.af_df.index, emodel.af_df["Fitted"],
2545
+ ax1.plot(af_df.index, af_df["Fitted"],
2458
2546
  label="Fitted", linestyle="--")
2459
- ax1.set_title(f"Actual vs Fitted: {emodel.endo_var}")
2547
+ ax1.set_title(title)
2460
2548
  ax1.set_ylabel(emodel.endo_var)
2461
2549
  ax1.grid(True)
2462
2550
  ax1.legend()
2463
2551
 
2464
- ax2.plot(emodel.residuals_df.index, emodel.residuals_df["Residuals"],
2552
+ ax2.plot(residuals_df.index, residuals_df["Residuals"],
2465
2553
  color="gray")
2466
2554
  ax2.axhline(0, color="red", linestyle="--", linewidth=1)
2467
2555
  ax2.set_title("Residuals")