ModelFlowIb 2.88__tar.gz → 2.89__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {modelflowib-2.88 → modelflowib-2.89}/PKG-INFO +1 -1
  2. {modelflowib-2.88 → modelflowib-2.89}/modelclass.py +48 -27
  3. {modelflowib-2.88 → modelflowib-2.89}/modelflowib.egg-info/PKG-INFO +1 -1
  4. {modelflowib-2.88 → modelflowib-2.89}/modelgrabexcel.py +0 -4
  5. {modelflowib-2.88 → modelflowib-2.89}/modelmanipulation.py +8 -3
  6. {modelflowib-2.88 → modelflowib-2.89}/modelpattern.py +0 -24
  7. {modelflowib-2.88 → modelflowib-2.89}/modelsolver_ng.py +213 -158
  8. {modelflowib-2.88 → modelflowib-2.89}/pyproject.toml +1 -1
  9. {modelflowib-2.88 → modelflowib-2.89}/README.md +0 -0
  10. {modelflowib-2.88 → modelflowib-2.89}/license.md +0 -0
  11. {modelflowib-2.88 → modelflowib-2.89}/modelBLfunk.py +0 -0
  12. {modelflowib-2.88 → modelflowib-2.89}/model_Excel.py +0 -0
  13. {modelflowib-2.88 → modelflowib-2.89}/model_cvx.py +0 -0
  14. {modelflowib-2.88 → modelflowib-2.89}/model_dynare.py +0 -0
  15. {modelflowib-2.88 → modelflowib-2.89}/model_financial_stability.py +0 -0
  16. {modelflowib-2.88 → modelflowib-2.89}/model_latex.py +0 -0
  17. {modelflowib-2.88 → modelflowib-2.89}/model_latex_class.py +0 -0
  18. {modelflowib-2.88 → modelflowib-2.89}/model_parquet_mixin.py +0 -0
  19. {modelflowib-2.88 → modelflowib-2.89}/modelcausality.py +0 -0
  20. {modelflowib-2.88 → modelflowib-2.89}/modelconstruct.py +0 -0
  21. {modelflowib-2.88 → modelflowib-2.89}/modelconstruct_estimation.py +0 -0
  22. {modelflowib-2.88 → modelflowib-2.89}/modeldash.py +0 -0
  23. {modelflowib-2.88 → modelflowib-2.89}/modeldashboot.py +0 -0
  24. {modelflowib-2.88 → modelflowib-2.89}/modeldashsidebar.py +0 -0
  25. {modelflowib-2.88 → modelflowib-2.89}/modeldekom.py +0 -0
  26. {modelflowib-2.88 → modelflowib-2.89}/modeldiff.py +0 -0
  27. {modelflowib-2.88 → modelflowib-2.89}/modeldisplay.py +0 -0
  28. {modelflowib-2.88 → modelflowib-2.89}/modelestimation.py +0 -0
  29. {modelflowib-2.88 → modelflowib-2.89}/modelestimator_new.py +0 -0
  30. {modelflowib-2.88 → modelflowib-2.89}/modelflowib.egg-info/SOURCES.txt +0 -0
  31. {modelflowib-2.88 → modelflowib-2.89}/modelflowib.egg-info/dependency_links.txt +0 -0
  32. {modelflowib-2.88 → modelflowib-2.89}/modelflowib.egg-info/requires.txt +0 -0
  33. {modelflowib-2.88 → modelflowib-2.89}/modelflowib.egg-info/top_level.txt +0 -0
  34. {modelflowib-2.88 → modelflowib-2.89}/modelgrab.py +0 -0
  35. {modelflowib-2.88 → modelflowib-2.89}/modelgrabgdx.py +0 -0
  36. {modelflowib-2.88 → modelflowib-2.89}/modelgrabgekkobank.py +0 -0
  37. {modelflowib-2.88 → modelflowib-2.89}/modelgrabwf2.py +0 -0
  38. {modelflowib-2.88 → modelflowib-2.89}/modelhelp.py +0 -0
  39. {modelflowib-2.88 → modelflowib-2.89}/modelhtml.py +0 -0
  40. {modelflowib-2.88 → modelflowib-2.89}/modelinput_shiny.py +0 -0
  41. {modelflowib-2.88 → modelflowib-2.89}/modelinvert.py +0 -0
  42. {modelflowib-2.88 → modelflowib-2.89}/modeljupyter.py +0 -0
  43. {modelflowib-2.88 → modelflowib-2.89}/modeljupytermagic.py +0 -0
  44. {modelflowib-2.88 → modelflowib-2.89}/modelmacrograb.py +0 -0
  45. {modelflowib-2.88 → modelflowib-2.89}/modelmf.py +0 -0
  46. {modelflowib-2.88 → modelflowib-2.89}/modelnet.py +0 -0
  47. {modelflowib-2.88 → modelflowib-2.89}/modelnewton.py +0 -0
  48. {modelflowib-2.88 → modelflowib-2.89}/modelnormalize.py +0 -0
  49. {modelflowib-2.88 → modelflowib-2.89}/modelreport.py +0 -0
  50. {modelflowib-2.88 → modelflowib-2.89}/modeluserfunk.py +0 -0
  51. {modelflowib-2.88 → modelflowib-2.89}/modelvis.py +0 -0
  52. {modelflowib-2.88 → modelflowib-2.89}/modelwidget.py +0 -0
  53. {modelflowib-2.88 → modelflowib-2.89}/modelwidget_core.py +0 -0
  54. {modelflowib-2.88 → modelflowib-2.89}/modelwidget_input.py +0 -0
  55. {modelflowib-2.88 → modelflowib-2.89}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.88
3
+ Version: 2.89
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -154,9 +154,14 @@ class BaseModel():
154
154
  var_description={}, model_description = '',
155
155
  var_groups = {}, reports = {}, equations_latex='',
156
156
  eviews_dict = {},use_fbmin= True,
157
- substitution = {},
157
+ substitution = {}, ng = False,
158
158
  **kwargs):
159
- ''' initialize a model'''
159
+ ''' initialize a model
160
+
161
+ ng : if True a call without solver= dispatches to the next-generation
162
+ solvers (xgenr_ng, sim_ng, newton_ng, newtonstack_ng) instead of
163
+ the legacy ones; can also be set later with ``mmodel.ng = True``'''
164
+ self.ng = ng
160
165
  if i_eq != '':
161
166
  self.funks = funks
162
167
  self.equations = i_eq if '$' in i_eq else mp.tofrml(i_eq, sep='\n')
@@ -225,14 +230,16 @@ class BaseModel():
225
230
  mmodel.equations_original = equations
226
231
  return mmodel
227
232
 
228
- def get_histmodel(self):
229
- """ return a model instance with a model which generates historic values for equations
233
+ def get_histmodel(self, tags=('IDENT', 'I')):
234
+ """ return a model instance with a model which generates historic values for equations
230
235
  marked by a frml name I or IDENT
231
-
236
+
237
+ tags is a string or a list of strings, the frml names to select (default IDENT and I)
238
+
232
239
  Uses :any:`find_hist_model`
233
-
240
+
234
241
  """
235
- hist_eq = mp.find_hist_model(self.equations)
242
+ hist_eq = mp.find_hist_model(self.equations, tags=tags)
236
243
  return type(self)(hist_eq, funks=self.funks)
237
244
 
238
245
  def analyzemodelnew(self, silent):
@@ -282,7 +289,8 @@ class BaseModel():
282
289
  mega = [(f,nt) for f,nt in mega_all if not pt.kw_frml_name(f.frmlname, 'CALC_ADD_FACTOR')]
283
290
  mega_calc_add_factor = [(f,nt) for f,nt in mega_all if pt.kw_frml_name(f.frmlname, 'CALC_ADD_FACTOR')]
284
291
  calc_add_factor_frml = [f'FRML <CALC> {f.expression}' for f,nt in mega_calc_add_factor]
285
- self.calc_add_factor_model = model(' '.join(calc_add_factor_frml),funks=self.funks,modelname='Calculate add factors')
292
+ self.calc_add_factor_model = model(' '.join(calc_add_factor_frml),funks=self.funks,modelname='Calculate add factors',
293
+ ng=self.ng)
286
294
  if not self.calc_add_factor_model.istopo:
287
295
  raise Exception('The add factor calculation model should be recursive')
288
296
  else:
@@ -631,7 +639,7 @@ class BaseModel():
631
639
  def fixed_period_guard(self, what):
632
640
  ''' Only the ng solvers can evaluate a variable at a fixed period, X(2002).
633
641
  Raises an exception naming *what* (the legacy evaluator) if the model has such references '''
634
- if getattr(self, 'fixed_periods', None):
642
+ if self.fixed_periods:
635
643
  refs = ', '.join(f'{var}({lag})' for var, lag in self.fixed_terms[:6])
636
644
  more = ' ...' if len(self.fixed_terms) > 6 else ''
637
645
  raise Exception(f'{self.name} refers to variables at fixed periods: {refs}{more}\n'
@@ -2636,7 +2644,7 @@ class Dekomp_Mixin():
2636
2644
  end_ = end if end != '' else self.current_per[-1]
2637
2645
 
2638
2646
  mfrml = model(self.allvar[varnavn]['frml'],
2639
- funks=self.funks) # calculate the formular
2647
+ funks=self.funks, ng=self.ng) # calculate the formular
2640
2648
  print_per = mfrml.smpl(start_, end_, altdf_)
2641
2649
 
2642
2650
  old_start, old_end = altdf_.index.slice_locs(
@@ -2677,8 +2685,9 @@ class Dekomp_Mixin():
2677
2685
  difdf = {e: smallalt - alldf[e] for e in eksperiments}
2678
2686
  # allres = {e : mfrml.xgenr(alldf[e],str(e[1]),str(e[1]),silent= True ) for e in eksperiments} # now evaluate each experiment
2679
2687
  # now evaluate each experiment
2680
- # a variable at a fixed period (X(2002)) is only evaluated by the ng xgenr
2681
- xgenr = mfrml.xgenr_ng if mfrml.fixed_periods else mfrml.xgenr
2688
+ # the ng xgenr with the ng option; a variable at a fixed period (X(2002))
2689
+ # is only evaluated by the ng xgenr
2690
+ xgenr = mfrml.xgenr_ng if mfrml.ng or mfrml.fixed_periods else mfrml.xgenr
2682
2691
  allres = {e: xgenr(
2683
2692
  alldf[e], e[1], e[1], silent=True) for e in eksperiments}
2684
2693
  # dataframes with the effect of each update
@@ -3743,8 +3752,9 @@ class Graph_Mixin():
3743
3752
  self.FVS_graph.add_edge(u, v)
3744
3753
 
3745
3754
  h,a = nx.hits(self.FVS_graph)
3746
- self.fbhubs = {k:v for k,v in sorted( h.items(), key=lambda x: x[1], reverse=True)}
3747
- self.fbaut = {k:v for k,v in sorted( a.items(), key=lambda x: x[1], reverse=True)}
3755
+ # rounded and ties broken by name, so the order is the same in every session
3756
+ self.fbhubs = {k:v for k,v in sorted( h.items(), key=lambda x: (-round(x[1], 12), x[0]))}
3757
+ self.fbaut = {k:v for k,v in sorted( a.items(), key=lambda x: (-round(x[1], 12), x[0]))}
3748
3758
 
3749
3759
  self.fblist = [v for v in self.fbhubs.keys()]
3750
3760
  except Exception as e:
@@ -3980,21 +3990,27 @@ class Graph_Mixin():
3980
3990
  is acyclic.
3981
3991
  """
3982
3992
 
3983
- # Create a copy of the graph to avoid modifying the original graph
3984
- H = G.copy()
3993
+ # Build a copy with sorted nodes and edges. The graphs are made from sets of
3994
+ # variable names, whose order changes between Python sessions (hash
3995
+ # randomization), and find_cycle follows the insertion order - so without
3996
+ # this the feedback set, and with it the last digits of a solution, could
3997
+ # differ from session to session.
3998
+ H = nx.DiGraph()
3999
+ H.add_nodes_from(sorted(G.nodes()))
4000
+ H.add_edges_from(sorted(G.edges()))
3985
4001
  fvs = set()
3986
-
4002
+
3987
4003
  # Function to calculate the degree of a vertex in the cycle
3988
4004
  def vertex_degree(vertex):
3989
4005
  return H.degree(vertex)
3990
-
4006
+
3991
4007
  # Iterate until the graph becomes acyclic
3992
4008
  while True:
3993
4009
  try:
3994
4010
  # Find a cycle in the graph
3995
4011
  cycle = nx.find_cycle(H, orientation='original')
3996
- # Find the vertex in the cycle with the highest degree
3997
- vertex_to_remove = max(cycle, key=lambda x: vertex_degree(x[0]))[0]
4012
+ # Find the vertex in the cycle with the highest degree, ties broken by name
4013
+ vertex_to_remove = max(cycle, key=lambda x: (vertex_degree(x[0]), x[0]))[0]
3998
4014
  # Add the vertex to the feedback vertex set
3999
4015
  fvs.add(vertex_to_remove)
4000
4016
  # Remove the vertex from the graph
@@ -4003,10 +4019,10 @@ class Graph_Mixin():
4003
4019
  # If no cycle is found, the graph is now acyclic, and we break the loop
4004
4020
  break
4005
4021
 
4006
- # Perform a topological sort on the modified graph
4007
- topological_sorted_order = list(nx.topological_sort(H))
4008
-
4009
- return list(fvs) , topological_sorted_order
4022
+ # Perform a topological sort on the modified graph, ties broken by name
4023
+ topological_sorted_order = list(nx.lexicographical_topological_sort(H))
4024
+
4025
+ return sorted(fvs) , topological_sorted_order
4010
4026
 
4011
4027
 
4012
4028
 
@@ -7358,8 +7374,10 @@ class Solver_Mixin():
7358
7374
  else:
7359
7375
  solverguess = 'newton_un_normalized'
7360
7376
 
7361
- # a variable at a fixed period, X(2002), is only evaluated by the ng solvers
7362
- if getattr(self, 'fixed_periods', None):
7377
+ # with ng=True (model option) the ng solvers are the default; a variable
7378
+ # at a fixed period, X(2002), is only evaluated by the ng solvers.
7379
+ # A normalized simultaneous model gets sim_ng - Newton must be asked for.
7380
+ if self.ng or self.fixed_periods:
7363
7381
  solverguess = {'xgenr': 'xgenr_ng', 'sim': 'sim_ng',
7364
7382
  'newtonstack': 'newtonstack_ng',
7365
7383
  'newtonstack_un_normalized': 'newtonstack_ng',
@@ -7411,7 +7429,8 @@ class Solver_Mixin():
7411
7429
  if not silent:
7412
7430
  print('Running calc_adjust_model ')
7413
7431
  print(f'Dummies set {self.find_fix_dummy_fixed(outdf)}')
7414
- self.calc_add_factor_model.current_per=self.current_per
7432
+ self.calc_add_factor_model.current_per=self.current_per
7433
+ self.calc_add_factor_model.ng = self.ng # follows a later mmodel.ng = ...
7415
7434
  out_add_factor = self.calc_add_factor_model(outdf,silent=silent) # calculate the adjustment factors
7416
7435
 
7417
7436
  calc_add_factordf = out_add_factor.loc[self.current_per,self.fix_add_factor] # new adjust values
@@ -9863,6 +9882,8 @@ class Fix_Mixin():
9863
9882
  selected = [v for up in pat.split() for v in fnmatch.filter(beh, up.upper())]
9864
9883
  exo = [v+'_X' for v in selected ]
9865
9884
  dummy = [v+'_D' for v in selected ]
9885
+ # _X/_D columns read as integers (all zero) can not take the float values
9886
+ dataframe[exo + dummy] = dataframe[exo + dummy].astype('float64')
9866
9887
  # breakpoint()
9867
9888
  with self.set_smpl(start,end,df=dataframe):
9868
9889
  dataframe.loc[self.current_per,dummy] = 1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.88
3
+ Version: 2.89
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -21,16 +21,12 @@ Some of the docstring are not very informative, to be improved.
21
21
 
22
22
  """
23
23
  import pandas as pd
24
- import networkx as nx
25
- import openpyxl
26
24
  from openpyxl import load_workbook
27
25
  from openpyxl.formula import Tokenizer
28
26
  from openpyxl.utils import get_column_letter
29
27
  from openpyxl.utils import cols_from_range,rows_from_range
30
28
  from openpyxl.worksheet.formula import ArrayFormula, DataTableFormula
31
29
 
32
- import matplotlib.pylab as plt
33
- import seaborn as sns
34
30
  from pathlib import Path
35
31
 
36
32
  import modelclass as mc
@@ -211,13 +211,18 @@ def find_res_dynare_new(equations):
211
211
 
212
212
  return '\n'.join(out)
213
213
 
214
- def find_hist_model(equations):
214
+ def find_hist_model(equations, tags=('IDENT', 'I')):
215
215
  ''' takes a unrolled model and create a model which can be run for historic periode \n
216
- and the identities are also calculeted'''
216
+ and the identities are also calculeted
217
+
218
+ tags is a string or a list of strings. A frml is included when its frml name
219
+ has one of the tags. The default is the identities (IDENT or I), but
220
+ for instance tags='HIST' makes a model of the frmls marked <HIST>'''
221
+ tags = [tags] if isinstance(tags, str) else tags
217
222
  hist = []
218
223
  for f in find_frml(equations):
219
224
  a, fr, n, udtryk = split_frml(f.upper())
220
- if kw_frml_name(n, 'IDENT') or kw_frml_name(n, 'I'):
225
+ if any(kw_frml_name(n, tag.upper()) for tag in tags):
221
226
  # identites are just replicated in order to calculate backdata
222
227
  hist.append(f.strip())
223
228
  return('\n'.join(hist))
@@ -15,7 +15,6 @@ import re
15
15
  import inspect
16
16
  from collections import namedtuple
17
17
  from collections import defaultdict
18
- from functools import lru_cache
19
18
  from typing import NamedTuple, Optional
20
19
 
21
20
  import modelmanipulation as mp
@@ -206,34 +205,11 @@ def model_parse(equations,funks=[]):
206
205
  # nterm = namedtuple('nterm', [ 'number', 'op', 'var', 'lag'])
207
206
  expressionre= udtrykre(funks)
208
207
  frmls = [split_frml(f) for f in find_frml(equations.upper())]
209
- check_lagged_functions(frmls)
210
208
  ibh = [(fatoms(*c),
211
209
  [ nterm(t[0],t[1],t[2], lag_field(t[3])) for t in expressionre.findall(c[3])])
212
210
  for c in frmls]
213
211
  return ibh
214
212
 
215
- @lru_cache(maxsize=32)
216
- def lagged_function_re(names):
217
- ''' A compiled pattern for a function name written with a lag or a period in quotes:
218
- EXP(-1), EXP(+2), EXP('2002Q1'). names is a tuple of function names '''
219
- return re.compile(r'(?<![A-Z0-9_{}.])(' + '|'.join(re.escape(f) for f in names)
220
- + r''')\(([+-][0-9]+|'[0-9][0-9A-Z-]*'|"[0-9][0-9A-Z-]*")\)''')
221
-
222
- def check_lagged_functions(frmls):
223
- ''' Raises an exception if a function name is written with a lag, EXP(-1). The tokenizer
224
- reads it as the function, so a variable with the name of a function would silently get a
225
- wrong value. frmls is a list of split_frml results, the expression is element 3.
226
- Uses funkname, so call udtrykre(funks) first '''
227
- guard = lagged_function_re(tuple(funkname))
228
- if not guard.search('\n'.join(c[3] for c in frmls)): # one search for the whole model
229
- return
230
- for c in frmls:
231
- if (m := guard.search(c[3])):
232
- f, arg = m.groups()
233
- constant = '' if arg[:1] in '\'"' else f'\nFor the function of a constant write {f}({arg}.0)'
234
- raise Exception(f'{f}({arg}) in {c[0].strip()}\n{f} is a function in the business language '
235
- f'and can not be a variable with a lag. Rename the variable.{constant}')
236
-
237
213
 
238
214
  def list_extract(equations, silent=True, add_auto_sublists=True):
239
215
  ''' creates lists used in a model
@@ -847,6 +847,8 @@ class SolverBase:
847
847
  + res-style feedback equations parts=...)
848
848
  'stackeq' dict of per-equation res-style (gen_eqdict) {var: eq}
849
849
  functions (stacked fbmin sweep)
850
+ 'stacksplit' stacked split: prolog/epilog gauss, (gen_2d, (pro,core,epi)
851
+ core res lines over given orders parts=...)
850
852
  ========== ================================================== =========
851
853
 
852
854
  The equation-to-source translation is done by the module-level ``gen_2d`` /
@@ -872,6 +874,12 @@ class SolverBase:
872
874
  databank, transpile_reset=transpile_reset,
873
875
  newdata=newdata, silent=silent)
874
876
 
877
+ if solvename == "stacksplit":
878
+ return self._makelos_stacksplit(
879
+ databank, kwargs["orders"], ljit=ljit, stringjit=stringjit,
880
+ chunk=chunk, transpile_reset=transpile_reset, newdata=newdata,
881
+ silent=silent, debug=kwargs.get("debug", 1))
882
+
875
883
  jitname = f"{m.name}_{solvename}_jit"
876
884
  nojitname = f"{m.name}_{solvename}_nojit"
877
885
  # rows of fixed periods (X(2002)) are compiled into the code
@@ -953,6 +961,48 @@ class SolverBase:
953
961
  setattr(m, attr, globals()["make_los"](m.funks, m.errfunk))
954
962
  return getattr(m, attr)
955
963
 
964
+ def _makelos_stacksplit(self, databank, orders, *, ljit=0, stringjit=False,
965
+ chunk=30, transpile_reset=False, newdata=False,
966
+ silent=True, debug=1):
967
+ """Compile + cache the stacked-split triple (pro, core, epi) for
968
+ ``newtonstack_ng`` with ``split=1``.
969
+
970
+ *orders* is ``(prolog, core, epilog)``, variable lists from the stacked
971
+ split. ``prolog`` / ``epilog`` are gauss lines (in-place on ``values``,
972
+ nodamp, in per-period topological order) run in a period loop once
973
+ before / after Newton; ``core`` is res lines (LHS -> ``outvalues``) run
974
+ in a period loop every Newton iteration. Jitted like the other ng
975
+ solvers, in its own transpiled file
976
+ ``modelsource/<model>_stacksplit_jit_jitsolver.py``.
977
+ """
978
+ m = self.m
979
+ attr = f"ng_stacksplit_{m.name}_{'jit' if ljit else 'nojit'}".replace(" ", "_")
980
+ newdata = self._fixrow_changed(attr, databank) or newdata
981
+ orders = tuple(tuple(o) for o in orders)
982
+ if (newdata or transpile_reset or not hasattr(m, attr)
983
+ or getattr(m, attr + "_orders", None) != orders):
984
+ if not silent:
985
+ print(f"makelos_ng compiles the stacked split evaluators for {m.name}"
986
+ + (" (jit)" if ljit else ""))
987
+ pro, core, epi = orders
988
+ parts = [("prolog", list(pro), True, "gauss"),
989
+ ("core", list(core), True, "res"),
990
+ ("epilog", list(epi), True, "gauss")]
991
+ solveout = partial(gen_2d, m, databank, debug=debug, chunk=chunk,
992
+ ljit=ljit, parts=parts)
993
+ if ljit and not stringjit:
994
+ funcs = self._import_transpiled(
995
+ f"{m.name}_stacksplit_jit", solveout,
996
+ ("prolog", "core", "epilog"), transpile_reset)
997
+ else:
998
+ make_los_text = solveout()
999
+ m.make_los_text = make_los_text
1000
+ exec(make_los_text, globals()) # creates make_los (returns 3 funcs)
1001
+ funcs = globals()["make_los"](m.funks, m.errfunk)
1002
+ setattr(m, attr, funcs)
1003
+ setattr(m, attr + "_orders", orders)
1004
+ return getattr(m, attr)
1005
+
956
1006
  #: the fbmin evaluator quadruple, in generation / return order
957
1007
  FBMIN_PARTNAMES = ("prolog", "dag", "fb", "epilog")
958
1008
 
@@ -2471,6 +2521,18 @@ class NewtonStackSolver(SolverBase):
2471
2521
  """Dump order: the declared (base) endogenous names of the Jacobian object."""
2472
2522
  return ctx.model.ng_newton_diff_stack.declared_endo_list
2473
2523
 
2524
+ def build_evaluator(self, ctx):
2525
+ """Compile the evaluators. With ``split=1`` (the default) the
2526
+ stacked-split triple from ``_prepare_split``: prolog and epilog as
2527
+ gauss lines run once before / after Newton, the core as res lines --
2528
+ see ``_build_split``. With ``split=0`` the model's res triple
2529
+ (prolog, core, epilog over the whole model, every Newton iteration)."""
2530
+ ctx.split = bool(self._opt(ctx, "split", True))
2531
+ if ctx.split:
2532
+ ctx.split = self._prepare_split(ctx)
2533
+ if not ctx.split:
2534
+ super().build_evaluator(ctx)
2535
+
2474
2536
  def prepare(self, ctx):
2475
2537
  """Build the stacked Jacobian solver, the residual mask and the flattened
2476
2538
  (period x variable) row/column index arrays used by ``iterate``."""
@@ -2479,12 +2541,9 @@ class NewtonStackSolver(SolverBase):
2479
2541
  silent = opt("silent", self.DEFAULT_SILENT)
2480
2542
  timeit = opt("timeit", False)
2481
2543
 
2482
- # stacked pro/core/epi split (``split=1``): Newton only on the core,
2483
- # prolog/epilog solved exactly by stacked sweeps -- see _prepare_split
2484
- split = bool(opt("split", False))
2485
- self.pro_pairs, self.epi_pairs, self.core_pairs = [], [], []
2486
- if split:
2487
- self._prepare_split(ctx)
2544
+ # stacked pro/core/epi split (``split=1``, set up in build_evaluator):
2545
+ # Newton and the stacked Jacobian only on the core
2546
+ split = ctx.split
2488
2547
 
2489
2548
  # the cached diff object is shared with newtonstack_fbmin_ng, so it is
2490
2549
  # tagged with what it was built for ('full' or the core equation list);
@@ -2560,33 +2619,29 @@ class NewtonStackSolver(SolverBase):
2560
2619
  [ctx.newton_col_endo for r in ctx.stackrows]).flatten() # unknowns
2561
2620
 
2562
2621
  def _prepare_split(self, ctx):
2563
- """Stacked pro/core/epi split (option ``split=1``).
2564
-
2565
- The per-period prolog/core/epilog decomposition lifted to the stacked
2566
- graph: *core* variables take part in some cycle of the collapsed
2567
- (variable-level) dependency graph -- or are ``___RES`` equations, or
2568
- lie on a path from the core back into the core (hybrid models) --
2569
- and become the only stacked Newton unknowns; *prolog* variables never
2570
- depend on the core at any lag or lead and are solved exactly by one
2571
- stacked topological sweep before Newton; *epilog* (reporting)
2572
- variables -- the rest, which never feed back into the core -- are
2573
- swept once after convergence. Conservative at the collapsed level: a
2574
- cycle whose lags sum to a nonzero value is not a true stacked cycle,
2575
- but its members still go to the core -- the split can only be too
2576
- small, never wrong.
2622
+ """Stacked pro/core/epi split (option ``split=1``), called from
2623
+ ``build_evaluator``.
2624
+
2625
+ The variables are classified once per model (``_build_split``, cached
2626
+ as ``m.ng_stack_split_struct``) and the three parts compiled like the
2627
+ other solvers' prolog/core/epilog (``makelos('stacksplit')``, jitted
2628
+ with ``ljit``): ``ctx.pro`` and ``ctx.epi`` are gauss lines run in a
2629
+ period loop once before / after Newton, ``ctx.solve`` the core as res
2630
+ lines run in a period loop every Newton iteration. Only the core
2631
+ equations are evaluated during Newton: evaluating a reporting equation
2632
+ at an intermediate iterate can raise (LOG of a transient negative
2633
+ value).
2634
+
2635
+ Returns False -- solve without split -- when the core is empty.
2577
2636
  """
2578
2637
  m, databank = ctx.model, ctx.databank
2579
2638
  opt = lambda n, d: self._opt(ctx, n, d)
2580
2639
  silent = opt("silent", self.DEFAULT_SILENT)
2581
- per = ctx.sol_periode
2582
2640
 
2583
- # purely structural: depends on the equations and the solve span, not
2584
- # on data -- deliberately NOT invalidated by newton_reset (that only
2585
- # refreshes the Jacobian values via the solver rebuild)
2586
- struct_stale = (not hasattr(m, "ng_stack_split_struct")
2587
- or m.ng_stack_split_struct["first"] != per[0]
2588
- or m.ng_stack_split_struct["last"] != per[-1]
2589
- or m.ng_stack_split_struct["n_per"] != len(per))
2641
+ # purely structural: depends only on the equations, so neither
2642
+ # newton_reset nor a new span invalidates it
2643
+ struct_stale = (getattr(m, "ng_stack_split_struct", {}).get("kind")
2644
+ != "periodloop")
2590
2645
  if struct_stale:
2591
2646
  if not silent:
2592
2647
  print("Building the stacked pro/core/epi split")
@@ -2596,53 +2651,91 @@ class NewtonStackSolver(SolverBase):
2596
2651
  print(f"Stacked split: {len(struct['core']):,} core / "
2597
2652
  f"{len(struct['pro']):,} prolog / "
2598
2653
  f"{len(struct['epi']):,} epilog variables")
2654
+ if not struct["core"]:
2655
+ if not silent:
2656
+ print("newtonstack_ng split=1: the core is empty, solving without split")
2657
+ return False
2599
2658
 
2600
- # per-equation evaluators for the sweeps (cached on the model by makelos)
2601
- eqdict = self.makelos(
2602
- "stackeq", databank, newdata=ctx.newdata,
2603
- transpile_reset=opt("transpile_reset", False), silent=silent)
2604
- rowof = [databank.index.get_loc(p) for p in per]
2605
- self.pro_pairs = [(eqdict[v], rowof[t]) for v, t in struct["pro_nodes"]]
2606
- self.epi_pairs = [(eqdict[v], rowof[t]) for v, t in struct["epi_nodes"]]
2607
- # the Newton residual is evaluated with the core equations only:
2608
- # prolog/epilog residuals are never read, and evaluating them at
2609
- # intermediate Newton states can raise (LOG of a transient negative
2610
- # core value in a reporting equation)
2611
- self.core_pairs = [(eqdict[v], r) for r in rowof
2612
- for v in struct["core"]]
2659
+ ctx.pro, ctx.solve, ctx.epi = self.makelos(
2660
+ "stacksplit", databank,
2661
+ orders=(struct["pro"], struct["core"], struct["epi"]),
2662
+ ljit=opt("ljit", False),
2663
+ stringjit=opt("stringjit", False),
2664
+ transpile_reset=opt("transpile_reset", False),
2665
+ chunk=opt("chunk", 30),
2666
+ newdata=ctx.newdata,
2667
+ silent=silent)
2668
+ m.genrcolumns = databank.columns.copy()
2669
+ m.genrindex = databank.index.copy()
2670
+ return True
2613
2671
 
2614
2672
  def _build_split(self, ctx):
2615
- """Compute the pro/core/epi variable sets on the collapsed dependency
2616
- graph and the stacked topological orders (variable name, period index)
2617
- of the pro and epi nodes. Only the structural differentiation of the
2618
- full model is used -- the derivatives are never evaluated."""
2619
- m, databank = ctx.model, ctx.databank
2620
- T = len(ctx.sol_periode)
2621
-
2622
- fulldiff = newton_diff(m, forcenum=True, df=databank, silent=True)
2623
-
2624
- def varlag(s):
2625
- # lag is an int, or the period as written ('2002', "'2002Q1'") for a
2626
- # variable at a fixed period: s is written like an equation, see tovarlag
2627
- if "(" in s:
2628
- name, lag = s.split("(", 1)
2629
- return name, pt.lag_value(pt.lag_field(lag[:-1]))
2630
- return s, 0
2631
-
2632
- endo = m.endogene
2673
+ """Classify the endogenous variables into prolog / core / epilog so
2674
+ that each part runs as a compiled period loop.
2675
+
2676
+ The collapsed (variable-level) dependency graph is built from the
2677
+ equations' own right-hand-side terms (``m.allvar[v]['terms']`` with
2678
+ their lags) -- exact, also for kinks like RECODE, which a derivative
2679
+ structure can miss. Lags and leads couple the periods; a self-edge
2680
+ with lag != 0 is a recursive chain across periods, not simultaneity,
2681
+ and is left out of the graph so cumulating variables stay outside the
2682
+ core.
2683
+
2684
+ * core: ``___RES`` equations, variables in a cycle of the collapsed
2685
+ graph or depending on their own current value, variables depending
2686
+ on themselves at both a lag and a lead (a true stacked cycle the
2687
+ collapsed graph can not see) or at a fixed period, and variables on
2688
+ a path from the core back into the core. The core is the Newton
2689
+ system.
2690
+ * prolog: never depends on the core, at any lag or lead -- its values
2691
+ are final before Newton starts.
2692
+ * epilog: never feeds the core -- computed after convergence; it may
2693
+ read core and prolog values at any lag or lead, they are final.
2694
+ * period-loop rule: a forward period loop is only exact when no
2695
+ prolog (epilog) variable reads a lead -- or a fixed period -- of a
2696
+ prolog (epilog) variable, itself included. Such variables go to
2697
+ the core, repeated until no variable moves (moving one can change
2698
+ the group of the variables depending on it).
2699
+
2700
+ Within a period the prolog and the epilog are ordered topologically
2701
+ on their current-period edges (acyclic: every collapsed cycle is in
2702
+ the core). The core is sorted; that is the row order of the stacked
2703
+ Jacobian. Depends only on the equations, not on the span.
2704
+ """
2705
+ m = ctx.model
2706
+ endo = set(m.endogene)
2633
2707
  eqname = lambda x: x + "___RES" if x + "___RES" in endo else x
2634
2708
  # (dependency equation, equation, lag): eq depends on dep's unknown at lag
2635
- cedges = {(eqname(varlag(rhs)[0]), v, varlag(rhs)[1])
2636
- for v, deps in fulldiff.diffendocur.items() for rhs in deps}
2709
+ edges = {(eqname(t.var), v, pt.lag_value(t.lag))
2710
+ for v in endo
2711
+ for t in m.allvar[v]["terms"][m.allvar[v]["assigpos"]:]
2712
+ if t.var and eqname(t.var) in endo}
2713
+ is_lead = lambda l: pt.is_fixed_lag(l) or l > 0
2637
2714
 
2638
- # collapsed variable-level graph; a self-edge with lag != 0 is a
2639
- # recursive chain across periods, not simultaneity -- leave it out so
2640
- # cumulating reporting variables stay sweepable
2641
2715
  cg = nx.DiGraph()
2642
- cg.add_nodes_from(fulldiff.endovar)
2643
- cg.add_edges_from((a, b) for a, b, l in cedges if a != b or l == 0)
2716
+ cg.add_nodes_from(endo)
2717
+ cg.add_edges_from((a, b) for a, b, l in edges if a != b or l == 0)
2718
+
2719
+ def closed(core):
2720
+ """The core plus the variables on a path from the core back into
2721
+ the core. As epilog variables they would only be computed after
2722
+ convergence, so the core equations depending on them would iterate
2723
+ against stale values (happens in hybrid models, where the ___RES
2724
+ equations forced into the core leave normalized equations between
2725
+ them)."""
2726
+ if not core:
2727
+ return core
2728
+ desc = set().union(*(nx.descendants(cg, v) for v in core))
2729
+ anc = set().union(*(nx.ancestors(cg, v) for v in core))
2730
+ return core | (desc & anc)
2731
+
2732
+ def groups(core):
2733
+ """(prolog, epilog) for a given core."""
2734
+ desc = (set().union(*(nx.descendants(cg, v) for v in core))
2735
+ if core else set())
2736
+ return endo - core - desc, desc - core
2644
2737
 
2645
- core = {v for v in fulldiff.endovar if v.endswith("___RES")}
2738
+ core = {v for v in endo if v.endswith("___RES")}
2646
2739
  for scc in nx.strongly_connected_components(cg):
2647
2740
  if len(scc) > 1:
2648
2741
  core |= scc
@@ -2660,75 +2753,34 @@ class NewtonStackSolver(SolverBase):
2660
2753
  # promoted: in that period it is its own current value. Conservative
2661
2754
  # when the period is outside the span.
2662
2755
  selflags = {}
2663
- for a, b, l in cedges:
2756
+ for a, b, l in edges:
2664
2757
  if a == b:
2665
2758
  selflags.setdefault(a, set()).add(l)
2666
2759
  core |= {v for v, lags in selflags.items()
2667
2760
  if any(pt.is_fixed_lag(l) for l in lags)
2668
2761
  or (min(lags) < 0 and max(lags) > 0)}
2669
2762
 
2670
- # a variable on a path from the core back into the core (a descendant
2671
- # AND an ancestor of the core) must be solved with the core: as an
2672
- # epilog variable it would only be swept after convergence, so the
2673
- # core equations depending on it would iterate against a stale value
2674
- # (wrong solution). It happens when the core is not one strongly
2675
- # connected block -- notably in a hybrid model, where the ___RES
2676
- # equations forced into the core leave normalized equations between
2677
- # them
2678
- if core:
2679
- desc = set().union(*(nx.descendants(cg, v) for v in core))
2680
- anc = set().union(*(nx.ancestors(cg, v) for v in core))
2681
- core |= desc & anc
2682
-
2683
- desc = (set().union(*(nx.descendants(cg, v) for v in core))
2684
- if core else set())
2685
- epi_vars = [v for v in fulldiff.endovar if v not in core and v in desc]
2686
- pro_vars = [v for v in fulldiff.endovar
2687
- if v not in core and v not in desc]
2688
-
2689
- # period number in the span of each fixed period (None outside the span)
2690
- fixedpos = {}
2691
- for a, b, l in cedges:
2692
- if pt.is_fixed_lag(l) and l not in fixedpos:
2693
- try:
2694
- fixedpos[l] = pt.fixed_period_loc(ctx.sol_periode, l)
2695
- except KeyError:
2696
- fixedpos[l] = None
2697
-
2698
- def source(l, t):
2699
- """Period number of a dependency at lag *l* seen from period *t*,
2700
- None outside the span (a fixed period is the same for every t)."""
2701
- s = fixedpos[l] if pt.is_fixed_lag(l) else t + l
2702
- return s if s is not None and 0 <= s < T else None
2703
-
2704
- def node_order(vars_):
2705
- """Stacked topological order of (var, t) nodes over *vars_* --
2706
- acyclic by construction (any collapsed cycle and any mixed-sign
2707
- self-dependence went to the core)."""
2708
- vset = set(vars_)
2763
+ core = closed(core)
2764
+ pro, epi = groups(core)
2765
+ # period-loop rule
2766
+ while True:
2767
+ bad = {b for a, b, l in edges if is_lead(l)
2768
+ and ((a in pro and b in pro) or (a in epi and b in epi))}
2769
+ if not bad:
2770
+ break
2771
+ core = closed(core | bad)
2772
+ pro, epi = groups(core)
2773
+
2774
+ def period_order(vars_):
2775
+ """Topological order of *vars_* on their current-period edges."""
2709
2776
  g = nx.DiGraph()
2710
- g.add_nodes_from((v, t) for v in vars_ for t in range(T))
2711
- g.add_edges_from(((a, s), (b, t))
2712
- for a, b, l in cedges
2713
- if a in vset and b in vset
2714
- for t in range(T) if (s := source(l, t)) is not None)
2715
- try:
2716
- return list(nx.topological_sort(g))
2717
- except nx.NetworkXUnfeasible:
2718
- # should be impossible -- the classification above is exact.
2719
- # If it ever fires the split logic has a hole: fall back to
2720
- # split=0 and please report the model structure.
2721
- raise RuntimeError(
2722
- "newtonstack_ng split=1: the pro/epi node graph contains "
2723
- "a cycle the classification missed -- run with split=0 "
2724
- "and report this model structure") from None
2725
-
2726
- return {"first": ctx.sol_periode[0], "last": ctx.sol_periode[-1],
2727
- "n_per": T,
2728
- "core": [v for v in fulldiff.endovar if v in core],
2729
- "pro": pro_vars, "epi": epi_vars,
2730
- "pro_nodes": node_order(pro_vars),
2731
- "epi_nodes": node_order(epi_vars)}
2777
+ g.add_nodes_from(vars_)
2778
+ g.add_edges_from((a, b) for a, b, l in edges
2779
+ if l == 0 and a != b and a in vars_ and b in vars_)
2780
+ return list(nx.lexicographical_topological_sort(g))
2781
+
2782
+ return {"kind": "periodloop", "core": sorted(core),
2783
+ "pro": period_order(pro), "epi": period_order(epi)}
2732
2784
 
2733
2785
  def iterate(self, ctx):
2734
2786
  """Stacked Newton over the whole span (relative-change stop). The equation
@@ -2758,12 +2810,12 @@ class NewtonStackSolver(SolverBase):
2758
2810
  colidx_endo = ctx.stackcolindex_endo
2759
2811
  resmask = ctx.is_residual_eq_stacked
2760
2812
 
2761
- # split mode: prolog variables never depend on the core -- one exact
2762
- # stacked sweep fixes them for the whole solve
2763
- if self.pro_pairs:
2764
- with m.timer("prolog sweep (stacked)", timeit):
2765
- for func, row in self.pro_pairs:
2766
- func(values, values, row)
2813
+ # split mode: prolog variables never depend on the core -- one period
2814
+ # loop before Newton makes them final for the whole solve
2815
+ if ctx.split:
2816
+ with m.timer("prolog (period loop)", timeit):
2817
+ for m.periode, row in zip(ctx.sol_periode, ctx.stackrows):
2818
+ ctx.pro(values, outvalues, row, alfa)
2767
2819
 
2768
2820
  residual = None
2769
2821
  last_base = last_step = None # previous Newton step, for backoff
@@ -2772,23 +2824,20 @@ class NewtonStackSolver(SolverBase):
2772
2824
  before = values[rowidx, colidx_endo]
2773
2825
  with m.timer("calculate new solution", timeit):
2774
2826
  # split mode: evaluate only the core equations -- see
2775
- # _prepare_split (prolog/epilog residuals never read,
2776
- # and reporting equations can raise at Newton iterates).
2777
- # If the evaluation raises (LOG/SQRT domain error at an
2778
- # infeasible iterate), halve the previous Newton step and
2779
- # re-evaluate.
2827
+ # _prepare_split. If the evaluation raises (LOG/SQRT
2828
+ # domain error at an infeasible iterate), halve the
2829
+ # previous Newton step and re-evaluate.
2780
2830
  for backoff in range(8):
2781
2831
  try:
2782
- if self.core_pairs:
2783
- for func, row in self.core_pairs:
2784
- func(values, outvalues, row)
2785
- ctx.ittotal += len(ctx.stackrows)
2832
+ if ctx.split:
2833
+ for m.periode, row in zip(ctx.sol_periode, ctx.stackrows):
2834
+ ctx.solve(values, outvalues, row, alfa)
2786
2835
  else:
2787
2836
  for m.periode, row in zip(ctx.sol_periode, ctx.stackrows):
2788
2837
  ctx.pro(values, outvalues, row, alfa)
2789
2838
  ctx.solve(values, outvalues, row, alfa)
2790
2839
  ctx.epi(values, outvalues, row, alfa)
2791
- ctx.ittotal += 1
2840
+ ctx.ittotal += len(ctx.stackrows)
2792
2841
  break
2793
2842
  except Exception:
2794
2843
  if last_step is None or backoff == 7:
@@ -2856,11 +2905,11 @@ class NewtonStackSolver(SolverBase):
2856
2905
  ctx.iteration = iteration
2857
2906
 
2858
2907
  # split mode: epilog (reporting) variables never feed back into the
2859
- # core -- one exact stacked sweep after convergence
2860
- if self.epi_pairs:
2861
- with m.timer("epilog sweep (stacked)", timeit):
2862
- for func, row in self.epi_pairs:
2863
- func(values, values, row)
2908
+ # core -- one period loop after convergence
2909
+ if ctx.split:
2910
+ with m.timer("epilog (period loop)", timeit):
2911
+ for m.periode, row in zip(ctx.sol_periode, ctx.stackrows):
2912
+ ctx.epi(values, outvalues, row, alfa)
2864
2913
 
2865
2914
  # record the equation residual F(y)-y per period at the last iteration (opt-in)
2866
2915
  if keep_residual and residual is not None:
@@ -3418,10 +3467,15 @@ class NewtonStackFbminSolver(SolverBase):
3418
3467
  self._dag_sweep(ctx)
3419
3468
  ctx.dag_sweeps += 1
3420
3469
 
3470
+ # the residual printed in the loop is from before that iteration's
3471
+ # Newton step, so evaluate it again at the final solution
3472
+ residual = self._fb_residual(ctx)
3473
+ newton_conv = np.max(np.abs(residual))
3474
+
3421
3475
  ctx.iteration = iteration
3422
3476
  ctx.convergence = converged
3423
3477
 
3424
- # the feedback residual at the last iteration (opt-in); a Series over
3478
+ # the feedback residual at the final solution (opt-in); a Series over
3425
3479
  # (period, variable) -- the fb variables may differ between periods
3426
3480
  if keep_residual and residual is not None:
3427
3481
  m.ng_stackfbmin_residual = pd.Series(
@@ -3431,10 +3485,10 @@ class NewtonStackFbminSolver(SolverBase):
3431
3485
 
3432
3486
  if not converged:
3433
3487
  print(f"Not converged in {iteration + 1} outer iterations "
3434
- f"(last max res~{newton_conv:,.6g})")
3488
+ f"(max feedback residual at the end {newton_conv:,.6g})")
3435
3489
  elif not silent:
3436
3490
  print(f"Solved in {iteration + 1} outer iterations "
3437
- f"(last max res~{newton_conv:,.6g})")
3491
+ f"(max feedback residual at the solution {newton_conv:,.6g})")
3438
3492
 
3439
3493
  def stats_lines(self, ctx):
3440
3494
  """Stacked fbmin statistics: size of the stacked feedback set vs the
@@ -3779,11 +3833,12 @@ class Solver_ng_Mixin:
3779
3833
  def newtonstack_ng(self, databank=None, *args, **kwargs):
3780
3834
  """Solve with next-generation stacked Newton (defaults to ``self.basedf``).
3781
3835
 
3782
- ``split=1``: pro/core/epi decomposition at the stacked level -- Newton
3783
- (and the stacked LU) only on the variables involved in collapsed-graph
3784
- cycles; upstream (prolog) variables swept exactly once before, pure
3785
- reporting (epilog) variables once after convergence. Analogous to the
3786
- per-period prolog/core/epilog split of the one-period solvers."""
3836
+ ``split=1`` (the default; ``split=0`` turns it off): pro/core/epi
3837
+ decomposition at the stacked level -- Newton
3838
+ (and the stacked LU) only on the core variables; upstream (prolog)
3839
+ variables computed in one period loop before, pure reporting (epilog)
3840
+ variables in one period loop after convergence. All three parts are
3841
+ compiled like the other solvers (jitted with ``ljit``)."""
3787
3842
  if databank is None:
3788
3843
  databank = self.basedf
3789
3844
  return NewtonStackSolver(self)(databank, *args, **kwargs)
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "modelflowib" # PyPI distribution name
7
- version = "2.88" # OVERWRITTEN at build time from MF_VERSION
7
+ version = "2.89" # OVERWRITTEN at build time from MF_VERSION
8
8
  # (set_pyproject_version.py / make_pip.py -v)
9
9
  description = "A tool to solve and manage dynamic economic and financial models"
10
10
  readme = "README.md"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes