ModelFlowIb 2.92__tar.gz → 2.93__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {modelflowib-2.92 → modelflowib-2.93}/PKG-INFO +1 -1
  2. {modelflowib-2.92 → modelflowib-2.93}/model_parquet_mixin.py +15 -4
  3. {modelflowib-2.92 → modelflowib-2.93}/modelcausality.py +4 -0
  4. {modelflowib-2.92 → modelflowib-2.93}/modelclass.py +74 -25
  5. modelflowib-2.93/modelfactorize.py +346 -0
  6. {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/PKG-INFO +1 -1
  7. {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/SOURCES.txt +2 -0
  8. {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/top_level.txt +2 -0
  9. {modelflowib-2.92 → modelflowib-2.93}/modelhelp.py +17 -0
  10. {modelflowib-2.92 → modelflowib-2.93}/modelnewton.py +50 -1
  11. modelflowib-2.93/modelode.py +552 -0
  12. {modelflowib-2.92 → modelflowib-2.93}/modelsolver_ng.py +99 -25
  13. {modelflowib-2.92 → modelflowib-2.93}/pyproject.toml +3 -3
  14. {modelflowib-2.92 → modelflowib-2.93}/README.md +0 -0
  15. {modelflowib-2.92 → modelflowib-2.93}/license.md +0 -0
  16. {modelflowib-2.92 → modelflowib-2.93}/modelBLfunk.py +0 -0
  17. {modelflowib-2.92 → modelflowib-2.93}/model_Excel.py +0 -0
  18. {modelflowib-2.92 → modelflowib-2.93}/model_cvx.py +0 -0
  19. {modelflowib-2.92 → modelflowib-2.93}/model_dynare.py +0 -0
  20. {modelflowib-2.92 → modelflowib-2.93}/model_financial_stability.py +0 -0
  21. {modelflowib-2.92 → modelflowib-2.93}/model_latex.py +0 -0
  22. {modelflowib-2.92 → modelflowib-2.93}/model_latex_class.py +0 -0
  23. {modelflowib-2.92 → modelflowib-2.93}/modelconstruct.py +0 -0
  24. {modelflowib-2.92 → modelflowib-2.93}/modelconstruct_estimation.py +0 -0
  25. {modelflowib-2.92 → modelflowib-2.93}/modeldash.py +0 -0
  26. {modelflowib-2.92 → modelflowib-2.93}/modeldashboot.py +0 -0
  27. {modelflowib-2.92 → modelflowib-2.93}/modeldashsidebar.py +0 -0
  28. {modelflowib-2.92 → modelflowib-2.93}/modeldekom.py +0 -0
  29. {modelflowib-2.92 → modelflowib-2.93}/modeldiff.py +0 -0
  30. {modelflowib-2.92 → modelflowib-2.93}/modeldisplay.py +0 -0
  31. {modelflowib-2.92 → modelflowib-2.93}/modelestimation.py +0 -0
  32. {modelflowib-2.92 → modelflowib-2.93}/modelestimator_new.py +0 -0
  33. {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/dependency_links.txt +0 -0
  34. {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/requires.txt +0 -0
  35. {modelflowib-2.92 → modelflowib-2.93}/modelgrab.py +0 -0
  36. {modelflowib-2.92 → modelflowib-2.93}/modelgrabexcel.py +0 -0
  37. {modelflowib-2.92 → modelflowib-2.93}/modelgrabgdx.py +0 -0
  38. {modelflowib-2.92 → modelflowib-2.93}/modelgrabgekkobank.py +0 -0
  39. {modelflowib-2.92 → modelflowib-2.93}/modelgrabwf2.py +0 -0
  40. {modelflowib-2.92 → modelflowib-2.93}/modelhtml.py +0 -0
  41. {modelflowib-2.92 → modelflowib-2.93}/modelinput_shiny.py +0 -0
  42. {modelflowib-2.92 → modelflowib-2.93}/modelinvert.py +0 -0
  43. {modelflowib-2.92 → modelflowib-2.93}/modeljupyter.py +0 -0
  44. {modelflowib-2.92 → modelflowib-2.93}/modeljupytermagic.py +0 -0
  45. {modelflowib-2.92 → modelflowib-2.93}/modelmacrograb.py +0 -0
  46. {modelflowib-2.92 → modelflowib-2.93}/modelmanipulation.py +0 -0
  47. {modelflowib-2.92 → modelflowib-2.93}/modelmf.py +0 -0
  48. {modelflowib-2.92 → modelflowib-2.93}/modelnet.py +0 -0
  49. {modelflowib-2.92 → modelflowib-2.93}/modelnormalize.py +0 -0
  50. {modelflowib-2.92 → modelflowib-2.93}/modelpattern.py +0 -0
  51. {modelflowib-2.92 → modelflowib-2.93}/modelreport.py +0 -0
  52. {modelflowib-2.92 → modelflowib-2.93}/modeluserfunk.py +0 -0
  53. {modelflowib-2.92 → modelflowib-2.93}/modelvis.py +0 -0
  54. {modelflowib-2.92 → modelflowib-2.93}/modelwidget.py +0 -0
  55. {modelflowib-2.92 → modelflowib-2.93}/modelwidget_core.py +0 -0
  56. {modelflowib-2.92 → modelflowib-2.93}/modelwidget_input.py +0 -0
  57. {modelflowib-2.92 → modelflowib-2.93}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.92
3
+ Version: 2.93
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -164,7 +164,7 @@ class Parquet_Mixin:
164
164
 
165
165
  # ── dump ──────────────────────────────────────────────────────────────
166
166
 
167
- def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
167
+ def modeldump(self, file_path='', keep=False, large=False, compact=False, derivatives=False, **kwargs):
168
168
  """Dump model to disk.
169
169
 
170
170
  Parameters
@@ -181,6 +181,12 @@ class Parquet_Mixin:
181
181
  If True (only effective when large=True), store float64
182
182
  columns as float32 to roughly halve file size.
183
183
  On load the data is restored to float64.
184
+ derivatives : bool
185
+ If True, also store the expressions for the derivatives that a Newton
186
+ solver has found (``model.diff_expressions``). A Newton solver in a later
187
+ session takes them from the model instead of finding them again, which is
188
+ the slow part of its first solve for a large model. They are used only if
189
+ the equations and the solver options are the same.
184
190
 
185
191
  A model instance without a lastdf holds no dataframe, so the feather
186
192
  format has nothing to be fast about. Such a model is always written as a
@@ -194,7 +200,7 @@ class Parquet_Mixin:
194
200
 
195
201
  if not large or nodata:
196
202
  # ── original JSON/gzip path ──────────────────────────────────
197
- return super().modeldump(file_path=file_path, keep=keep, **kwargs)
203
+ return super().modeldump(file_path=file_path, keep=keep, derivatives=derivatives, **kwargs)
198
204
 
199
205
  # ── fast feather/zip path ────────────────────────────────────────
200
206
  pathname = Path(file_path)
@@ -216,8 +222,10 @@ class Parquet_Mixin:
216
222
  'format': 'feather_zip',
217
223
  'frml': self.equations,
218
224
  'modelname': self.name,
219
- 'oldkwargs': self.oldkwargs,
225
+ 'oldkwargs': self.__dict__.get('oldkwargs', {}), # a model never solved has none
220
226
  'run_on_load': self.__dict__.get('run_on_load', False),
227
+ 'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
228
+ 'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
221
229
  'var_description': self.var_description,
222
230
  'equations_latex': self.equations_latex,
223
231
  'wb_MFMSAOPTIONS': (
@@ -383,8 +391,11 @@ class Parquet_Mixin:
383
391
  # while threads work, build the model object (parses equations)
384
392
  frml = meta['frml']
385
393
  modelname = meta['modelname']
386
- mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
394
+ # a model dumped with straight=True is loaded like that, unless the call says otherwise
395
+ model_kwargs = {**({'straight': True} if meta.get('straight', False) else {}), **kwargs}
396
+ mmodel = cls(frml, modelname=modelname, funks=funks, **model_kwargs)
387
397
  mmodel.oldkwargs = meta.get('oldkwargs', {})
398
+ mmodel.diff_expressions = meta.get('diff_expressions', {})
388
399
  mmodel.run_on_load = meta.get('run_on_load', False)
389
400
  if run is None:
390
401
  run = mmodel.run_on_load
@@ -736,6 +736,10 @@ if AnyWidget is not None:
736
736
  self.observe(self._refresh, names=self._settings)
737
737
  self._refresh()
738
738
 
739
+ def __repr__(self):
740
+ # the text/plain output, shown where the widget is not run (a static book)
741
+ return f'Interactive causality viewer for {self.var}, run the notebook to use it'
742
+
739
743
  def _refresh(self, change=None):
740
744
  navn = self.var.upper()
741
745
  if navn not in self.mmodel.allvar:
@@ -116,6 +116,7 @@ from modelhelp import cutout, update_var
116
116
  from modelnormalize import normal
117
117
  from modeldekom import totdif
118
118
  from modelhelp import debug_var
119
+ from modelhelp import nonfinite_places
119
120
 
120
121
 
121
122
  # functions used in BL language
@@ -1027,7 +1028,8 @@ class BaseModel():
1027
1028
  convergence = True
1028
1029
  for after, before in zip(itafter, itbefore):
1029
1030
  # print(before,after)
1030
- if before > absconv and abs(after-before)/abs(before) > relconv:
1031
+ # NaN fails before > absconv, so it has to be tested on its own
1032
+ if not np.all(np.isfinite(after)) or (before > absconv and abs(after-before)/abs(before) > relconv):
1031
1033
  convergence = False
1032
1034
  break
1033
1035
  if convergence:
@@ -3174,7 +3176,7 @@ class Description_Mixin():
3174
3176
  out = (ind.replace('&','&#38;').replace('<', '&lt;').replace('>', '&gt;').replace('\n', '&#10;')
3175
3177
  .replace('æ', '&#230;').replace('ø', '&#248;').replace('å', '&#229;')
3176
3178
  .replace('Æ', '&#198;').replace('Ø', '&#216;').replace('Å', '&#197;')
3177
- .replace('$','&#36;')
3179
+ .replace('$','&#36;').replace('"', '&quot;')
3178
3180
  )
3179
3181
  return out
3180
3182
 
@@ -6863,11 +6865,15 @@ class Json_Mixin():
6863
6865
 
6864
6866
  return pathname, dump_data
6865
6867
 
6866
- def modeldump_base(self, outfile='',keep=False,data=True):
6868
+ def modeldump_base(self, outfile='',keep=False,data=True,derivatives=False):
6867
6869
  '''Dumps a model and its lastdf to a json file
6868
6870
 
6869
6871
  if keep=True the model.keep_solutions will alse be dumped
6870
6872
 
6873
+ if derivatives=True the expressions for the derivatives, which a Newton solver has found
6874
+ (model.diff_expressions), are dumped too. A Newton solver in a later session then takes them
6875
+ from the model instead of finding them again - the slow part of its first solve for a large model
6876
+
6871
6877
  if data=False, or the model instance has no lastdf, only the model
6872
6878
  structure is dumped and the file has no dataframes'''
6873
6879
  # print('dump ready')
@@ -6928,6 +6934,8 @@ class Json_Mixin():
6928
6934
  'modelname': self.name,
6929
6935
  'oldkwargs': self.__dict__.get('oldkwargs',{}),
6930
6936
  'run_on_load': self.__dict__.get('run_on_load', False),
6937
+ 'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
6938
+ 'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
6931
6939
  'var_description': self.var_description,
6932
6940
  'equations_latex': self.equations_latex ,
6933
6941
  'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
@@ -7065,8 +7073,11 @@ class Json_Mixin():
7065
7073
  current_per = None
7066
7074
  modelname = input['modelname']
7067
7075
  # breakpoint()
7068
- mmodel = cls(frml, modelname=modelname, funks=funks,**kwargs)
7076
+ # a model dumped with straight=True is loaded like that, unless the call says otherwise
7077
+ model_kwargs = {**({'straight': True} if input.get('straight', False) else {}), **kwargs}
7078
+ mmodel = cls(frml, modelname=modelname, funks=funks,**model_kwargs)
7069
7079
  mmodel.oldkwargs = input.get('oldkwargs',{})
7080
+ mmodel.diff_expressions = input.get('diff_expressions', {})
7070
7081
  mmodel.run_on_load = input.get('run_on_load', False)
7071
7082
  if run is None:
7072
7083
  run = mmodel.run_on_load and has_data
@@ -7123,12 +7134,12 @@ class Json_Mixin():
7123
7134
 
7124
7135
  class Zip_Mixin():
7125
7136
  '''This experimental class zips a dumped file '''
7126
- def modeldump(self, file_path='',keep=False,zip_file=True,data=True):
7137
+ def modeldump(self, file_path='',keep=False,zip_file=True,data=True,derivatives=False):
7127
7138
  pathname, dump_data = self._dump_pathname(file_path, self.has_data, data=data)
7128
7139
 
7129
7140
  pathname.parent.mkdir(parents=True, exist_ok=True)
7130
7141
 
7131
- json_string = self.modeldump_base(keep=keep,data=dump_data)
7142
+ json_string = self.modeldump_base(keep=keep,data=dump_data,derivatives=derivatives)
7132
7143
 
7133
7144
 
7134
7145
  if zip_file:
@@ -7640,7 +7651,8 @@ class Solver_Mixin():
7640
7651
  # this is how convergence is measured
7641
7652
  convplace = [databank.columns.get_loc(c) for c in convvar]
7642
7653
  convergence = True
7643
- endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
7654
+ endovar = list(self.endogene)
7655
+ endoplace = [databank.columns.get_loc(c) for c in endovar]
7644
7656
  # breakpoint()
7645
7657
  if ldumpvar:
7646
7658
  self.dumplist = []
@@ -7668,6 +7680,7 @@ class Solver_Mixin():
7668
7680
  values[row, c] = values[row-1, c]
7669
7681
  itbefore = values[row, convplace]
7670
7682
  self.pro2d(values, values, row, 1.0)
7683
+ corenan = False
7671
7684
  for iteration in range(max_iterations):
7672
7685
  with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
7673
7686
  self.solve2d(values, values, row, alfa)
@@ -7679,18 +7692,30 @@ class Solver_Mixin():
7679
7692
  if iteration > first_test:
7680
7693
  itafter = values[row, convplace]
7681
7694
  select = absconv <= np.abs(itbefore)
7695
+ # NaN drops out of select, so it has to be tested on its own
7696
+ nonfinite = nonfinite_places(itafter)
7682
7697
  convergence = (
7683
- np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
7698
+ np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
7684
7699
  if convergence:
7685
7700
  if not silent:
7686
7701
  print(
7687
7702
  f'{self.periode} Solved in {iteration} iterations')
7688
7703
  break
7704
+ if len(nonfinite):
7705
+ corenan = True
7706
+ bad = [convvar[i] for i in nonfinite]
7707
+ print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
7708
+ break
7689
7709
  itbefore = itafter
7690
7710
  else:
7691
7711
  print(f'{self.periode} not converged in {iteration} iterations')
7692
-
7712
+
7693
7713
  self.epi2d(values, values, row, 1.0)
7714
+ # the convergence test only sees the core, prolog/epilog can also be NaN
7715
+ nonfinite = nonfinite_places(values[row, endoplace])
7716
+ if len(nonfinite) and not corenan:
7717
+ bad = [endovar[i] for i in nonfinite]
7718
+ print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
7694
7719
  pbar.update()
7695
7720
  endtime = time.time()
7696
7721
 
@@ -7975,7 +8000,8 @@ class Solver_Mixin():
7975
8000
  # convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
7976
8001
  convplace = [self.allvar[c]['startnr'] -
7977
8002
  self.allvar[c]['maxlead'] for c in convvar]
7978
- endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
8003
+ endovar = list(self.endogene)
8004
+ endoplace = [databank.columns.get_loc(c) for c in endovar]
7979
8005
 
7980
8006
  convergence = True
7981
8007
 
@@ -8010,6 +8036,7 @@ class Solver_Mixin():
8010
8036
  itbeforeold = [a[c] for c in convplace]
8011
8037
  itbefore = a[convplace]
8012
8038
  this_pro1d(a, 1.0)
8039
+ corenan = False
8013
8040
  for iteration in range(max_iterations):
8014
8041
  with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
8015
8042
  this_solve1d(a, alfa)
@@ -8022,20 +8049,32 @@ class Solver_Mixin():
8022
8049
  itafter = a[convplace]
8023
8050
  # breakpoint()
8024
8051
  select = absconv <= np.abs(itbefore)
8052
+ # NaN drops out of select, so it has to be tested on its own
8053
+ nonfinite = nonfinite_places(itafter)
8025
8054
  convergence = (
8026
- np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
8055
+ np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
8027
8056
  if convergence:
8028
8057
  if not silent:
8029
8058
  print(
8030
8059
  f'{self.periode} Solved in {iteration} iterations')
8031
8060
  break
8061
+ if len(nonfinite):
8062
+ corenan = True
8063
+ bad = [convvar[i] for i in nonfinite]
8064
+ print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
8065
+ break
8032
8066
  itbefore = itafter
8033
8067
  else:
8034
8068
  print(f'{self.periode} not converged in {iteration} iterations')
8035
-
8069
+
8036
8070
  this_epi1d(a, 1.0)
8037
8071
 
8038
8072
  self.saveeval3(values, row, a)
8073
+ # the convergence test only sees the core, prolog/epilog can also be NaN
8074
+ nonfinite = nonfinite_places(values[row, endoplace])
8075
+ if len(nonfinite) and not corenan:
8076
+ bad = [endovar[i] for i in nonfinite]
8077
+ print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
8039
8078
 
8040
8079
  if ldumpvar:
8041
8080
  self.dumpdf = pd.DataFrame(self.dumplist)
@@ -9617,9 +9656,23 @@ class Solver_Mixin():
9617
9656
 
9618
9657
  def errfunk(self, values, linenr, overhead=4, overeq=0):
9619
9658
  '''Development function to handle run time errors in model calculations'''
9620
-
9659
+
9621
9660
  import pandas as pd
9622
-
9661
+
9662
+ if getattr(self, 'errfunk_defer', False):
9663
+ # a Newton trial step: keep the error point, the solver dumps it only if it gives up
9664
+ self.errfunk_deferred = (values.copy(), linenr, overhead, overeq)
9665
+ return
9666
+
9667
+ if hasattr(self, 'dumplist'):
9668
+ # first, so the returns below keep it, and it must never mask the real error
9669
+ dumplist = self.dumplist
9670
+ del self.dumplist
9671
+ try:
9672
+ self.dumpdf = pd.DataFrame(dumplist, columns=['fair', 'per', 'iteration'] + list(self.dump))
9673
+ except Exception as e:
9674
+ print('>> Could not build .dumpdf:', e)
9675
+
9623
9676
  def find_log_arguments(text):
9624
9677
  """Return all arguments inside balanced LOG(...) calls."""
9625
9678
  out = []
@@ -9666,7 +9719,8 @@ class Solver_Mixin():
9666
9719
  outeq = self.allvar[errvar]['frml']
9667
9720
 
9668
9721
  print('>>:', outeq,)
9669
- print('In:', self.periode,'\n')
9722
+ per = getattr(self, 'periode', None) # not set if the error comes before the period loop
9723
+ print('In:', per if per is not None else 'unknown period', '\n')
9670
9724
  print('Below you will find information which might help you identify the problem')
9671
9725
  terms = self.allvar[errvar]['terms']
9672
9726
 
@@ -9679,7 +9733,7 @@ class Solver_Mixin():
9679
9733
 
9680
9734
 
9681
9735
  try:
9682
- basepos = self.errdump.index.get_loc(self.periode)
9736
+ basepos = self.errdump.index.get_loc(per)
9683
9737
  rows = []
9684
9738
  seen = set()
9685
9739
 
@@ -9718,8 +9772,8 @@ class Solver_Mixin():
9718
9772
  for arg in log_args:
9719
9773
  try:
9720
9774
  val = self.errdump.mfcalc(
9721
- f'<{self.periode}> {tmpvar} = {arg}'
9722
- ).loc[self.periode, tmpvar]
9775
+ f'<{per}> {tmpvar} = {arg}'
9776
+ ).loc[per, tmpvar]
9723
9777
  status = 'Bad log argument' if pd.isna(val) or val <= 0 else 'ok'
9724
9778
  except Exception:
9725
9779
  val = ''
@@ -9735,13 +9789,8 @@ class Solver_Mixin():
9735
9789
 
9736
9790
  print('\nA snapshot of the data at the error point is at .errdump')
9737
9791
  print('Also .lastdf contains a copy of .errdump for inspecting')
9738
-
9739
- if hasattr(self, 'dumplist'):
9740
- self.dumpdf = pd.DataFrame(self.dumplist)
9741
- del self.dumplist
9742
- self.dumpdf.columns = ['fair', 'per', 'iteration'] + self.dump
9743
-
9744
-
9792
+
9793
+
9745
9794
  def show_iterations(self, pat='*', per='', last=0, change=False,top=.9):
9746
9795
  '''
9747
9796
  shows the last iterations for the most recent simulation.
@@ -0,0 +1,346 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ modelfactorize.py - the linear solvers behind the ``factorizer=`` option of the Newton solvers.
4
+
5
+ A Newton step solves ``J x = b`` with the (sparse) Jacobian J. ``make_solver(J, factorizer)`` factorizes J and
6
+ returns a function ``solve(b) -> x``. The choices:
7
+
8
+ =========== ============================================================================= ====================
9
+ factorizer method needs
10
+ =========== ============================================================================= ====================
11
+ None SuperLU through scipy (handled in modelnewton, with ``permc_spec``) -
12
+ 'superlu' the same
13
+ 'umfpack' UMFPACK through cvxopt (handled in modelnewton) cvxopt
14
+ 'pardiso' LU with MKL PARDISO. Fast and multi-threaded, but it pivots statically, so pypardiso
15
+ a Jacobian with many zeros on the diagonal can come out inaccurate. The
16
+ solve is refined against J and warns when the refinement does not get there.
17
+ 'normal' The step as a least squares problem, min |J x - b|^2: the normal equations pypardiso
18
+ J'J x = J'b (the Gauss-Newton step on the sum of squared equation errors).
19
+ J'J is symmetric positive definite, so PARDISO factorizes it (Cholesky)
20
+ without pivoting. J is scaled first, and the solve is refined against J
21
+ itself, which brings back the accuracy the squared condition number costs.
22
+ The fastest choice for a large stacked system.
23
+ 'mumps' LU with MUMPS: real pivoting and METIS ordering. Used in this process when python-mumps
24
+ ``import mumps`` works. Otherwise in a helper process, started with the
25
+ python named by the environment variable MODELFLOW_MUMPS_PYTHON (the
26
+ python.exe of an environment where python-mumps is installed).
27
+ a callable ``factorizer(J)`` must return the function ``solve(b) -> x`` -
28
+ =========== ============================================================================= ====================
29
+
30
+ The solve functions are bound methods; the object behind (``solve.__self__``) has ``info`` (a text on the
31
+ factorization) and ``last`` (relative linear residual and number of refinements of the latest solve).
32
+
33
+ Run as a script this file is the MUMPS helper process (one command per line on stdin, one answer line on stdout)::
34
+
35
+ factor <A.npz> -> ok <the info text>
36
+ solve <b.npy> <x.npy> -> ok <relative residual> <refinements>
37
+ quit
38
+ """
39
+ import os
40
+ import sys
41
+ import time
42
+
43
+ import numpy as np
44
+ import scipy.sparse as sps
45
+
46
+ REFINE_TOL = 1e-11 # relative linear residual (max norm) the refinement aims for
47
+ WARN_TOL = 1e-8 # a warning when the refinement ends above this
48
+ FAIL_TOL = 1e-2 # an exception when it ends above this: the step would be wrong
49
+
50
+
51
+ class RefinedSolver:
52
+ """A factorization of J, or of something close to J, with iterative refinement against J itself.
53
+
54
+ Subclasses set ``self.A`` (J in csr form) and ``self.info`` and implement ``_apply(r)``: the approximate
55
+ solution of ``J x = r``."""
56
+
57
+ name = ''
58
+ max_refine = 20
59
+
60
+ def __init__(self, A, verbose=False):
61
+ self.verbose = verbose
62
+ self.last = (np.nan, 0)
63
+ self.info = ''
64
+ t0 = time.time()
65
+ self.A = sps.csr_matrix(A)
66
+ self._factorize()
67
+ self.info = (f'{self.name}: {self.A.shape[0]:,} unknowns, {self.A.nnz:,} non-zeros, '
68
+ f'{time.time() - t0:.1f} s{self.info}')
69
+ if verbose:
70
+ print(self.info, flush=True)
71
+
72
+ def _factorize(self):
73
+ raise NotImplementedError
74
+
75
+ def _apply(self, r):
76
+ raise NotImplementedError
77
+
78
+ def _release(self):
79
+ """Free memory the garbage collector does not see (PARDISO keeps its factors inside MKL)."""
80
+
81
+ def release(self):
82
+ """Free the factorization now. The solve function can not be used afterwards."""
83
+ if self.__dict__.get('released'):
84
+ return
85
+ self.released = True
86
+ try:
87
+ self._release()
88
+ finally:
89
+ for attr in ('A', 'AsT', 'U', 'dr', 'dc', 's', 'ctx'):
90
+ self.__dict__.pop(attr, None)
91
+
92
+ def __del__(self):
93
+ try:
94
+ self.release()
95
+ except Exception:
96
+ pass
97
+
98
+ def solve(self, b):
99
+ if self.__dict__.get('released'):
100
+ raise RuntimeError(f"factorizer='{self.name}': this factorization has been released, factorize again")
101
+ b = np.ascontiguousarray(b, dtype=float)
102
+ nb = max(np.abs(b).max(), 1e-300)
103
+ x = self._apply(b)
104
+ rel = np.inf
105
+ for k in range(self.max_refine + 1):
106
+ r = b - self.A @ x
107
+ rel = np.abs(r).max() / nb
108
+ if not np.isfinite(rel) or rel < REFINE_TOL or k == self.max_refine:
109
+ break
110
+ x = x + self._apply(r)
111
+ self.last = (rel, k)
112
+ if not rel < FAIL_TOL:
113
+ raise ArithmeticError(f"factorizer='{self.name}' can not solve this linear system: relative residual "
114
+ f"{rel:.2g} after {k} refinements. Use another factorizer")
115
+ if not rel < WARN_TOL:
116
+ print(f"factorizer='{self.name}': the linear solve is not accurate, relative residual {rel:.2g} "
117
+ f'after {k} refinements', flush=True)
118
+ elif self.verbose:
119
+ print(f' {self.name} solve: relative linear residual {rel:.2g} after {k} refinements', flush=True)
120
+ return x
121
+
122
+
123
+ def _pypardiso():
124
+ try:
125
+ import pypardiso
126
+ except ImportError as e:
127
+ raise ImportError("factorizer='pardiso' and 'normal' need the package pypardiso "
128
+ '(conda install -c conda-forge pypardiso)') from e
129
+ return pypardiso
130
+
131
+
132
+ class _PardisoRelease:
133
+ """PARDISO keeps the factors in MKL's own memory, which is only given back by a call with phase -1.
134
+ pypardiso does not make that call when its solver object is deleted, so without this every
135
+ factorization stays in memory until the process ends."""
136
+
137
+ def _release(self):
138
+ s = self.__dict__.get('s')
139
+ if s is not None:
140
+ s.free_memory(everything=True)
141
+
142
+
143
+ class PardisoSolver(_PardisoRelease, RefinedSolver):
144
+ """LU of J with MKL PARDISO (static pivoting)."""
145
+ name = 'pardiso'
146
+
147
+ def _factorize(self):
148
+ self.s = _pypardiso().PyPardisoSolver()
149
+ self.s.factorize(self.A)
150
+
151
+ def _apply(self, r):
152
+ return self.s.solve(self.A, r)
153
+
154
+
155
+ class NormalSolver(_PardisoRelease, RefinedSolver):
156
+ """The normal equations J'J x = J'b with a PARDISO Cholesky factorization.
157
+
158
+ J is scaled to As = Dr J Dc (rows to largest element 1, then columns to norm 1), which decides the conditioning
159
+ of As'As. With x = Dc y the approximate solution of J x = r is Dc (As'As)^-1 As' Dr r."""
160
+ name = 'normal'
161
+ max_refine = 40
162
+
163
+ def _factorize(self):
164
+ A = self.A
165
+ rowmax = np.asarray(abs(A).max(axis=1).todense()).ravel()
166
+ if not rowmax.all():
167
+ raise ValueError("factorizer='normal': the Jacobian has an empty row, the system is singular")
168
+ self.dr = 1 / rowmax
169
+ As = sps.diags(self.dr) @ A
170
+ colnorm = np.sqrt(np.asarray(As.multiply(As).sum(axis=0)).ravel())
171
+ if not colnorm.all():
172
+ raise ValueError("factorizer='normal': the Jacobian has an empty column, the system is singular")
173
+ self.dc = 1 / colnorm
174
+ self.AsT = (As @ sps.diags(self.dc)).T.tocsr()
175
+ self.U = sps.triu(self.AsT @ self.AsT.T, format='csr') # PARDISO takes the upper triangle
176
+ pypardiso = _pypardiso()
177
+ self.info = f", J'J {2 * self.U.nnz - A.shape[0]:,} non-zeros"
178
+ try:
179
+ self.s = pypardiso.PyPardisoSolver(mtype=2) # real symmetric positive definite
180
+ self.s.factorize(self.U)
181
+ except pypardiso.pardiso_wrapper.PyPardisoError:
182
+ # J'J is positive definite in exact arithmetic only: with a badly conditioned J a pivot can come out
183
+ # as zero or negative. Then the symmetric indefinite factorization, which pivots.
184
+ try:
185
+ self.s.free_memory(everything=True) # what the failed attempt left in MKL
186
+ except Exception:
187
+ pass
188
+ self.s = pypardiso.PyPardisoSolver(mtype=-2)
189
+ self.s.factorize(self.U)
190
+ self.info += ', not numerically positive definite: symmetric indefinite factorization'
191
+
192
+ def _apply(self, r):
193
+ return self.dc * self.s.solve(self.U, self.AsT @ (self.dr * r))
194
+
195
+
196
+ class MumpsSolver(RefinedSolver):
197
+ """LU of J with MUMPS in this process (python-mumps)."""
198
+ name = 'mumps'
199
+ max_refine = 6
200
+
201
+ def _factorize(self):
202
+ import mumps
203
+ A = self.A.copy()
204
+ A.eliminate_zeros()
205
+ self.ctx = mumps.Context()
206
+ self.ctx.set_matrix(A.tocoo())
207
+ t0 = time.time()
208
+ try:
209
+ self.ctx.analyze(ordering='metis')
210
+ except Exception:
211
+ self.ctx.analyze()
212
+ t1 = time.time()
213
+ self.ctx.factor(reuse_analysis=True)
214
+ self.info = f' (analyze {t1 - t0:.1f} s, factorize {time.time() - t1:.1f} s)'
215
+
216
+ def _apply(self, r):
217
+ return self.ctx.solve(r)
218
+
219
+
220
+ class MumpsProcessSolver:
221
+ """LU of J with MUMPS in a helper process: this file run by another python, where python-mumps is installed.
222
+
223
+ The matrix, the right hand sides and the solutions are exchanged as files in a temporary folder."""
224
+ name = 'mumps'
225
+
226
+ def __init__(self, A, verbose=False, python=None):
227
+ import subprocess
228
+ import tempfile
229
+ from pathlib import Path
230
+ self.verbose = verbose
231
+ self.last = (np.nan, 0)
232
+ python = Path(python or os.environ['MODELFLOW_MUMPS_PYTHON'])
233
+ if not python.exists():
234
+ raise FileNotFoundError(f'MODELFLOW_MUMPS_PYTHON: {python} does not exist')
235
+ self._tmp = tempfile.TemporaryDirectory(prefix='modelflow_mumps_')
236
+ self.tmp = Path(self._tmp.name)
237
+ env = dict(os.environ)
238
+ if os.name == 'nt':
239
+ # the helper must find the DLLs of its own environment first, not those of the calling one
240
+ root = python.parent
241
+ env['PATH'] = os.pathsep.join([str(root), str(root / 'Library' / 'bin'), str(root / 'Scripts'),
242
+ r'C:\Windows\System32', r'C:\Windows'])
243
+ self.p = subprocess.Popen([str(python), '-u', os.path.abspath(__file__)],
244
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, env=env)
245
+ sps.save_npz(self.tmp / 'A.npz', sps.csr_matrix(A), compressed=False)
246
+ self.info = ' '.join(self._ask(f'factor {self.tmp / "A.npz"}')[1:]) + ', helper process'
247
+ if verbose:
248
+ print(self.info, flush=True)
249
+
250
+ def _ask(self, cmd):
251
+ self.p.stdin.write(cmd + '\n')
252
+ self.p.stdin.flush()
253
+ ans = self.p.stdout.readline().split()
254
+ if not ans or ans[0] != 'ok':
255
+ raise RuntimeError(f'MUMPS helper process: {" ".join(ans) or "stopped without an answer"}')
256
+ return ans
257
+
258
+ def solve(self, b):
259
+ if self.__dict__.get('released'):
260
+ raise RuntimeError("factorizer='mumps': this factorization has been released, factorize again")
261
+ np.save(self.tmp / 'b.npy', np.ascontiguousarray(b, dtype=float))
262
+ ans = self._ask(f'solve {self.tmp / "b.npy"} {self.tmp / "x.npy"}')
263
+ self.last = (float(ans[1]), int(ans[2]))
264
+ if not self.last[0] < WARN_TOL:
265
+ print(f"factorizer='mumps': the linear solve is not accurate, relative residual {ans[1]} "
266
+ f'after {ans[2]} refinements', flush=True)
267
+ elif self.verbose:
268
+ print(f' mumps solve: relative linear residual {ans[1]} after {ans[2]} refinements', flush=True)
269
+ return np.load(self.tmp / 'x.npy')
270
+
271
+ def release(self):
272
+ """Stop the helper process, which frees its factorization."""
273
+ if self.__dict__.get('released'):
274
+ return
275
+ self.released = True
276
+ try:
277
+ self.p.stdin.write('quit\n')
278
+ self.p.stdin.flush()
279
+ self.p.wait(timeout=20)
280
+ except Exception:
281
+ pass
282
+ try:
283
+ self._tmp.cleanup()
284
+ except Exception:
285
+ pass
286
+
287
+ def __del__(self):
288
+ self.release()
289
+
290
+
291
+ def _mumps(A, verbose=False):
292
+ try:
293
+ import mumps # noqa: F401
294
+ except ImportError:
295
+ if os.environ.get('MODELFLOW_MUMPS_PYTHON'):
296
+ return MumpsProcessSolver(A, verbose)
297
+ raise ImportError("factorizer='mumps' needs the package python-mumps in this environment, or the environment "
298
+ 'variable MODELFLOW_MUMPS_PYTHON set to the python of an environment that has it '
299
+ '(conda create -n mumpsenv -c conda-forge python numpy scipy python-mumps)') from None
300
+ return MumpsSolver(A, verbose)
301
+
302
+
303
+ FACTORIZERS = {'pardiso': PardisoSolver, 'normal': NormalSolver, 'mumps': _mumps}
304
+ BUILTIN = (None, 'superlu', 'umfpack') # handled in modelnewton
305
+
306
+
307
+ def make_solver(A, factorizer, verbose=False):
308
+ """Factorize A with the chosen factorizer and return the function solve(b) -> x."""
309
+ if callable(factorizer):
310
+ return factorizer(A)
311
+ try:
312
+ maker = FACTORIZERS[factorizer.lower()]
313
+ except (KeyError, AttributeError):
314
+ raise ValueError(f'factorizer={factorizer!r} is not known. Use one of None, '
315
+ f"'superlu', 'umfpack', {', '.join(repr(k) for k in FACTORIZERS)} or a function") from None
316
+ return maker(A, verbose).solve
317
+
318
+
319
+ def release(solve):
320
+ """Free the factorization behind a solve function from make_solver now, instead of when the garbage
321
+ collector gets to it. Other solve functions (SuperLU, a user function without ``release``) are left alone."""
322
+ owner = getattr(solve, '__self__', None)
323
+ if owner is not None and callable(getattr(owner, 'release', None)):
324
+ owner.release()
325
+
326
+
327
+ if __name__ == '__main__': # the MUMPS helper process
328
+ solver = None
329
+ for line in sys.stdin:
330
+ cmd = line.split()
331
+ if not cmd:
332
+ continue
333
+ try:
334
+ if cmd[0] == 'factor':
335
+ solver = MumpsSolver(sps.load_npz(cmd[1]))
336
+ print('ok', solver.info, flush=True)
337
+ elif cmd[0] == 'solve':
338
+ import contextlib
339
+ with contextlib.redirect_stdout(sys.stderr): # a warning must not disturb the protocol
340
+ x = solver.solve(np.load(cmd[1]))
341
+ np.save(cmd[2], x)
342
+ print('ok', f'{solver.last[0]:.3g}', solver.last[1], flush=True)
343
+ elif cmd[0] == 'quit':
344
+ break
345
+ except Exception as e:
346
+ print('error', repr(e)[:300].replace('\n', ' '), flush=True)