ModelFlowIb 2.92__tar.gz → 2.93__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modelflowib-2.92 → modelflowib-2.93}/PKG-INFO +1 -1
- {modelflowib-2.92 → modelflowib-2.93}/model_parquet_mixin.py +15 -4
- {modelflowib-2.92 → modelflowib-2.93}/modelcausality.py +4 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelclass.py +74 -25
- modelflowib-2.93/modelfactorize.py +346 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/PKG-INFO +1 -1
- {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/SOURCES.txt +2 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/top_level.txt +2 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelhelp.py +17 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelnewton.py +50 -1
- modelflowib-2.93/modelode.py +552 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelsolver_ng.py +99 -25
- {modelflowib-2.92 → modelflowib-2.93}/pyproject.toml +3 -3
- {modelflowib-2.92 → modelflowib-2.93}/README.md +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/license.md +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelBLfunk.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_Excel.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_cvx.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_dynare.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_financial_stability.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_latex.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/model_latex_class.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelconstruct.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelconstruct_estimation.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldash.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldashboot.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldashsidebar.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldekom.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldiff.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeldisplay.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelestimation.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelestimator_new.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/dependency_links.txt +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelflowib.egg-info/requires.txt +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelgrab.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelgrabexcel.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelgrabgdx.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelgrabgekkobank.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelgrabwf2.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelhtml.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelinput_shiny.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelinvert.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeljupyter.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeljupytermagic.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelmacrograb.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelmanipulation.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelmf.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelnet.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelnormalize.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelpattern.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelreport.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modeluserfunk.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelvis.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelwidget.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelwidget_core.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/modelwidget_input.py +0 -0
- {modelflowib-2.92 → modelflowib-2.93}/setup.cfg +0 -0
|
@@ -164,7 +164,7 @@ class Parquet_Mixin:
|
|
|
164
164
|
|
|
165
165
|
# ── dump ──────────────────────────────────────────────────────────────
|
|
166
166
|
|
|
167
|
-
def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
|
|
167
|
+
def modeldump(self, file_path='', keep=False, large=False, compact=False, derivatives=False, **kwargs):
|
|
168
168
|
"""Dump model to disk.
|
|
169
169
|
|
|
170
170
|
Parameters
|
|
@@ -181,6 +181,12 @@ class Parquet_Mixin:
|
|
|
181
181
|
If True (only effective when large=True), store float64
|
|
182
182
|
columns as float32 to roughly halve file size.
|
|
183
183
|
On load the data is restored to float64.
|
|
184
|
+
derivatives : bool
|
|
185
|
+
If True, also store the expressions for the derivatives that a Newton
|
|
186
|
+
solver has found (``model.diff_expressions``). A Newton solver in a later
|
|
187
|
+
session takes them from the model instead of finding them again, which is
|
|
188
|
+
the slow part of its first solve for a large model. They are used only if
|
|
189
|
+
the equations and the solver options are the same.
|
|
184
190
|
|
|
185
191
|
A model instance without a lastdf holds no dataframe, so the feather
|
|
186
192
|
format has nothing to be fast about. Such a model is always written as a
|
|
@@ -194,7 +200,7 @@ class Parquet_Mixin:
|
|
|
194
200
|
|
|
195
201
|
if not large or nodata:
|
|
196
202
|
# ── original JSON/gzip path ──────────────────────────────────
|
|
197
|
-
return super().modeldump(file_path=file_path, keep=keep, **kwargs)
|
|
203
|
+
return super().modeldump(file_path=file_path, keep=keep, derivatives=derivatives, **kwargs)
|
|
198
204
|
|
|
199
205
|
# ── fast feather/zip path ────────────────────────────────────────
|
|
200
206
|
pathname = Path(file_path)
|
|
@@ -216,8 +222,10 @@ class Parquet_Mixin:
|
|
|
216
222
|
'format': 'feather_zip',
|
|
217
223
|
'frml': self.equations,
|
|
218
224
|
'modelname': self.name,
|
|
219
|
-
'oldkwargs': self.oldkwargs,
|
|
225
|
+
'oldkwargs': self.__dict__.get('oldkwargs', {}), # a model never solved has none
|
|
220
226
|
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
227
|
+
'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
|
|
228
|
+
'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
|
|
221
229
|
'var_description': self.var_description,
|
|
222
230
|
'equations_latex': self.equations_latex,
|
|
223
231
|
'wb_MFMSAOPTIONS': (
|
|
@@ -383,8 +391,11 @@ class Parquet_Mixin:
|
|
|
383
391
|
# while threads work, build the model object (parses equations)
|
|
384
392
|
frml = meta['frml']
|
|
385
393
|
modelname = meta['modelname']
|
|
386
|
-
|
|
394
|
+
# a model dumped with straight=True is loaded like that, unless the call says otherwise
|
|
395
|
+
model_kwargs = {**({'straight': True} if meta.get('straight', False) else {}), **kwargs}
|
|
396
|
+
mmodel = cls(frml, modelname=modelname, funks=funks, **model_kwargs)
|
|
387
397
|
mmodel.oldkwargs = meta.get('oldkwargs', {})
|
|
398
|
+
mmodel.diff_expressions = meta.get('diff_expressions', {})
|
|
388
399
|
mmodel.run_on_load = meta.get('run_on_load', False)
|
|
389
400
|
if run is None:
|
|
390
401
|
run = mmodel.run_on_load
|
|
@@ -736,6 +736,10 @@ if AnyWidget is not None:
|
|
|
736
736
|
self.observe(self._refresh, names=self._settings)
|
|
737
737
|
self._refresh()
|
|
738
738
|
|
|
739
|
+
def __repr__(self):
|
|
740
|
+
# the text/plain output, shown where the widget is not run (a static book)
|
|
741
|
+
return f'Interactive causality viewer for {self.var}, run the notebook to use it'
|
|
742
|
+
|
|
739
743
|
def _refresh(self, change=None):
|
|
740
744
|
navn = self.var.upper()
|
|
741
745
|
if navn not in self.mmodel.allvar:
|
|
@@ -116,6 +116,7 @@ from modelhelp import cutout, update_var
|
|
|
116
116
|
from modelnormalize import normal
|
|
117
117
|
from modeldekom import totdif
|
|
118
118
|
from modelhelp import debug_var
|
|
119
|
+
from modelhelp import nonfinite_places
|
|
119
120
|
|
|
120
121
|
|
|
121
122
|
# functions used in BL language
|
|
@@ -1027,7 +1028,8 @@ class BaseModel():
|
|
|
1027
1028
|
convergence = True
|
|
1028
1029
|
for after, before in zip(itafter, itbefore):
|
|
1029
1030
|
# print(before,after)
|
|
1030
|
-
|
|
1031
|
+
# NaN fails before > absconv, so it has to be tested on its own
|
|
1032
|
+
if not np.all(np.isfinite(after)) or (before > absconv and abs(after-before)/abs(before) > relconv):
|
|
1031
1033
|
convergence = False
|
|
1032
1034
|
break
|
|
1033
1035
|
if convergence:
|
|
@@ -3174,7 +3176,7 @@ class Description_Mixin():
|
|
|
3174
3176
|
out = (ind.replace('&','&').replace('<', '<').replace('>', '>').replace('\n', ' ')
|
|
3175
3177
|
.replace('æ', 'æ').replace('ø', 'ø').replace('å', 'å')
|
|
3176
3178
|
.replace('Æ', 'Æ').replace('Ø', 'Ø').replace('Å', 'Å')
|
|
3177
|
-
.replace('$','$')
|
|
3179
|
+
.replace('$','$').replace('"', '"')
|
|
3178
3180
|
)
|
|
3179
3181
|
return out
|
|
3180
3182
|
|
|
@@ -6863,11 +6865,15 @@ class Json_Mixin():
|
|
|
6863
6865
|
|
|
6864
6866
|
return pathname, dump_data
|
|
6865
6867
|
|
|
6866
|
-
def modeldump_base(self, outfile='',keep=False,data=True):
|
|
6868
|
+
def modeldump_base(self, outfile='',keep=False,data=True,derivatives=False):
|
|
6867
6869
|
'''Dumps a model and its lastdf to a json file
|
|
6868
6870
|
|
|
6869
6871
|
if keep=True the model.keep_solutions will alse be dumped
|
|
6870
6872
|
|
|
6873
|
+
if derivatives=True the expressions for the derivatives, which a Newton solver has found
|
|
6874
|
+
(model.diff_expressions), are dumped too. A Newton solver in a later session then takes them
|
|
6875
|
+
from the model instead of finding them again - the slow part of its first solve for a large model
|
|
6876
|
+
|
|
6871
6877
|
if data=False, or the model instance has no lastdf, only the model
|
|
6872
6878
|
structure is dumped and the file has no dataframes'''
|
|
6873
6879
|
# print('dump ready')
|
|
@@ -6928,6 +6934,8 @@ class Json_Mixin():
|
|
|
6928
6934
|
'modelname': self.name,
|
|
6929
6935
|
'oldkwargs': self.__dict__.get('oldkwargs',{}),
|
|
6930
6936
|
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
6937
|
+
'straight': bool(self.__dict__.get('straight', False)), # loads the same way again
|
|
6938
|
+
'diff_expressions': self.__dict__.get('diff_expressions', {}) if derivatives else {},
|
|
6931
6939
|
'var_description': self.var_description,
|
|
6932
6940
|
'equations_latex': self.equations_latex ,
|
|
6933
6941
|
'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
|
|
@@ -7065,8 +7073,11 @@ class Json_Mixin():
|
|
|
7065
7073
|
current_per = None
|
|
7066
7074
|
modelname = input['modelname']
|
|
7067
7075
|
# breakpoint()
|
|
7068
|
-
|
|
7076
|
+
# a model dumped with straight=True is loaded like that, unless the call says otherwise
|
|
7077
|
+
model_kwargs = {**({'straight': True} if input.get('straight', False) else {}), **kwargs}
|
|
7078
|
+
mmodel = cls(frml, modelname=modelname, funks=funks,**model_kwargs)
|
|
7069
7079
|
mmodel.oldkwargs = input.get('oldkwargs',{})
|
|
7080
|
+
mmodel.diff_expressions = input.get('diff_expressions', {})
|
|
7070
7081
|
mmodel.run_on_load = input.get('run_on_load', False)
|
|
7071
7082
|
if run is None:
|
|
7072
7083
|
run = mmodel.run_on_load and has_data
|
|
@@ -7123,12 +7134,12 @@ class Json_Mixin():
|
|
|
7123
7134
|
|
|
7124
7135
|
class Zip_Mixin():
|
|
7125
7136
|
'''This experimental class zips a dumped file '''
|
|
7126
|
-
def modeldump(self, file_path='',keep=False,zip_file=True,data=True):
|
|
7137
|
+
def modeldump(self, file_path='',keep=False,zip_file=True,data=True,derivatives=False):
|
|
7127
7138
|
pathname, dump_data = self._dump_pathname(file_path, self.has_data, data=data)
|
|
7128
7139
|
|
|
7129
7140
|
pathname.parent.mkdir(parents=True, exist_ok=True)
|
|
7130
7141
|
|
|
7131
|
-
json_string = self.modeldump_base(keep=keep,data=dump_data)
|
|
7142
|
+
json_string = self.modeldump_base(keep=keep,data=dump_data,derivatives=derivatives)
|
|
7132
7143
|
|
|
7133
7144
|
|
|
7134
7145
|
if zip_file:
|
|
@@ -7640,7 +7651,8 @@ class Solver_Mixin():
|
|
|
7640
7651
|
# this is how convergence is measured
|
|
7641
7652
|
convplace = [databank.columns.get_loc(c) for c in convvar]
|
|
7642
7653
|
convergence = True
|
|
7643
|
-
|
|
7654
|
+
endovar = list(self.endogene)
|
|
7655
|
+
endoplace = [databank.columns.get_loc(c) for c in endovar]
|
|
7644
7656
|
# breakpoint()
|
|
7645
7657
|
if ldumpvar:
|
|
7646
7658
|
self.dumplist = []
|
|
@@ -7668,6 +7680,7 @@ class Solver_Mixin():
|
|
|
7668
7680
|
values[row, c] = values[row-1, c]
|
|
7669
7681
|
itbefore = values[row, convplace]
|
|
7670
7682
|
self.pro2d(values, values, row, 1.0)
|
|
7683
|
+
corenan = False
|
|
7671
7684
|
for iteration in range(max_iterations):
|
|
7672
7685
|
with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
|
|
7673
7686
|
self.solve2d(values, values, row, alfa)
|
|
@@ -7679,18 +7692,30 @@ class Solver_Mixin():
|
|
|
7679
7692
|
if iteration > first_test:
|
|
7680
7693
|
itafter = values[row, convplace]
|
|
7681
7694
|
select = absconv <= np.abs(itbefore)
|
|
7695
|
+
# NaN drops out of select, so it has to be tested on its own
|
|
7696
|
+
nonfinite = nonfinite_places(itafter)
|
|
7682
7697
|
convergence = (
|
|
7683
|
-
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
|
|
7698
|
+
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
|
|
7684
7699
|
if convergence:
|
|
7685
7700
|
if not silent:
|
|
7686
7701
|
print(
|
|
7687
7702
|
f'{self.periode} Solved in {iteration} iterations')
|
|
7688
7703
|
break
|
|
7704
|
+
if len(nonfinite):
|
|
7705
|
+
corenan = True
|
|
7706
|
+
bad = [convvar[i] for i in nonfinite]
|
|
7707
|
+
print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
7708
|
+
break
|
|
7689
7709
|
itbefore = itafter
|
|
7690
7710
|
else:
|
|
7691
7711
|
print(f'{self.periode} not converged in {iteration} iterations')
|
|
7692
|
-
|
|
7712
|
+
|
|
7693
7713
|
self.epi2d(values, values, row, 1.0)
|
|
7714
|
+
# the convergence test only sees the core, prolog/epilog can also be NaN
|
|
7715
|
+
nonfinite = nonfinite_places(values[row, endoplace])
|
|
7716
|
+
if len(nonfinite) and not corenan:
|
|
7717
|
+
bad = [endovar[i] for i in nonfinite]
|
|
7718
|
+
print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
7694
7719
|
pbar.update()
|
|
7695
7720
|
endtime = time.time()
|
|
7696
7721
|
|
|
@@ -7975,7 +8000,8 @@ class Solver_Mixin():
|
|
|
7975
8000
|
# convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
7976
8001
|
convplace = [self.allvar[c]['startnr'] -
|
|
7977
8002
|
self.allvar[c]['maxlead'] for c in convvar]
|
|
7978
|
-
|
|
8003
|
+
endovar = list(self.endogene)
|
|
8004
|
+
endoplace = [databank.columns.get_loc(c) for c in endovar]
|
|
7979
8005
|
|
|
7980
8006
|
convergence = True
|
|
7981
8007
|
|
|
@@ -8010,6 +8036,7 @@ class Solver_Mixin():
|
|
|
8010
8036
|
itbeforeold = [a[c] for c in convplace]
|
|
8011
8037
|
itbefore = a[convplace]
|
|
8012
8038
|
this_pro1d(a, 1.0)
|
|
8039
|
+
corenan = False
|
|
8013
8040
|
for iteration in range(max_iterations):
|
|
8014
8041
|
with self.timer(f'Evaluate {self.periode}/{iteration} ', timeon) as t:
|
|
8015
8042
|
this_solve1d(a, alfa)
|
|
@@ -8022,20 +8049,32 @@ class Solver_Mixin():
|
|
|
8022
8049
|
itafter = a[convplace]
|
|
8023
8050
|
# breakpoint()
|
|
8024
8051
|
select = absconv <= np.abs(itbefore)
|
|
8052
|
+
# NaN drops out of select, so it has to be tested on its own
|
|
8053
|
+
nonfinite = nonfinite_places(itafter)
|
|
8025
8054
|
convergence = (
|
|
8026
|
-
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all()
|
|
8055
|
+
np.abs((itafter-itbefore)[select])/np.abs(itbefore[select]) <= relconv).all() and not len(nonfinite)
|
|
8027
8056
|
if convergence:
|
|
8028
8057
|
if not silent:
|
|
8029
8058
|
print(
|
|
8030
8059
|
f'{self.periode} Solved in {iteration} iterations')
|
|
8031
8060
|
break
|
|
8061
|
+
if len(nonfinite):
|
|
8062
|
+
corenan = True
|
|
8063
|
+
bad = [convvar[i] for i in nonfinite]
|
|
8064
|
+
print(f"{self.periode} not converged, NaN or inf in {len(bad)} variables after {iteration} iterations: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
8065
|
+
break
|
|
8032
8066
|
itbefore = itafter
|
|
8033
8067
|
else:
|
|
8034
8068
|
print(f'{self.periode} not converged in {iteration} iterations')
|
|
8035
|
-
|
|
8069
|
+
|
|
8036
8070
|
this_epi1d(a, 1.0)
|
|
8037
8071
|
|
|
8038
8072
|
self.saveeval3(values, row, a)
|
|
8073
|
+
# the convergence test only sees the core, prolog/epilog can also be NaN
|
|
8074
|
+
nonfinite = nonfinite_places(values[row, endoplace])
|
|
8075
|
+
if len(nonfinite) and not corenan:
|
|
8076
|
+
bad = [endovar[i] for i in nonfinite]
|
|
8077
|
+
print(f"{self.periode} NaN or inf in {len(bad)} endogenous variables: {' '.join(bad[:10])}{' ...' if len(bad) > 10 else ''}")
|
|
8039
8078
|
|
|
8040
8079
|
if ldumpvar:
|
|
8041
8080
|
self.dumpdf = pd.DataFrame(self.dumplist)
|
|
@@ -9617,9 +9656,23 @@ class Solver_Mixin():
|
|
|
9617
9656
|
|
|
9618
9657
|
def errfunk(self, values, linenr, overhead=4, overeq=0):
|
|
9619
9658
|
'''Development function to handle run time errors in model calculations'''
|
|
9620
|
-
|
|
9659
|
+
|
|
9621
9660
|
import pandas as pd
|
|
9622
|
-
|
|
9661
|
+
|
|
9662
|
+
if getattr(self, 'errfunk_defer', False):
|
|
9663
|
+
# a Newton trial step: keep the error point, the solver dumps it only if it gives up
|
|
9664
|
+
self.errfunk_deferred = (values.copy(), linenr, overhead, overeq)
|
|
9665
|
+
return
|
|
9666
|
+
|
|
9667
|
+
if hasattr(self, 'dumplist'):
|
|
9668
|
+
# first, so the returns below keep it, and it must never mask the real error
|
|
9669
|
+
dumplist = self.dumplist
|
|
9670
|
+
del self.dumplist
|
|
9671
|
+
try:
|
|
9672
|
+
self.dumpdf = pd.DataFrame(dumplist, columns=['fair', 'per', 'iteration'] + list(self.dump))
|
|
9673
|
+
except Exception as e:
|
|
9674
|
+
print('>> Could not build .dumpdf:', e)
|
|
9675
|
+
|
|
9623
9676
|
def find_log_arguments(text):
|
|
9624
9677
|
"""Return all arguments inside balanced LOG(...) calls."""
|
|
9625
9678
|
out = []
|
|
@@ -9666,7 +9719,8 @@ class Solver_Mixin():
|
|
|
9666
9719
|
outeq = self.allvar[errvar]['frml']
|
|
9667
9720
|
|
|
9668
9721
|
print('>>:', outeq,)
|
|
9669
|
-
|
|
9722
|
+
per = getattr(self, 'periode', None) # not set if the error comes before the period loop
|
|
9723
|
+
print('In:', per if per is not None else 'unknown period', '\n')
|
|
9670
9724
|
print('Below you will find information which might help you identify the problem')
|
|
9671
9725
|
terms = self.allvar[errvar]['terms']
|
|
9672
9726
|
|
|
@@ -9679,7 +9733,7 @@ class Solver_Mixin():
|
|
|
9679
9733
|
|
|
9680
9734
|
|
|
9681
9735
|
try:
|
|
9682
|
-
basepos = self.errdump.index.get_loc(
|
|
9736
|
+
basepos = self.errdump.index.get_loc(per)
|
|
9683
9737
|
rows = []
|
|
9684
9738
|
seen = set()
|
|
9685
9739
|
|
|
@@ -9718,8 +9772,8 @@ class Solver_Mixin():
|
|
|
9718
9772
|
for arg in log_args:
|
|
9719
9773
|
try:
|
|
9720
9774
|
val = self.errdump.mfcalc(
|
|
9721
|
-
f'<{
|
|
9722
|
-
).loc[
|
|
9775
|
+
f'<{per}> {tmpvar} = {arg}'
|
|
9776
|
+
).loc[per, tmpvar]
|
|
9723
9777
|
status = 'Bad log argument' if pd.isna(val) or val <= 0 else 'ok'
|
|
9724
9778
|
except Exception:
|
|
9725
9779
|
val = ''
|
|
@@ -9735,13 +9789,8 @@ class Solver_Mixin():
|
|
|
9735
9789
|
|
|
9736
9790
|
print('\nA snapshot of the data at the error point is at .errdump')
|
|
9737
9791
|
print('Also .lastdf contains a copy of .errdump for inspecting')
|
|
9738
|
-
|
|
9739
|
-
|
|
9740
|
-
self.dumpdf = pd.DataFrame(self.dumplist)
|
|
9741
|
-
del self.dumplist
|
|
9742
|
-
self.dumpdf.columns = ['fair', 'per', 'iteration'] + self.dump
|
|
9743
|
-
|
|
9744
|
-
|
|
9792
|
+
|
|
9793
|
+
|
|
9745
9794
|
def show_iterations(self, pat='*', per='', last=0, change=False,top=.9):
|
|
9746
9795
|
'''
|
|
9747
9796
|
shows the last iterations for the most recent simulation.
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
modelfactorize.py - the linear solvers behind the ``factorizer=`` option of the Newton solvers.
|
|
4
|
+
|
|
5
|
+
A Newton step solves ``J x = b`` with the (sparse) Jacobian J. ``make_solver(J, factorizer)`` factorizes J and
|
|
6
|
+
returns a function ``solve(b) -> x``. The choices:
|
|
7
|
+
|
|
8
|
+
=========== ============================================================================= ====================
|
|
9
|
+
factorizer method needs
|
|
10
|
+
=========== ============================================================================= ====================
|
|
11
|
+
None SuperLU through scipy (handled in modelnewton, with ``permc_spec``) -
|
|
12
|
+
'superlu' the same
|
|
13
|
+
'umfpack' UMFPACK through cvxopt (handled in modelnewton) cvxopt
|
|
14
|
+
'pardiso' LU with MKL PARDISO. Fast and multi-threaded, but it pivots statically, so pypardiso
|
|
15
|
+
a Jacobian with many zeros on the diagonal can come out inaccurate. The
|
|
16
|
+
solve is refined against J and warns when the refinement does not get there.
|
|
17
|
+
'normal' The step as a least squares problem, min |J x - b|^2: the normal equations pypardiso
|
|
18
|
+
J'J x = J'b (the Gauss-Newton step on the sum of squared equation errors).
|
|
19
|
+
J'J is symmetric positive definite, so PARDISO factorizes it (Cholesky)
|
|
20
|
+
without pivoting. J is scaled first, and the solve is refined against J
|
|
21
|
+
itself, which brings back the accuracy the squared condition number costs.
|
|
22
|
+
The fastest choice for a large stacked system.
|
|
23
|
+
'mumps' LU with MUMPS: real pivoting and METIS ordering. Used in this process when python-mumps
|
|
24
|
+
``import mumps`` works. Otherwise in a helper process, started with the
|
|
25
|
+
python named by the environment variable MODELFLOW_MUMPS_PYTHON (the
|
|
26
|
+
python.exe of an environment where python-mumps is installed).
|
|
27
|
+
a callable ``factorizer(J)`` must return the function ``solve(b) -> x`` -
|
|
28
|
+
=========== ============================================================================= ====================
|
|
29
|
+
|
|
30
|
+
The solve functions are bound methods; the object behind (``solve.__self__``) has ``info`` (a text on the
|
|
31
|
+
factorization) and ``last`` (relative linear residual and number of refinements of the latest solve).
|
|
32
|
+
|
|
33
|
+
Run as a script this file is the MUMPS helper process (one command per line on stdin, one answer line on stdout)::
|
|
34
|
+
|
|
35
|
+
factor <A.npz> -> ok <the info text>
|
|
36
|
+
solve <b.npy> <x.npy> -> ok <relative residual> <refinements>
|
|
37
|
+
quit
|
|
38
|
+
"""
|
|
39
|
+
import os
|
|
40
|
+
import sys
|
|
41
|
+
import time
|
|
42
|
+
|
|
43
|
+
import numpy as np
|
|
44
|
+
import scipy.sparse as sps
|
|
45
|
+
|
|
46
|
+
REFINE_TOL = 1e-11 # relative linear residual (max norm) the refinement aims for
|
|
47
|
+
WARN_TOL = 1e-8 # a warning when the refinement ends above this
|
|
48
|
+
FAIL_TOL = 1e-2 # an exception when it ends above this: the step would be wrong
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class RefinedSolver:
|
|
52
|
+
"""A factorization of J, or of something close to J, with iterative refinement against J itself.
|
|
53
|
+
|
|
54
|
+
Subclasses set ``self.A`` (J in csr form) and ``self.info`` and implement ``_apply(r)``: the approximate
|
|
55
|
+
solution of ``J x = r``."""
|
|
56
|
+
|
|
57
|
+
name = ''
|
|
58
|
+
max_refine = 20
|
|
59
|
+
|
|
60
|
+
def __init__(self, A, verbose=False):
|
|
61
|
+
self.verbose = verbose
|
|
62
|
+
self.last = (np.nan, 0)
|
|
63
|
+
self.info = ''
|
|
64
|
+
t0 = time.time()
|
|
65
|
+
self.A = sps.csr_matrix(A)
|
|
66
|
+
self._factorize()
|
|
67
|
+
self.info = (f'{self.name}: {self.A.shape[0]:,} unknowns, {self.A.nnz:,} non-zeros, '
|
|
68
|
+
f'{time.time() - t0:.1f} s{self.info}')
|
|
69
|
+
if verbose:
|
|
70
|
+
print(self.info, flush=True)
|
|
71
|
+
|
|
72
|
+
def _factorize(self):
|
|
73
|
+
raise NotImplementedError
|
|
74
|
+
|
|
75
|
+
def _apply(self, r):
|
|
76
|
+
raise NotImplementedError
|
|
77
|
+
|
|
78
|
+
def _release(self):
|
|
79
|
+
"""Free memory the garbage collector does not see (PARDISO keeps its factors inside MKL)."""
|
|
80
|
+
|
|
81
|
+
def release(self):
|
|
82
|
+
"""Free the factorization now. The solve function can not be used afterwards."""
|
|
83
|
+
if self.__dict__.get('released'):
|
|
84
|
+
return
|
|
85
|
+
self.released = True
|
|
86
|
+
try:
|
|
87
|
+
self._release()
|
|
88
|
+
finally:
|
|
89
|
+
for attr in ('A', 'AsT', 'U', 'dr', 'dc', 's', 'ctx'):
|
|
90
|
+
self.__dict__.pop(attr, None)
|
|
91
|
+
|
|
92
|
+
def __del__(self):
|
|
93
|
+
try:
|
|
94
|
+
self.release()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
def solve(self, b):
|
|
99
|
+
if self.__dict__.get('released'):
|
|
100
|
+
raise RuntimeError(f"factorizer='{self.name}': this factorization has been released, factorize again")
|
|
101
|
+
b = np.ascontiguousarray(b, dtype=float)
|
|
102
|
+
nb = max(np.abs(b).max(), 1e-300)
|
|
103
|
+
x = self._apply(b)
|
|
104
|
+
rel = np.inf
|
|
105
|
+
for k in range(self.max_refine + 1):
|
|
106
|
+
r = b - self.A @ x
|
|
107
|
+
rel = np.abs(r).max() / nb
|
|
108
|
+
if not np.isfinite(rel) or rel < REFINE_TOL or k == self.max_refine:
|
|
109
|
+
break
|
|
110
|
+
x = x + self._apply(r)
|
|
111
|
+
self.last = (rel, k)
|
|
112
|
+
if not rel < FAIL_TOL:
|
|
113
|
+
raise ArithmeticError(f"factorizer='{self.name}' can not solve this linear system: relative residual "
|
|
114
|
+
f"{rel:.2g} after {k} refinements. Use another factorizer")
|
|
115
|
+
if not rel < WARN_TOL:
|
|
116
|
+
print(f"factorizer='{self.name}': the linear solve is not accurate, relative residual {rel:.2g} "
|
|
117
|
+
f'after {k} refinements', flush=True)
|
|
118
|
+
elif self.verbose:
|
|
119
|
+
print(f' {self.name} solve: relative linear residual {rel:.2g} after {k} refinements', flush=True)
|
|
120
|
+
return x
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _pypardiso():
|
|
124
|
+
try:
|
|
125
|
+
import pypardiso
|
|
126
|
+
except ImportError as e:
|
|
127
|
+
raise ImportError("factorizer='pardiso' and 'normal' need the package pypardiso "
|
|
128
|
+
'(conda install -c conda-forge pypardiso)') from e
|
|
129
|
+
return pypardiso
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _PardisoRelease:
|
|
133
|
+
"""PARDISO keeps the factors in MKL's own memory, which is only given back by a call with phase -1.
|
|
134
|
+
pypardiso does not make that call when its solver object is deleted, so without this every
|
|
135
|
+
factorization stays in memory until the process ends."""
|
|
136
|
+
|
|
137
|
+
def _release(self):
|
|
138
|
+
s = self.__dict__.get('s')
|
|
139
|
+
if s is not None:
|
|
140
|
+
s.free_memory(everything=True)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class PardisoSolver(_PardisoRelease, RefinedSolver):
|
|
144
|
+
"""LU of J with MKL PARDISO (static pivoting)."""
|
|
145
|
+
name = 'pardiso'
|
|
146
|
+
|
|
147
|
+
def _factorize(self):
|
|
148
|
+
self.s = _pypardiso().PyPardisoSolver()
|
|
149
|
+
self.s.factorize(self.A)
|
|
150
|
+
|
|
151
|
+
def _apply(self, r):
|
|
152
|
+
return self.s.solve(self.A, r)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class NormalSolver(_PardisoRelease, RefinedSolver):
|
|
156
|
+
"""The normal equations J'J x = J'b with a PARDISO Cholesky factorization.
|
|
157
|
+
|
|
158
|
+
J is scaled to As = Dr J Dc (rows to largest element 1, then columns to norm 1), which decides the conditioning
|
|
159
|
+
of As'As. With x = Dc y the approximate solution of J x = r is Dc (As'As)^-1 As' Dr r."""
|
|
160
|
+
name = 'normal'
|
|
161
|
+
max_refine = 40
|
|
162
|
+
|
|
163
|
+
def _factorize(self):
|
|
164
|
+
A = self.A
|
|
165
|
+
rowmax = np.asarray(abs(A).max(axis=1).todense()).ravel()
|
|
166
|
+
if not rowmax.all():
|
|
167
|
+
raise ValueError("factorizer='normal': the Jacobian has an empty row, the system is singular")
|
|
168
|
+
self.dr = 1 / rowmax
|
|
169
|
+
As = sps.diags(self.dr) @ A
|
|
170
|
+
colnorm = np.sqrt(np.asarray(As.multiply(As).sum(axis=0)).ravel())
|
|
171
|
+
if not colnorm.all():
|
|
172
|
+
raise ValueError("factorizer='normal': the Jacobian has an empty column, the system is singular")
|
|
173
|
+
self.dc = 1 / colnorm
|
|
174
|
+
self.AsT = (As @ sps.diags(self.dc)).T.tocsr()
|
|
175
|
+
self.U = sps.triu(self.AsT @ self.AsT.T, format='csr') # PARDISO takes the upper triangle
|
|
176
|
+
pypardiso = _pypardiso()
|
|
177
|
+
self.info = f", J'J {2 * self.U.nnz - A.shape[0]:,} non-zeros"
|
|
178
|
+
try:
|
|
179
|
+
self.s = pypardiso.PyPardisoSolver(mtype=2) # real symmetric positive definite
|
|
180
|
+
self.s.factorize(self.U)
|
|
181
|
+
except pypardiso.pardiso_wrapper.PyPardisoError:
|
|
182
|
+
# J'J is positive definite in exact arithmetic only: with a badly conditioned J a pivot can come out
|
|
183
|
+
# as zero or negative. Then the symmetric indefinite factorization, which pivots.
|
|
184
|
+
try:
|
|
185
|
+
self.s.free_memory(everything=True) # what the failed attempt left in MKL
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
self.s = pypardiso.PyPardisoSolver(mtype=-2)
|
|
189
|
+
self.s.factorize(self.U)
|
|
190
|
+
self.info += ', not numerically positive definite: symmetric indefinite factorization'
|
|
191
|
+
|
|
192
|
+
def _apply(self, r):
|
|
193
|
+
return self.dc * self.s.solve(self.U, self.AsT @ (self.dr * r))
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class MumpsSolver(RefinedSolver):
|
|
197
|
+
"""LU of J with MUMPS in this process (python-mumps)."""
|
|
198
|
+
name = 'mumps'
|
|
199
|
+
max_refine = 6
|
|
200
|
+
|
|
201
|
+
def _factorize(self):
|
|
202
|
+
import mumps
|
|
203
|
+
A = self.A.copy()
|
|
204
|
+
A.eliminate_zeros()
|
|
205
|
+
self.ctx = mumps.Context()
|
|
206
|
+
self.ctx.set_matrix(A.tocoo())
|
|
207
|
+
t0 = time.time()
|
|
208
|
+
try:
|
|
209
|
+
self.ctx.analyze(ordering='metis')
|
|
210
|
+
except Exception:
|
|
211
|
+
self.ctx.analyze()
|
|
212
|
+
t1 = time.time()
|
|
213
|
+
self.ctx.factor(reuse_analysis=True)
|
|
214
|
+
self.info = f' (analyze {t1 - t0:.1f} s, factorize {time.time() - t1:.1f} s)'
|
|
215
|
+
|
|
216
|
+
def _apply(self, r):
|
|
217
|
+
return self.ctx.solve(r)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class MumpsProcessSolver:
|
|
221
|
+
"""LU of J with MUMPS in a helper process: this file run by another python, where python-mumps is installed.
|
|
222
|
+
|
|
223
|
+
The matrix, the right hand sides and the solutions are exchanged as files in a temporary folder."""
|
|
224
|
+
name = 'mumps'
|
|
225
|
+
|
|
226
|
+
def __init__(self, A, verbose=False, python=None):
|
|
227
|
+
import subprocess
|
|
228
|
+
import tempfile
|
|
229
|
+
from pathlib import Path
|
|
230
|
+
self.verbose = verbose
|
|
231
|
+
self.last = (np.nan, 0)
|
|
232
|
+
python = Path(python or os.environ['MODELFLOW_MUMPS_PYTHON'])
|
|
233
|
+
if not python.exists():
|
|
234
|
+
raise FileNotFoundError(f'MODELFLOW_MUMPS_PYTHON: {python} does not exist')
|
|
235
|
+
self._tmp = tempfile.TemporaryDirectory(prefix='modelflow_mumps_')
|
|
236
|
+
self.tmp = Path(self._tmp.name)
|
|
237
|
+
env = dict(os.environ)
|
|
238
|
+
if os.name == 'nt':
|
|
239
|
+
# the helper must find the DLLs of its own environment first, not those of the calling one
|
|
240
|
+
root = python.parent
|
|
241
|
+
env['PATH'] = os.pathsep.join([str(root), str(root / 'Library' / 'bin'), str(root / 'Scripts'),
|
|
242
|
+
r'C:\Windows\System32', r'C:\Windows'])
|
|
243
|
+
self.p = subprocess.Popen([str(python), '-u', os.path.abspath(__file__)],
|
|
244
|
+
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, env=env)
|
|
245
|
+
sps.save_npz(self.tmp / 'A.npz', sps.csr_matrix(A), compressed=False)
|
|
246
|
+
self.info = ' '.join(self._ask(f'factor {self.tmp / "A.npz"}')[1:]) + ', helper process'
|
|
247
|
+
if verbose:
|
|
248
|
+
print(self.info, flush=True)
|
|
249
|
+
|
|
250
|
+
def _ask(self, cmd):
|
|
251
|
+
self.p.stdin.write(cmd + '\n')
|
|
252
|
+
self.p.stdin.flush()
|
|
253
|
+
ans = self.p.stdout.readline().split()
|
|
254
|
+
if not ans or ans[0] != 'ok':
|
|
255
|
+
raise RuntimeError(f'MUMPS helper process: {" ".join(ans) or "stopped without an answer"}')
|
|
256
|
+
return ans
|
|
257
|
+
|
|
258
|
+
def solve(self, b):
|
|
259
|
+
if self.__dict__.get('released'):
|
|
260
|
+
raise RuntimeError("factorizer='mumps': this factorization has been released, factorize again")
|
|
261
|
+
np.save(self.tmp / 'b.npy', np.ascontiguousarray(b, dtype=float))
|
|
262
|
+
ans = self._ask(f'solve {self.tmp / "b.npy"} {self.tmp / "x.npy"}')
|
|
263
|
+
self.last = (float(ans[1]), int(ans[2]))
|
|
264
|
+
if not self.last[0] < WARN_TOL:
|
|
265
|
+
print(f"factorizer='mumps': the linear solve is not accurate, relative residual {ans[1]} "
|
|
266
|
+
f'after {ans[2]} refinements', flush=True)
|
|
267
|
+
elif self.verbose:
|
|
268
|
+
print(f' mumps solve: relative linear residual {ans[1]} after {ans[2]} refinements', flush=True)
|
|
269
|
+
return np.load(self.tmp / 'x.npy')
|
|
270
|
+
|
|
271
|
+
def release(self):
|
|
272
|
+
"""Stop the helper process, which frees its factorization."""
|
|
273
|
+
if self.__dict__.get('released'):
|
|
274
|
+
return
|
|
275
|
+
self.released = True
|
|
276
|
+
try:
|
|
277
|
+
self.p.stdin.write('quit\n')
|
|
278
|
+
self.p.stdin.flush()
|
|
279
|
+
self.p.wait(timeout=20)
|
|
280
|
+
except Exception:
|
|
281
|
+
pass
|
|
282
|
+
try:
|
|
283
|
+
self._tmp.cleanup()
|
|
284
|
+
except Exception:
|
|
285
|
+
pass
|
|
286
|
+
|
|
287
|
+
def __del__(self):
|
|
288
|
+
self.release()
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _mumps(A, verbose=False):
|
|
292
|
+
try:
|
|
293
|
+
import mumps # noqa: F401
|
|
294
|
+
except ImportError:
|
|
295
|
+
if os.environ.get('MODELFLOW_MUMPS_PYTHON'):
|
|
296
|
+
return MumpsProcessSolver(A, verbose)
|
|
297
|
+
raise ImportError("factorizer='mumps' needs the package python-mumps in this environment, or the environment "
|
|
298
|
+
'variable MODELFLOW_MUMPS_PYTHON set to the python of an environment that has it '
|
|
299
|
+
'(conda create -n mumpsenv -c conda-forge python numpy scipy python-mumps)') from None
|
|
300
|
+
return MumpsSolver(A, verbose)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
FACTORIZERS = {'pardiso': PardisoSolver, 'normal': NormalSolver, 'mumps': _mumps}
|
|
304
|
+
BUILTIN = (None, 'superlu', 'umfpack') # handled in modelnewton
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def make_solver(A, factorizer, verbose=False):
|
|
308
|
+
"""Factorize A with the chosen factorizer and return the function solve(b) -> x."""
|
|
309
|
+
if callable(factorizer):
|
|
310
|
+
return factorizer(A)
|
|
311
|
+
try:
|
|
312
|
+
maker = FACTORIZERS[factorizer.lower()]
|
|
313
|
+
except (KeyError, AttributeError):
|
|
314
|
+
raise ValueError(f'factorizer={factorizer!r} is not known. Use one of None, '
|
|
315
|
+
f"'superlu', 'umfpack', {', '.join(repr(k) for k in FACTORIZERS)} or a function") from None
|
|
316
|
+
return maker(A, verbose).solve
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def release(solve):
|
|
320
|
+
"""Free the factorization behind a solve function from make_solver now, instead of when the garbage
|
|
321
|
+
collector gets to it. Other solve functions (SuperLU, a user function without ``release``) are left alone."""
|
|
322
|
+
owner = getattr(solve, '__self__', None)
|
|
323
|
+
if owner is not None and callable(getattr(owner, 'release', None)):
|
|
324
|
+
owner.release()
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
if __name__ == '__main__': # the MUMPS helper process
|
|
328
|
+
solver = None
|
|
329
|
+
for line in sys.stdin:
|
|
330
|
+
cmd = line.split()
|
|
331
|
+
if not cmd:
|
|
332
|
+
continue
|
|
333
|
+
try:
|
|
334
|
+
if cmd[0] == 'factor':
|
|
335
|
+
solver = MumpsSolver(sps.load_npz(cmd[1]))
|
|
336
|
+
print('ok', solver.info, flush=True)
|
|
337
|
+
elif cmd[0] == 'solve':
|
|
338
|
+
import contextlib
|
|
339
|
+
with contextlib.redirect_stdout(sys.stderr): # a warning must not disturb the protocol
|
|
340
|
+
x = solver.solve(np.load(cmd[1]))
|
|
341
|
+
np.save(cmd[2], x)
|
|
342
|
+
print('ok', f'{solver.last[0]:.3g}', solver.last[1], flush=True)
|
|
343
|
+
elif cmd[0] == 'quit':
|
|
344
|
+
break
|
|
345
|
+
except Exception as e:
|
|
346
|
+
print('error', repr(e)[:300].replace('\n', ' '), flush=True)
|