ModelFlowIb 2.93__tar.gz → 2.94__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {modelflowib-2.93 → modelflowib-2.94}/PKG-INFO +1 -1
  2. {modelflowib-2.93 → modelflowib-2.94}/modelconstruct_estimation.py +54 -1
  3. {modelflowib-2.93 → modelflowib-2.94}/modelestimator_new.py +95 -7
  4. {modelflowib-2.93 → modelflowib-2.94}/modelflowib.egg-info/PKG-INFO +1 -1
  5. {modelflowib-2.93 → modelflowib-2.94}/modelnewton.py +168 -8
  6. {modelflowib-2.93 → modelflowib-2.94}/modelsolver_ng.py +15 -8
  7. {modelflowib-2.93 → modelflowib-2.94}/pyproject.toml +1 -1
  8. {modelflowib-2.93 → modelflowib-2.94}/README.md +0 -0
  9. {modelflowib-2.93 → modelflowib-2.94}/license.md +0 -0
  10. {modelflowib-2.93 → modelflowib-2.94}/modelBLfunk.py +0 -0
  11. {modelflowib-2.93 → modelflowib-2.94}/model_Excel.py +0 -0
  12. {modelflowib-2.93 → modelflowib-2.94}/model_cvx.py +0 -0
  13. {modelflowib-2.93 → modelflowib-2.94}/model_dynare.py +0 -0
  14. {modelflowib-2.93 → modelflowib-2.94}/model_financial_stability.py +0 -0
  15. {modelflowib-2.93 → modelflowib-2.94}/model_latex.py +0 -0
  16. {modelflowib-2.93 → modelflowib-2.94}/model_latex_class.py +0 -0
  17. {modelflowib-2.93 → modelflowib-2.94}/model_parquet_mixin.py +0 -0
  18. {modelflowib-2.93 → modelflowib-2.94}/modelcausality.py +0 -0
  19. {modelflowib-2.93 → modelflowib-2.94}/modelclass.py +0 -0
  20. {modelflowib-2.93 → modelflowib-2.94}/modelconstruct.py +0 -0
  21. {modelflowib-2.93 → modelflowib-2.94}/modeldash.py +0 -0
  22. {modelflowib-2.93 → modelflowib-2.94}/modeldashboot.py +0 -0
  23. {modelflowib-2.93 → modelflowib-2.94}/modeldashsidebar.py +0 -0
  24. {modelflowib-2.93 → modelflowib-2.94}/modeldekom.py +0 -0
  25. {modelflowib-2.93 → modelflowib-2.94}/modeldiff.py +0 -0
  26. {modelflowib-2.93 → modelflowib-2.94}/modeldisplay.py +0 -0
  27. {modelflowib-2.93 → modelflowib-2.94}/modelestimation.py +0 -0
  28. {modelflowib-2.93 → modelflowib-2.94}/modelfactorize.py +0 -0
  29. {modelflowib-2.93 → modelflowib-2.94}/modelflowib.egg-info/SOURCES.txt +0 -0
  30. {modelflowib-2.93 → modelflowib-2.94}/modelflowib.egg-info/dependency_links.txt +0 -0
  31. {modelflowib-2.93 → modelflowib-2.94}/modelflowib.egg-info/requires.txt +0 -0
  32. {modelflowib-2.93 → modelflowib-2.94}/modelflowib.egg-info/top_level.txt +0 -0
  33. {modelflowib-2.93 → modelflowib-2.94}/modelgrab.py +0 -0
  34. {modelflowib-2.93 → modelflowib-2.94}/modelgrabexcel.py +0 -0
  35. {modelflowib-2.93 → modelflowib-2.94}/modelgrabgdx.py +0 -0
  36. {modelflowib-2.93 → modelflowib-2.94}/modelgrabgekkobank.py +0 -0
  37. {modelflowib-2.93 → modelflowib-2.94}/modelgrabwf2.py +0 -0
  38. {modelflowib-2.93 → modelflowib-2.94}/modelhelp.py +0 -0
  39. {modelflowib-2.93 → modelflowib-2.94}/modelhtml.py +0 -0
  40. {modelflowib-2.93 → modelflowib-2.94}/modelinput_shiny.py +0 -0
  41. {modelflowib-2.93 → modelflowib-2.94}/modelinvert.py +0 -0
  42. {modelflowib-2.93 → modelflowib-2.94}/modeljupyter.py +0 -0
  43. {modelflowib-2.93 → modelflowib-2.94}/modeljupytermagic.py +0 -0
  44. {modelflowib-2.93 → modelflowib-2.94}/modelmacrograb.py +0 -0
  45. {modelflowib-2.93 → modelflowib-2.94}/modelmanipulation.py +0 -0
  46. {modelflowib-2.93 → modelflowib-2.94}/modelmf.py +0 -0
  47. {modelflowib-2.93 → modelflowib-2.94}/modelnet.py +0 -0
  48. {modelflowib-2.93 → modelflowib-2.94}/modelnormalize.py +0 -0
  49. {modelflowib-2.93 → modelflowib-2.94}/modelode.py +0 -0
  50. {modelflowib-2.93 → modelflowib-2.94}/modelpattern.py +0 -0
  51. {modelflowib-2.93 → modelflowib-2.94}/modelreport.py +0 -0
  52. {modelflowib-2.93 → modelflowib-2.94}/modeluserfunk.py +0 -0
  53. {modelflowib-2.93 → modelflowib-2.94}/modelvis.py +0 -0
  54. {modelflowib-2.93 → modelflowib-2.94}/modelwidget.py +0 -0
  55. {modelflowib-2.93 → modelflowib-2.94}/modelwidget_core.py +0 -0
  56. {modelflowib-2.93 → modelflowib-2.94}/modelwidget_input.py +0 -0
  57. {modelflowib-2.93 → modelflowib-2.94}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.93
3
+ Version: 2.94
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -2366,7 +2366,8 @@ class Makemodel(BaseExplode):
2366
2366
  self.markdown_model = (mfmod_list_to_codeblock(self.original_statements))
2367
2367
 
2368
2368
  elif self.type_input == 'single':
2369
- extracted_frml = self.original_statements.replace('\n', '')
2369
+ # join the lines with a space, so a tag ending a line stays apart from the equation
2370
+ extracted_frml = ' '.join(line.strip() for line in self.original_statements.splitlines()).strip()
2370
2371
  self.markdown_model = ''
2371
2372
 
2372
2373
  else:
@@ -2685,6 +2686,46 @@ class Makemodel(BaseExplode):
2685
2686
  cols[name] = ser
2686
2687
  return pd.DataFrame(cols)
2687
2688
 
2689
+ def actual_fitted(self, level: bool = True):
2690
+ """Actual, fitted and residuals of all estimated equations as one DataFrame.
2691
+
2692
+ The columns have two levels: the endogenous variable, and ``Actual``,
2693
+ ``Fitted``, ``Residuals``. Take one equation with
2694
+ ``mm.actual_fitted()['CON']``.
2695
+
2696
+ level : bool, default True
2697
+ True: the endogenous variable itself. When the left-hand side is
2698
+ ``DLOG(CON)`` this is ``CON``, its one-step fitted value (the
2699
+ estimated equation solved for ``CON``, lagged values are the
2700
+ actual ones) and the residual in the units of ``CON``.
2701
+ False: the left-hand side as it was estimated (``DLOG(CON)``).
2702
+
2703
+ Each equation covers its own estimation sample, so rows outside it
2704
+ are NaN. An equation which can't be solved for its variable is left
2705
+ out with ``level=True``.
2706
+
2707
+ Per-equation access: ``rec['estimator_object'].af_level_df`` (or
2708
+ ``.af_df`` and ``.residuals_df``) on the entries of
2709
+ :attr:`estimation_records`; ``modelestimator_new.plot_actual_vs_fitted
2710
+ (est, level=True)`` draws the chart of the report.
2711
+ """
2712
+ import pandas as pd
2713
+ frames = {}
2714
+ for rec in self.estimation_records:
2715
+ est = rec.get('estimator_object')
2716
+ if not _is_estimator_backend_instance(est):
2717
+ continue
2718
+ frame = est.af_level_df if level else est.af_df.join(est.residuals_df)
2719
+ if frame is None:
2720
+ continue
2721
+ name = getattr(est, 'endo_var', '') or rec.get('frmlname', '')
2722
+ if level: # the variable is the outer column level
2723
+ frame = frame.rename(columns=lambda c: c.removeprefix(f'{est.endo_var}__'))
2724
+ frames[name] = frame
2725
+ if not frames:
2726
+ return pd.DataFrame()
2727
+ return pd.concat(frames, axis=1)
2728
+
2688
2729
  @property
2689
2730
  def markdown_with_estimation(self) -> str:
2690
2731
  """Original Markdown input with compact estimation tables inserted.
@@ -3085,6 +3126,18 @@ class Listmodels(BaseExplode):
3085
3126
  return pd.DataFrame()
3086
3127
  return pd.concat(frames, axis=1)
3087
3128
 
3129
+ def actual_fitted(self, level: bool = True):
3130
+ """Actual, fitted and residuals of all member Makemodels, combined column-wise.
3131
+
3132
+ See :meth:`Makemodel.actual_fitted` for the layout.
3133
+ """
3134
+ import pandas as pd
3135
+ frames = [mex.actual_fitted(level=level) for mex in self.makemodels]
3136
+ frames = [f for f in frames if not f.empty]
3137
+ if not frames:
3138
+ return pd.DataFrame()
3139
+ return pd.concat(frames, axis=1)
3140
+
3088
3141
  @property
3089
3142
  def markdown_with_estimation(self) -> str:
3090
3143
  """Concatenate member Makemodel markdown-with-estimation strings."""
@@ -1378,6 +1378,67 @@ class EstimatorBackend(Eq_parent, ABC):
1378
1378
  .rename(columns=lambda s: s.lower().capitalize())
1379
1379
  )
1380
1380
 
1381
+ # ---- A/F and residuals of the endogenous variable itself ---------------
1382
+
1383
+ @cached_property
1384
+ def lhs_is_transformed(self) -> bool:
1385
+ """True when the left-hand side is a transformation of the endogenous
1386
+ variable (``DLOG(CON)``, ``DIFF(CON)``, ``LOG(CON/P)`` ...), so
1387
+ :attr:`af_df` is not in the units of the variable itself."""
1388
+ lhs = self.org_eq_clean.split("=", 1)[0]
1389
+ return "".join(lhs.split()).upper() != self.endo_var.upper()
1390
+
1391
+ @cached_property
1392
+ def af_level_df(self) -> Optional[pd.DataFrame]:
1393
+ """Actual, fitted and residuals of the endogenous variable itself.
1394
+
1395
+ :attr:`af_df` and :attr:`residuals_df` are in the units of the
1396
+ left-hand side as it was estimated, e.g. ``DLOG(CON)``. Here the
1397
+ estimated equation is solved for the variable (``CON``):
1398
+
1399
+ * ``Actual`` the variable,
1400
+ * ``Fitted`` the normalized equation with the estimated
1401
+ coefficients, evaluated on the data. Lagged values of the variable
1402
+ are the actual ones, so this is the one-step (static) fit, not a
1403
+ dynamic simulation,
1404
+ * ``Residuals`` ``Actual - Fitted``, in the units of the variable.
1405
+
1406
+ The columns are prefixed with the variable name and ``__``
1407
+ (``CON__Actual``, ``CON__Fitted``, ``CON__Residuals``), so frames of
1408
+ several equations can be joined side by side.
1409
+
1410
+ When the left-hand side is the variable itself the frame holds the
1411
+ same numbers as :attr:`af_df` and :attr:`residuals_df`. ``None`` (and
1412
+ a message) when the equation can't be solved for the variable.
1413
+ """
1414
+ prefix = f"{self.endo_var}__"
1415
+ if not self.lhs_is_transformed:
1416
+ return self.af_df.join(self.residuals_df).add_prefix(prefix)
1417
+ start, end = self.smpl
1418
+ try:
1419
+ normalized = nz.normal(
1420
+ self.org_eq_baked, add_add_factor=False
1421
+ ).normalized
1422
+ lhs, rhs = normalized.split("=", 1)
1423
+ if lhs.strip().upper() != self.endo_var.upper():
1424
+ raise ValueError(f"normalized to {lhs.strip()!r}")
1425
+ res = self.eq_var_df.mfcalc(
1426
+ f"<{start},{end}> FITTED_LEVEL = {rhs}"
1427
+ )
1428
+ out = pd.DataFrame({
1429
+ "Actual": res.loc[start:end, self.endo_var],
1430
+ "Fitted": res.loc[start:end, "FITTED_LEVEL"],
1431
+ })
1432
+ except Exception as e:
1433
+ print(
1434
+ f"{type(self).__name__}({self.endo_var}): no actual/fitted "
1435
+ f"for the variable itself, the equation could not be solved "
1436
+ f"for it ({type(e).__name__}: {str(e)[:100]})"
1437
+ )
1438
+ return None
1439
+ out["Residuals"] = out["Actual"] - out["Fitted"]
1440
+ return out.add_prefix(prefix)
1441
+
1381
1442
  @cached_property
1382
1443
  def estimation_smpl(self) -> Tuple[Any, Any]:
1383
1444
  """The usable sample, derived from non-missing residuals.
@@ -2088,6 +2149,11 @@ class LSResult:
2088
2149
  title_html = self._build_title_html()
2089
2150
  summary_html = self.olsmodel._render_summary_html()
2090
2151
  plot_html = self._build_plot_html(plot_format)
2152
+ # the same for the endogenous variable itself, when the left-hand
2153
+ # side is a transformation of it (DLOG(CON) -> CON)
2154
+ if (self.olsmodel.lhs_is_transformed
2155
+ and self.olsmodel.af_level_df is not None):
2156
+ plot_html += self._build_plot_html(plot_format, level=True)
2091
2157
  return title_html + summary_html + plot_html
2092
2158
 
2093
2159
  def show(self, plot_format: str = "svg") -> None:
@@ -2124,17 +2190,21 @@ class LSResult:
2124
2190
  f"<pre><code>{normalized}</code></pre></p>"
2125
2191
  )
2126
2192
 
2127
- def _build_plot_html(self, plot_format: str) -> str:
2193
+ def _build_plot_html(self, plot_format: str, level: bool = False) -> str:
2128
2194
  buf = BytesIO()
2129
2195
  plot_actual_vs_fitted(
2130
2196
  self.olsmodel, save_to=buf, format=plot_format,
2131
- figsize=(8, 5), dpi=150,
2197
+ figsize=(8, 5), dpi=150, level=level,
2132
2198
  )
2133
2199
  buf.seek(0)
2134
2200
  b64 = base64.b64encode(buf.read()).decode("utf-8")
2135
2201
  mime = "svg+xml" if plot_format == "svg" else "png"
2202
+ heading = (
2203
+ f"Actual vs Fitted Plot: {self.olsmodel.endo_var} itself "
2204
+ "(one-step fit)" if level else "Actual vs Fitted Plot"
2205
+ )
2136
2206
  return (
2137
- "<h3>Actual vs Fitted Plot</h3>"
2207
+ f"<h3>{heading}</h3>"
2138
2208
  f'<img style="max-width:100%; height:auto;" '
2139
2209
  f'src="data:image/{mime};base64,{b64}" />'
2140
2210
  )
@@ -2440,28 +2510,46 @@ def plot_actual_vs_fitted(
2440
2510
  format: str = "png",
2441
2511
  figsize: Tuple[float, float] = (7, 3),
2442
2512
  dpi: int = 150,
2513
+ level: bool = False,
2443
2514
  ):
2444
2515
  """Plot Actual vs Fitted (top panel) and Residuals (bottom panel).
2445
2516
 
2517
+ ``level=False`` plots the left-hand side as it was estimated (``af_df``,
2518
+ ``residuals_df``), ``level=True`` the endogenous variable itself
2519
+ (``af_level_df``): ``CON`` when the left-hand side is ``DLOG(CON)``.
2520
+
2446
2521
  If ``save_to`` is a file-like object, the plot is saved there and the
2447
2522
  figure is closed; otherwise it is shown interactively.
2448
2523
  """
2524
+ if level:
2525
+ if emodel.af_level_df is None:
2526
+ raise ValueError(
2527
+ f"{emodel.endo_var}: no actual/fitted for the variable itself"
2528
+ )
2529
+ prefix = f"{emodel.endo_var}__"
2530
+ af_df = residuals_df = emodel.af_level_df.rename(
2531
+ columns=lambda c: c.removeprefix(prefix))
2532
+ title = f"Actual vs Fitted: {emodel.endo_var} itself (one-step fit)"
2533
+ else:
2534
+ af_df, residuals_df = emodel.af_df, emodel.residuals_df
2535
+ title = f"Actual vs Fitted: {emodel.endo_var}"
2536
+
2449
2537
  fig = plt.figure(figsize=figsize, dpi=dpi)
2450
2538
  gs = GridSpec(3, 1, height_ratios=[2, 0.1, 1])
2451
2539
 
2452
2540
  ax1 = fig.add_subplot(gs[0])
2453
2541
  ax2 = fig.add_subplot(gs[2], sharex=ax1)
2454
2542
 
2455
- ax1.plot(emodel.af_df.index, emodel.af_df["Actual"],
2543
+ ax1.plot(af_df.index, af_df["Actual"],
2456
2544
  label="Actual", linewidth=2)
2457
- ax1.plot(emodel.af_df.index, emodel.af_df["Fitted"],
2545
+ ax1.plot(af_df.index, af_df["Fitted"],
2458
2546
  label="Fitted", linestyle="--")
2459
- ax1.set_title(f"Actual vs Fitted: {emodel.endo_var}")
2547
+ ax1.set_title(title)
2460
2548
  ax1.set_ylabel(emodel.endo_var)
2461
2549
  ax1.grid(True)
2462
2550
  ax1.legend()
2463
2551
 
2464
- ax2.plot(emodel.residuals_df.index, emodel.residuals_df["Residuals"],
2552
+ ax2.plot(residuals_df.index, residuals_df["Residuals"],
2465
2553
  color="gray")
2466
2554
  ax2.axhline(0, color="red", linestyle="--", linewidth=1)
2467
2555
  ax2.set_title("Residuals")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.93
3
+ Version: 2.94
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -541,12 +541,175 @@ class newton_diff():
541
541
  dmelt['lag'] = dmelt['lag'].astype(int)
542
542
  return dmelt
543
543
 
544
-
544
+ def _stacked_plan(self, per, is_residual_eq=None):
545
+ '''The fill plan of the stacked jacobian: its sparsity pattern for the periods per.
546
+
547
+ The pattern depends on the derivative model and the periods only, so it is made once and
548
+ kept in self._stackplan. _stacked_from_plan fills in the values.
549
+
550
+ The diagonal -1 is a part of the plan: on all rows of a normalized model, else on the
551
+ rows which are not residual equations when is_residual_eq is given (one value per
552
+ equation, or per equation and period) - the same rule as get_solvestacked.
553
+
554
+ A derivative is placed like get_diff_melted and get_diff_mat_tot do: row = number*nvar+var,
555
+ column = (number+lag)*nvar+pvar, dropped when number+lag is outside the periods. A
556
+ derivative with respect to a variable at a fixed period (X(2002)) goes to the column of
557
+ that period, and is dropped when the period is not among the periods (fixed_to_rowlag).
558
+
559
+ Returns a namespace with:
560
+ key what the plan was made for
561
+ keep bool (periods, derivatives): the derivative is in the matrix.
562
+ Derivatives in the order of sorted(self.diff_model.endogene)
563
+ slot the place in data of every kept value, in the order of values[keep]
564
+ const_slot the places in data of the diagonal -1
565
+ indices, indptr, shape, nnz the csc structure. Entries on the same place share a slot
566
+ '''
567
+ from types import SimpleNamespace
568
+
569
+ per = pd.Index(per)
570
+ normalized = bool(self.mmodel.normalized)
571
+ nres = None if (is_residual_eq is None or normalized) else np.asarray(is_residual_eq,dtype=bool)
572
+ key = (tuple(per), normalized, None if nres is None else nres.tobytes())
573
+ plan = getattr(self,'_stackplan',None)
574
+ if plan is not None and plan.key == key:
575
+ return plan
576
+
577
+ with ttimer('make the fill plan of the stacked jacobian',self.timeit):
578
+ coltup = [split_diffname(c) for c in sorted(self.diff_model.endogene)]
579
+ endo = self.mmodel.endogene
580
+ var = np.array([self.placdic[v] for v,p,l in coltup],dtype=np.int64)
581
+ pvar = np.array([self.placdic[p+'___RES' if (p+'___RES' in endo) else p] for v,p,l in coltup],dtype=np.int64)
582
+ nper,nvar = len(per),len(self.endovar)
583
+ size = nper*nvar
584
+
585
+ number = np.arange(nper,dtype=np.int64)[:,None]
586
+ fixed = [pt.is_fixed_lag(l) for v,p,l in coltup]
587
+ lag = np.array([0 if f else l for (v,p,l),f in zip(coltup,fixed)],dtype=np.int64)
588
+ pnumber = number + lag[None,:] # the number of the period of pvar
589
+ for j in np.flatnonzero(fixed):
590
+ try:
591
+ pnumber[:,j] = pt.fixed_period_loc(per,coltup[j][2])
592
+ except KeyError:
593
+ pnumber[:,j] = -1 # not among the periods: data
594
+ keep = (pnumber >= 0) & (pnumber <= nper-1)
595
+ row = (number*nvar + var[None,:])[keep]
596
+ col = (pnumber*nvar + pvar[None,:])[keep]
597
+
598
+ if normalized:
599
+ diag = np.arange(size,dtype=np.int64)
600
+ elif nres is not None:
601
+ if len(nres) == nvar:
602
+ nres = np.tile(nres,nper)
603
+ if len(nres) != size:
604
+ raise ValueError(f'is_residual_eq has {len(nres)} values, it should have {nvar} (equations) or {size} (equations*periods)')
605
+ diag = np.flatnonzero(~nres).astype(np.int64)
606
+ else:
607
+ diag = np.zeros(0,dtype=np.int64)
608
+
609
+ # column-major and rows sorted = the order of csc. Entries on the same place get the same slot
610
+ place = np.concatenate([col,diag])*size + np.concatenate([row,diag])
611
+ uniq,inv = np.unique(place,return_inverse=True)
612
+ inv = inv.ravel()
613
+ nnz = len(uniq)
614
+ itype = np.int32 if max(nnz,size) < np.iinfo(np.int32).max else np.int64
615
+ self._stackplan = SimpleNamespace(
616
+ key = key,
617
+ keep = keep,
618
+ slot = inv[:len(row)],
619
+ const_slot = inv[len(row):],
620
+ indices = (uniq % size).astype(itype),
621
+ indptr = np.searchsorted(uniq // size,np.arange(size+1)).astype(itype),
622
+ shape = (size,size),
623
+ nnz = nnz)
624
+ return self._stackplan
625
+
626
+ def _stacked_from_plan(self, plan, values):
627
+ '''The stacked jacobian from a fill plan (_stacked_plan) and the values of the derivatives:
628
+ a (periods, derivatives) array with the derivatives in the order of sorted(self.diff_model.endogene).
629
+
630
+ NaN counts as 0.0, like the fillna in get_diff_melted, inf is left as it is. Values on the
631
+ same place are added. A new matrix with its own arrays every time: no caller can damage the plan.'''
632
+ v = np.asarray(values,dtype=float)[plan.keep]
633
+ v[np.isnan(v)] = 0.0
634
+ data = np.bincount(plan.slot,weights=v,minlength=plan.nnz)
635
+ data[plan.const_slot] -= 1.0
636
+ return sp.sparse.csc_matrix((data,plan.indices.copy(),plan.indptr.copy()),shape=plan.shape)
637
+
638
+
639
+
640
+ def get_diff_mat_tot(self, df=None, is_residual_eq=None):
641
+ """
642
+ The stacked Jacobian for all periods in the model's `current_per`, a new csc matrix.
643
+
644
+ The sparsity pattern is made once (_stacked_plan), a call evaluates the derivative model
645
+ and places the values (_stacked_from_plan).
646
+
647
+ Args:
648
+ df (pandas.DataFrame, optional): Input DataFrame for the calculations. If not provided,
649
+ the model's internal DataFrame (`self.df`) is used.
650
+ is_residual_eq (bool array, optional): True for the residual equations (___RES), one
651
+ value per equation or per equation and period.
652
+
653
+ Notes:
654
+ - The diagonal -1 is in the matrix: on all rows of a normalized model, else on the rows
655
+ which are not residual equations when is_residual_eq is given.
656
+ - A derivative which evaluates to zero stays in the pattern, so the pattern is the same
657
+ from call to call.
658
+ - self.stacked is the matrix returned.
659
+ - get_diff_mat_tot_melt is the former version (without is_residual_eq).
660
+ - self.stackfill = 'melt' makes the matrix the former way (get_diff_mat_tot_melt and
661
+ the diagonal subtracted afterwards); 'plan' or not set is the fill plan.
662
+ """
663
+ stackfill = (getattr(self,'stackfill',None) or 'plan').lower()
664
+ if stackfill not in ('plan','melt'):
665
+ raise ValueError(f"stackfill={stackfill!r} is not known, use 'plan' (the fill plan) or 'melt' (the former fill)")
666
+ if stackfill == 'melt':
667
+ with ttimer('stacked jacobian the former way (stackfill=melt)',self.timeit):
668
+ # normalized: get_diff_mat_tot_melt has subtracted the identity
669
+ stacked = self.get_diff_mat_tot_melt(df=df)
670
+ if is_residual_eq is not None and not self.mmodel.normalized:
671
+ notres = ~np.asarray(is_residual_eq,dtype=bool)
672
+ if len(notres) == self.nvar:
673
+ notres = np.tile(notres,len(self.numbers))
674
+ if len(notres) != stacked.shape[0]:
675
+ raise ValueError(f'is_residual_eq has {len(notres)} values, expected {self.nvar} or {stacked.shape[0]}')
676
+ stacked = stacked - sp.sparse.diags(notres.astype(float),format='csc')
677
+ self.stacked = stacked.tocsc()
678
+ return self.stacked
679
+ _per = self.mmodel.current_per
680
+ if not hasattr(_per,'__iter__'):
681
+ _per = [_per]
682
+ _df = self.df if type(df) != pd.DataFrame else df
683
+ self.df = _df
684
+ _df = _df.pipe(lambda df0: df0.rename(columns={c: c.upper() for c in df0.columns}))
685
+ # as in get_diff_melted: the columns of the derivative model are added before the call
686
+ _df = insertModelVar(_df, self.diff_model)
687
+
688
+ self.diff_model.current_per = _per
689
+ with ttimer('calculate derivatives',self.timeit):
690
+ reseval = self.diff_model.res_ng if self.use_res_ng else self.diff_model.res
691
+ out = reseval(_df,silent=self.silent,stats=0,ljit=self.ljit,chunk=self.nchunk)
692
+ with ttimer('stacked jacobian from the fill plan',self.timeit):
693
+ rows = out.index.get_indexer(_per)
694
+ if (rows < 0).any():
695
+ raise KeyError('Periods of the stacked jacobian are not in the dataframe index')
696
+ cpos = out.columns.get_indexer(sorted(self.diff_model.endogene))
697
+ per = out.index[rows]
698
+ self.numbers = list(range(len(per)))
699
+ self.maxnumber = len(per)-1
700
+ self.numbers_to_date = dict(enumerate(per))
701
+ self.nvar = len(self.endovar)
702
+ values = out.iloc[rows,cpos].to_numpy(dtype=float)
703
+ plan = self._stacked_plan(per,is_residual_eq)
704
+ self.stacked = self._stacked_from_plan(plan,values)
705
+ return self.stacked
545
706
 
546
- def get_diff_mat_tot(self, df=None):
707
+ def get_diff_mat_tot_melt(self, df=None):
547
708
  """
709
+ The former get_diff_mat_tot, kept to test the new one against.
710
+
548
711
  Generate a stacked Jacobian matrix for the entire model across the current period.
549
-
712
+
550
713
  This function constructs a sparse matrix representing the Jacobian for the specified
551
714
  period range, either normalized or unnormalized, depending on the model's configuration.
552
715
 
@@ -749,11 +912,8 @@ class newton_diff():
749
912
  # if update or not hasattr(self,'stacked'):
750
913
 
751
914
  # breakpoint()
752
- if is_residual_eq is not None and not self.mmodel.normalized :
753
- diag_mask = sp.sparse.diags((~is_residual_eq).astype(float))
754
- self.stacked = self.get_diff_mat_tot(df=df) - diag_mask
755
- else:
756
- self.stacked = self.get_diff_mat_tot(df=df)
915
+ # the diagonal -1 of the rows which are not residual equations is placed by get_diff_mat_tot
916
+ self.stacked = self.get_diff_mat_tot(df=df,is_residual_eq=is_residual_eq)
757
917
  if getattr(self,'factorizer',None) == 'umfpack':
758
918
  self.solvestacked = self._factorize_umfpack(self.stacked)
759
919
  elif self._other_factorizer():
@@ -2579,6 +2579,12 @@ class NewtonStackSolver(SolverBase):
2579
2579
  """Dump order: the declared (base) endogenous names of the Jacobian object."""
2580
2580
  return ctx.model.ng_newton_diff_stack.declared_endo_list
2581
2581
 
2582
+ def conv_order(self, ctx):
2583
+ """Convergence order: the declared (base) endogenous names of the Jacobian object.
2584
+ iterate tests convergence on all stacked unknowns; the base-class default
2585
+ (model.coreorder) would start the feedback-set search, minutes on a large core."""
2586
+ return ctx.model.ng_newton_diff_stack.declared_endo_list
2587
+
2582
2588
  def build_evaluator(self, ctx):
2583
2589
  """Compile the evaluators. With ``split=1`` (the default) the
2584
2590
  stacked-split triple from ``_prepare_split``: prolog and epilog as
@@ -2628,6 +2634,9 @@ class NewtonStackSolver(SolverBase):
2628
2634
  ctx.is_residual_eq_stacked = np.tile(is_residual_eq, len(ctx.sol_periode))
2629
2635
 
2630
2636
  m.ng_newton_diff_stack.timeit = timeit
2637
+ # how the stacked Jacobian is filled: 'plan' (fill plan, default) or
2638
+ # 'melt' (the former fill). Read at every build of the matrix.
2639
+ m.ng_newton_diff_stack.stackfill = opt("stackfill", "plan")
2631
2640
  # ordering of the stacked LU (get_solvestacked defaults to 'NATURAL',
2632
2641
  # band-preserving, when unset) and factorization engine. Applied on
2633
2642
  # every call; when they change, the cached stacked solver is dropped --
@@ -3182,6 +3191,7 @@ class NewtonStackFbminSolver(SolverBase):
3182
3191
  delattr(m, attr)
3183
3192
  diff = m.ng_newton_diff_stack
3184
3193
  diff.timeit = timeit
3194
+ diff.stackfill = opt("stackfill", "plan") # as in newtonstack_ng
3185
3195
  per = ctx.sol_periode
3186
3196
 
3187
3197
  struct_stale = (opt("newton_reset", False)
@@ -3393,14 +3403,11 @@ class NewtonStackFbminSolver(SolverBase):
3393
3403
  struct = m.ng_stackfbmin_struct
3394
3404
  fb, dag = struct["fb_nodes"], struct["dag_nodes"]
3395
3405
 
3396
- stacked = diff.get_diff_mat_tot(df=df)
3397
- if not m.normalized:
3398
- # normalized rows in a mixed system still need their -I term
3399
- isres = np.array([v.endswith("___RES") for v in diff.endovar],
3400
- dtype=bool)
3401
- isres_stacked = np.tile(isres, struct["n_per"])
3402
- stacked = stacked - sps.diags((~isres_stacked).astype(float))
3403
- stacked = stacked.tocsr()
3406
+ # normalized rows in a mixed system still need their -I term:
3407
+ # get_diff_mat_tot places it from the mask of residual equations
3408
+ isres = np.array([v.endswith("___RES") for v in diff.endovar],
3409
+ dtype=bool)
3410
+ stacked = diff.get_diff_mat_tot(df=df, is_residual_eq=isres).tocsr()
3404
3411
 
3405
3412
  jred = stacked[fb][:, fb].toarray()
3406
3413
  if len(dag):
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "modelflowib" # PyPI distribution name
7
- version = "2.93" # OVERWRITTEN at build time from MF_VERSION
7
+ version = "2.94" # OVERWRITTEN at build time from MF_VERSION
8
8
  # (set_pyproject_version.py / make_pip.py -v)
9
9
  description = "A tool to solve and manage dynamic economic and financial models"
10
10
  readme = "README.md"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes